From 19faed1f47b3b7a8f4b28d4f99f8c4d464b2d97c Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Fri, 7 Aug 2026 16:37:37 -0300 Subject: [PATCH 01/52] =?UTF-8?q?feat(k8s):=20trace-instrument=20the=20wor?= =?UTF-8?q?kflow=20=E2=80=94=20every=20error=20clear,=20waits=20live?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The platform now nests each custom-scope workflow inside its provisioning run (SWM injects the trace carrier into the action parameters; the np CLI adopts it and traces one step per workflow fragment). What was missing is the WHY: a failed step read as "exited with status 1" while the real reason scrolled past in the log. One seam fixes that structurally: `log error` — every one of its 700+ call sites across the k8s scripts now also lands the message as a tracing.error facet ON the step it happened in, via the vendored shell SDK adopting NP_TRACE at call time. A silent failure (no log error on the way down) is covered by an ERR/EXIT trap pair that reports the failing command itself. Failures in a custom scope now read exactly like native ones: which step, and what went wrong. The two long waits (ALB active, deployment active) also mark their step `waiting` with live progress labels, flushed per beat with a hard 2s bound so a down tracing API costs at most ~6% of a wait's poll budget and never extends its wall-clock timeout. All of it is best-effort and inert by default: no nptrace.sh, no NP_API_KEY, or no NP_TRACE means plain logging, byte-identical to before (asserted byte-for-byte in tests). Heartbeat call sites are guarded so overrides reusing these scripts without k8s/logging stay clean. --- k8s/deployment/wait_deployment_active | 3 + k8s/logging | 92 ++ k8s/scope/networking/wait_for_alb | 9 + k8s/utils/tests/trace_logging.bats | 143 +++ nptrace.sh | 1426 +++++++++++++++++++++++++ 5 files changed, 1673 insertions(+) create mode 100644 k8s/utils/tests/trace_logging.bats create mode 100755 nptrace.sh diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index 7575a603..e6cc062c 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -150,6 +150,9 @@ while true; do if [ "$iteration" -eq 1 ] || [ $(( iteration % HEARTBEAT_INTERVAL )) -eq 0 ]; then elapsed_s=$(( iteration * 10 )) log info "⏳ Still waiting — Ready: $ready/$desired, Available: $current/$desired (attempt $iteration/$MAX_ITERATIONS, ${elapsed_s}s elapsed)" + if command -v np_scope_wait_heartbeat >/dev/null 2>&1; then + np_scope_wait_heartbeat "deployment-active" "$elapsed_s" "$TIMEOUT" "ready-$ready-of-$desired" + fi fi POD_SELECTOR="deployment_id=${DEPLOYMENT_ID}" diff --git a/k8s/logging b/k8s/logging index d0df55d7..18db608f 100644 --- a/k8s/logging +++ b/k8s/logging @@ -38,4 +38,96 @@ log() { echo "$message" fi fi + + # Every `log error`, anywhere in any script, also lands on the TRACE — as a + # tracing.error facet on the workflow step it happened in. This is what makes + # a failed custom scope read like a failed native one: not just "step exited + # 1", but the actual message. A no-op unless the platform launched this + # workflow traced (see the tracing section below). + if [ "$msg_num" -ge 3 ]; then + _np_scopes_trace_error "$message" || true + fi } + +# ============================================================================= +# Tracing — best-effort, structural +# +# When the platform launches a workflow, NP_TRACE carries the trace context of +# the step each fragment runs inside (the np CLI re-points it per step). With +# the vendored shell SDK sourced: +# +# • every `log error` lands as a tracing.error facet ON that step +# • an uncaught command failure surfaces the failing command itself +# • long waits report live progress (np_scope_wait_heartbeat) +# +# All of it is best-effort: no SDK file, no NP_API_KEY, or no NP_TRACE means +# plain logging, byte-identical to before. A down tracing API never blocks the +# workflow: every emit is a local file write, and flushes are hard-bounded. +# ============================================================================= + +# Record an observed failure on the step this fragment runs inside. Adoption is +# per-call on purpose: NP_TRACE changes as the CLI moves between steps, so the +# error must attach to the step CURRENT at the moment it happened. +_np_scopes_trace_error() { + command -v np_trace_adopt >/dev/null 2>&1 || return 0 + [ -n "${NP_TRACE:-}" ] || return 0 + local _lt_node + _lt_node=$(np_trace_adopt 2>/dev/null) || return 0 + [ -n "$_lt_node" ] || return 0 + np_trace_error "$_lt_node" --message "$1" ${2:+--code "$2"} + # Remember which step already carries a real message, so the exit trap does + # not shadow it with a generic one. + _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" + return 0 +} + +# np_scope_wait_heartbeat [state] +# +# Mark the current step `waiting` and record progress, flushed immediately so +# the wait is visible LIVE, not after the workflow ends. The flush is bounded +# to 2s per beat: with the API down, a 30s-cadence wait loses at most ~6% of +# its poll budget — the deadline is wall-clock, so the timeout is never +# extended. Always defined; a no-op when the workflow is untraced. +np_scope_wait_heartbeat() { + command -v np_trace_adopt >/dev/null 2>&1 || return 0 + [ -n "${NP_TRACE:-}" ] || return 0 + local _hb_node + _hb_node=$(np_trace_adopt 2>/dev/null) || return 0 + [ -n "$_hb_node" ] || return 0 + np_trace_labels "$_hb_node" \ + "wait.what=${1:-}" "wait.elapsed_s=${2:-0}" "wait.timeout_s=${3:-0}" \ + ${4:+"wait.state=$4"} + np_trace_waiting "$_hb_node" + NP_TRACE_FLUSH_TIMEOUT=2 np_trace_flush + return 0 +} + +# A silent failure (no `log error` on the way down) must still be clear: the +# ERR trap remembers the last failing top-level command, and the EXIT trap +# reports it against the step that was current when the shell died. +_np_scopes_on_err() { + _NP_SCOPES_LAST_ERR="$BASH_COMMAND (exit ${1:-1})" +} + +_np_scopes_on_exit() { + local _ex_rc="${1:-0}" + if [ "$_ex_rc" -ne 0 ] && [ "${_NP_SCOPES_ERRED_ON:-}" != "${NP_TRACE:-}" ]; then + _np_scopes_trace_error \ + "${_NP_SCOPES_LAST_ERR:-workflow shell exited with status $_ex_rc}" || true + fi + np_trace_flush +} + +_NP_SCOPES_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [ -z "${NP_TRACE_LOADED:-}" ] \ + && [ -f "$_NP_SCOPES_ROOT/nptrace.sh" ] \ + && [ -n "${NP_API_KEY:-}" ] \ + && [ -n "${NP_TRACE:-}" ]; then + # shellcheck source=/dev/null + . "$_NP_SCOPES_ROOT/nptrace.sh" + # --no-trap: the exit flush is ours, so the uncaught-failure report and the + # flush share ONE trap in a defined order. + np_trace_init --producer "nullplatform-scopes@1" --api-key "$NP_API_KEY" --no-trap + trap '_np_scopes_on_err $?' ERR + trap '_np_scopes_on_exit $?' EXIT +fi diff --git a/k8s/scope/networking/wait_for_alb b/k8s/scope/networking/wait_for_alb index 0909f39a..7cbb4f4f 100644 --- a/k8s/scope/networking/wait_for_alb +++ b/k8s/scope/networking/wait_for_alb @@ -49,6 +49,12 @@ polls_since_heartbeat=0 heartbeats_emitted=0 log info "⏳ Waiting up to ${TIMEOUT_SECONDS}s for ALB '$ALB_NAME' to become active..." +# Flip the step to `waiting` in the trace right away — an operator watching the +# provision sees WHAT it is blocked on, live, not after the workflow ends. +# (Guarded: overrides may reuse this script without k8s/logging loaded.) +if command -v np_scope_wait_heartbeat >/dev/null 2>&1; then + np_scope_wait_heartbeat "alb-active" 0 "$TIMEOUT_SECONDS" "pending" +fi state="" alb_arn="" @@ -80,6 +86,9 @@ while [ "$(date +%s)" -lt "$deadline" ]; do heartbeats_emitted=$((heartbeats_emitted + 1)) elapsed=$((heartbeats_emitted * polls_per_heartbeat * poll_interval)) log info "⏳ Still waiting for ALB '$ALB_NAME' to become active (${state:-pending}, ~${elapsed}s elapsed)" + if command -v np_scope_wait_heartbeat >/dev/null 2>&1; then + np_scope_wait_heartbeat "alb-active" "$elapsed" "$TIMEOUT_SECONDS" "${state:-pending}" + fi polls_since_heartbeat=0 fi diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats new file mode 100644 index 00000000..f2336a24 --- /dev/null +++ b/k8s/utils/tests/trace_logging.bats @@ -0,0 +1,143 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for the tracing hooks in k8s/logging +# +# (Lives under utils/tests because the runner discovers k8s//tests; +# the file under test is k8s/logging.) +# +# The contract under test: with the platform's trace context present, every +# `log error` and every uncaught failure lands ON the workflow step as a +# tracing.error facet, waits report progress — and with anything missing, the +# behavior is byte-identical to plain logging. +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + + export LOGGING="$PROJECT_ROOT/k8s/logging" + + # A traced environment, as the np CLI provides it: NP_TRACE points at the + # step this fragment runs inside; the state dir is per-test; the endpoint is + # unroutable so nothing ever leaves the machine and flushes fail fast. + export NP_TRACE="1|trace-9|scope-provision-42~apply-manifests@0.0" + export NP_API_KEY="test-key" + export NP_TRACE_DIR="$BATS_TEST_TMPDIR/nptrace" + export NP_TRACE_BASE_URL="http://127.0.0.1:1" + export NP_TRACE_AUTH_URL="http://127.0.0.1:1" + export NP_TRACE_FLUSH_TIMEOUT=1 + export NP_TRACE_MAX_RETRIES=0 +} + +teardown() { + unset NP_TRACE NP_API_KEY NP_TRACE_DIR NP_TRACE_BASE_URL NP_TRACE_AUTH_URL + unset NP_TRACE_FLUSH_TIMEOUT NP_TRACE_MAX_RETRIES +} + +# Run a snippet in a fresh bash with logging sourced, then print every spooled +# envelope (the spool is the wire: it is what the API would receive). +run_logged() { + run "$BASH" -c " + source '$LOGGING' + $1 + for f in \"\$NP_TRACE_DIR\"/spool/*.json \"\$NP_TRACE_DIR\"/failed/*.json; do + [ -f \"\$f\" ] && cat \"\$f\" && echo + done + true + " +} + +@test "log error lands as a tracing.error facet on the current step" { + run_logged 'log error "❌ kubectl apply failed: forbidden"' + [ "$status" -eq 0 ] + echo "$output" | grep -q '"run_id":"scope-provision-42~apply-manifests@0.0"' + echo "$output" | grep -q '"tracing.error"' + echo "$output" | grep -q 'kubectl apply failed: forbidden' +} + +@test "the error attaches to the step CURRENT at the moment it happened" { + # NP_TRACE moves between steps; adoption is per-call, so a later error must + # land on the later step. + run_logged ' + log error "first failure" + export NP_TRACE="1|trace-9|scope-provision-42~wait-for-alb@0.0" + log error "second failure" + ' + [ "$status" -eq 0 ] + echo "$output" | grep '"first failure"' | grep -q 'apply-manifests@0.0' + echo "$output" | grep '"second failure"' | grep -q 'wait-for-alb@0.0' +} + +@test "an uncaught command failure surfaces the failing command" { + run "$BASH" -c " + source '$LOGGING' + "$BASH" -c 'exit 7' # a real failing command, no log error anywhere + " || true + run "$BASH" -c " + source '$LOGGING' + trap - EXIT # neutralize for inspection after the inner shell + ( source '$LOGGING'; bash -c 'exit 7' ) || true + for f in \"\$NP_TRACE_DIR\"/spool/*.json \"\$NP_TRACE_DIR\"/failed/*.json; do + [ -f \"\$f\" ] && cat \"\$f\" && echo + done + true + " + [ "$status" -eq 0 ] + echo "$output" | grep -q '"tracing.error"' + echo "$output" | grep -q 'exit 7' +} + +@test "a log error already recorded is not shadowed by the exit trap" { + run "$BASH" -c " + ( source '$LOGGING'; log error 'the real reason'; exit 3 ) || true + source '$LOGGING'; trap - EXIT ERR + for f in \"\$NP_TRACE_DIR\"/spool/*.json \"\$NP_TRACE_DIR\"/failed/*.json; do + [ -f \"\$f\" ] && cat \"\$f\" && echo + done + true + " + [ "$status" -eq 0 ] + echo "$output" | grep -q 'the real reason' + # exactly one error facet: the generic exit report stood down + [ "$(echo "$output" | grep -c 'tracing.error')" -eq 1 ] +} + +@test "wait heartbeat marks the step waiting with progress labels" { + run_logged 'np_scope_wait_heartbeat "alb-active" 90 300 "pending"' + [ "$status" -eq 0 ] + echo "$output" | grep -q '"status":"waiting"' + echo "$output" | grep -q '"wait.what":"alb-active"' + echo "$output" | grep -q '"wait.elapsed_s":"90"' + echo "$output" | grep -q '"wait.timeout_s":"300"' +} + +@test "without NP_TRACE, logging is byte-identical to plain logging" { + unset NP_TRACE + run "$BASH" -c "source '$LOGGING'; log error 'plain'; log info 'hello'" + [ "$status" -eq 0 ] + [ "$output" = "plain +hello" ] + [ ! -d "$NP_TRACE_DIR/spool" ] || [ -z "$(ls -A "$NP_TRACE_DIR/spool" 2>/dev/null)" ] +} + +@test "without NP_API_KEY, logging is byte-identical to plain logging" { + unset NP_API_KEY + run "$BASH" -c "source '$LOGGING'; log error 'plain'" + [ "$status" -eq 0 ] + [ "$output" = "plain" ] + [ ! -d "$NP_TRACE_DIR/spool" ] || [ -z "$(ls -A "$NP_TRACE_DIR/spool" 2>/dev/null)" ] +} + +@test "heartbeat is a defined no-op when the workflow is untraced" { + unset NP_TRACE + run "$BASH" -c "source '$LOGGING'; np_scope_wait_heartbeat what 1 2; echo rc=\$?" + [ "$status" -eq 0 ] + [[ "$output" == *"rc=0"* ]] +} + +@test "stdout and stderr routing of log() is unchanged when traced" { + run "$BASH" -c "source '$LOGGING'; log info 'to-stdout' 2>/dev/null" + [[ "$output" == *"to-stdout"* ]] + run "$BASH" -c "source '$LOGGING'; log error 'to-stderr' 2>&1 >/dev/null" + [[ "$output" == *"to-stderr"* ]] +} diff --git a/nptrace.sh b/nptrace.sh new file mode 100755 index 00000000..4e4a006f --- /dev/null +++ b/nptrace.sh @@ -0,0 +1,1426 @@ +#!/bin/sh + +# ---- src/header.sh ---- +# nullplatform tracing for POSIX shell — producer SDK for the nullplatform +# tracing API. Zero runtime dependencies beyond curl and the POSIX toolset. +# +# Generated file: edit src/*.sh and run ./build.sh. + +if [ -n "${NP_TRACE_LOADED:-}" ]; then + return 0 2>/dev/null || exit 0 +fi +NP_TRACE_LOADED=1 +NP_TRACE_VERSION="0.1.0" + +# ---- src/compat.sh ---- +# compat.sh — portability shims. The ONLY place OS differences live. + +# Unix milliseconds. GNU date supports %N; busybox and BSD may not, and they +# fail in two DIFFERENT ways: +# +# busybox 1.38 / BSD -> "1786045823%3N" the format leaks through literally +# busybox 1.37 -> "1786045823" the format is silently DROPPED +# +# The second is the dangerous one: the result is clean digits that merely happen +# to be seconds, so a digits-only check accepts it and every timestamp is then +# 1000x too small — which silently destroys UUIDv7 ordering, since the seconds +# value lands in a 48-bit millisecond field and decodes to 1970. +# +# Length is what separates them: Unix milliseconds have been 13 digits since +# 2001-09-09 and stay 13 until 2286, while seconds are 10. Anything shorter than +# 13 is not milliseconds, whatever it looks like. +np__epoch_ms() { + _cm_ms=$(date -u +%s%3N 2>/dev/null) || _cm_ms='' + case "$_cm_ms" in + '' | *[!0-9]*) _cm_ms='' ;; + esac + if [ -n "$_cm_ms" ] && [ "${#_cm_ms}" -ge 13 ]; then + printf '%s' "$_cm_ms" + return 0 + fi + # Second precision. Event ids stay unique via their random bits. + printf '%s000' "$(date -u +%s)" +} + +# Exactly $1 lowercase hex characters from the kernel CSPRNG. +np__rand_hex() { + _rh_want=$1 + _rh_bytes=$(( (_rh_want + 1) / 2 )) + od -An -tx1 -N"$_rh_bytes" /dev/urandom | tr -d ' \n' | cut -c1-"$_rh_want" +} + +# RFC 3339 UTC, second precision — the envelope `time` field. +np__iso8601() { + date -u +%Y-%m-%dT%H:%M:%SZ +} + +# ---- src/json.sh ---- +# json.sh — JSON emission. There is no parser here beyond one field extractor +# for the auth response; the SDK only ever WRITES JSON. + +# Escape a string for a JSON string body (no surrounding quotes). +# +# Fast path: a string made only of unmistakably safe characters is returned +# unchanged, so the common label/id case never forks an awk. The allowlist is +# deliberately conservative — routing an unusual string to the slow path is +# always correct, only slower. +# +# Slow path: awk under LC_ALL=C, so length/substr are BYTE oriented on every +# awk (gawk, mawk, busybox). UTF-8 sequences pass through byte for byte, which +# is valid JSON; only the seven shorthand escapes and C0 controls are rewritten. +# Records are read line by line and rejoined with \n rather than using a +# multi-character RS, whose behaviour POSIX leaves undefined. +np__json_escape() { + case "$1" in + *[!A-Za-z0-9\ ._:/@=+,-]*) ;; + *) printf '%s' "$1"; return 0 ;; + esac + printf '%s' "$1" | LC_ALL=C awk ' + function esc(s, i, c, n, o) { + o = "" + n = length(s) + for (i = 1; i <= n; i++) { + c = substr(s, i, 1) + if (c == "\\") { o = o "\\\\" } + else if (c == "\"") { o = o "\\\"" } + else if (c == "\t") { o = o "\\t" } + else if (c == "\r") { o = o "\\r" } + else if (c == "\b") { o = o "\\b" } + else if (c == "\f") { o = o "\\f" } + else if (c < " ") { o = o sprintf("\\u%04x", ORD[c]) } + else { o = o c } + } + return o + } + BEGIN { + ORS = "" + for (i = 0; i < 256; i++) { ORD[sprintf("%c", i)] = i } + out = "" + } + { + if (NR > 1) { out = out "\\n" } + out = out esc($0) + } + END { printf "%s", out } + ' +} + +# A complete quoted JSON string. +np__json_str() { + printf '"%s"' "$(np__json_escape "$1")" +} + +# A JSON object from alternating key/value arguments. Values are emitted as +# JSON strings. A pair whose key or value is empty is OMITTED — an absent +# optional is absent, never the string "". +np__json_obj() { + _jo_out='' + while [ "$#" -ge 2 ]; do + if [ -n "$1" ] && [ -n "$2" ]; then + if [ -n "$_jo_out" ]; then + _jo_out="$_jo_out," + fi + _jo_out="$_jo_out$(np__json_str "$1"):$(np__json_str "$2")" + fi + shift 2 + done + printf '{%s}' "$_jo_out" +} + +# As np__json_obj, but each value is already-formed JSON inserted verbatim. +# Use for nested objects, arrays, numbers, and booleans. +np__json_obj_raw() { + _jor_out='' + while [ "$#" -ge 2 ]; do + if [ -n "$1" ] && [ -n "$2" ]; then + if [ -n "$_jor_out" ]; then + _jor_out="$_jor_out," + fi + _jor_out="$_jor_out$(np__json_str "$1"):$2" + fi + shift 2 + done + printf '{%s}' "$_jor_out" +} + +# ---- src/uuid.sh ---- +# uuid.sh — UUIDv7. The event id MUST be a v7: the API derives the storage +# partition from its embedded millisecond timestamp and rejects anything else. +# +# Layout: 48-bit big-endian ms timestamp | version nibble 7 | 12 random bits +# | variant bits 10 | 62 random bits. + +np__uuidv7() { + _u7_ts=$(printf '%012x' "$(np__epoch_ms)") + _u7_r=$(np__rand_hex 19) + + # The variant nibble must be one of 8, 9, a, b. Fold a random hex digit into + # that range rather than drawing again. + case $(printf '%s' "$_u7_r" | cut -c1) in + 0 | 1 | 2 | 3) _u7_var=8 ;; + 4 | 5 | 6 | 7) _u7_var=9 ;; + 8 | 9 | a | b) _u7_var=a ;; + *) _u7_var=b ;; + esac + + printf '%s-%s-7%s-%s%s-%s\n' \ + "$(printf '%s' "$_u7_ts" | cut -c1-8)" \ + "$(printf '%s' "$_u7_ts" | cut -c9-12)" \ + "$(printf '%s' "$_u7_r" | cut -c2-4)" \ + "$_u7_var" \ + "$(printf '%s' "$_u7_r" | cut -c5-7)" \ + "$(printf '%s' "$_u7_r" | cut -c8-19)" +} + +# Mint a per-occurrence token for a repeatable operation's run_id. Time-ordered, +# so minted ids sort by creation time. +np_trace_occurrence() { + np__uuidv7 +} + +# ---- src/identity.sh ---- +# identity.sh — the node identity grammar. A hand-port of the tracing API's +# contract module; these functions and their tests are the drift safety net. +# +# child_run_id = parent_run_id "~" key "@" attempt "." iteration +# +# One charset covers every producer-authored segment: [A-Za-z0-9_.-]+. The +# delimiter '~' and the coordinate marker '@' sit outside it, which is what +# makes the grammar collision-proof — no named id can ever parse as a derived +# one. + +NP_ID_DELIMITER='~' +NP_MAX_RUN_ID_LENGTH=1024 +NP_MAX_KEY_LENGTH=256 +NP_MAX_TRACE_ID_LENGTH=256 + +np__is_identifier() { + case "${1:-}" in + '') return 1 ;; + *[!A-Za-z0-9_.-]*) return 1 ;; + *) return 0 ;; + esac +} + +# Print a reason and return 1, or return 0 silently. +np__identifier_violation() { + if [ -z "$1" ]; then + printf 'must be non-empty' + return 1 + fi + if [ "${#1}" -gt "$2" ]; then + printf 'exceeds %s chars' "$2" + return 1 + fi + if ! np__is_identifier "$1"; then + printf "must be identifier-charset: letters, digits, '_', '.', '-'" + return 1 + fi + return 0 +} + +np__key_violation() { + np__identifier_violation "${1:-}" "$NP_MAX_KEY_LENGTH" +} + +np__named_id_violation() { + np__identifier_violation "${1:-}" "$NP_MAX_RUN_ID_LENGTH" +} + +np__trace_id_violation() { + np__identifier_violation "${1:-}" "$NP_MAX_TRACE_ID_LENGTH" +} + +# The derived id of a keyed child. +np__derive_child_id() { + printf '%s%s%s@%s.%s' "$1" "$NP_ID_DELIMITER" "$2" "$3" "$4" +} + +# Everything before the FIRST delimiter — the nearest named ancestor. Every +# keyed descendant of a named run shares its scope root at any depth. +np__scope_root_of() { + case "$1" in + *"$NP_ID_DELIMITER"*) printf '%s' "${1%%"$NP_ID_DELIMITER"*}" ;; + *) printf '%s' "$1" ;; + esac +} + +# Parse the LAST hop of a derived id. Prints " ". +# Returns 1 for a named id (no delimiter) or a malformed tail. +np__parse_node_id() { + case "$1" in + *"$NP_ID_DELIMITER"*) ;; + *) return 1 ;; + esac + _pn_parent=${1%"$NP_ID_DELIMITER"*} + _pn_tail=${1##*"$NP_ID_DELIMITER"} + case "$_pn_tail" in + *@*.*) ;; + *) return 1 ;; + esac + _pn_key=${_pn_tail%%@*} + _pn_coord=${_pn_tail#*@} + _pn_attempt=${_pn_coord%%.*} + _pn_iteration=${_pn_coord#*.} + if [ -z "$_pn_parent" ] || [ -z "$_pn_key" ]; then + return 1 + fi + case "$_pn_attempt" in + '' | *[!0-9]*) return 1 ;; + esac + case "$_pn_iteration" in + '' | *[!0-9]*) return 1 ;; + esac + printf '%s %s %s %s' "$_pn_parent" "$_pn_key" "$_pn_attempt" "$_pn_iteration" +} + +# Join parts into a stable id, dropping empty parts. Use instead of +# hand-interpolation so an absent part never leaves a dangling separator. +# The joiner is '-', a charset character, so the result stays a legal named id. +np_trace_key() { + _k_out='' + for _k_part in "$@"; do + if [ -n "$_k_part" ]; then + if [ -n "$_k_out" ]; then + _k_out="$_k_out-" + fi + _k_out="$_k_out$_k_part" + fi + done + printf '%s' "$_k_out" +} + +# ---- src/wire.sh ---- +# wire.sh — contract constants, hand-ported from the tracing API's wire +# package. When the API's contract changes, this file and identity.sh are what +# must be re-ported; their tests are the safety net. + +NP_TYPE_NODE_RUN='node.run' +NP_TYPE_NODE_DATASET='node.dataset' +NP_TYPE_NODE_JOB='node.job' + +NP_TYPE_EDGE_PARENT='edge.parent' +NP_TYPE_EDGE_TRIGGERED_BY='edge.triggered_by' +NP_TYPE_EDGE_RETRY_OF='edge.retry_of' +NP_TYPE_EDGE_CONTINUES='edge.continues' +NP_TYPE_EDGE_CORRELATES='edge.correlates' +NP_TYPE_EDGE_COMPENSATES='edge.compensates' +NP_TYPE_EDGE_PRODUCES='edge.produces' +NP_TYPE_EDGE_CONSUMES='edge.consumes' +NP_TYPE_EDGE_INSTANCE_OF='edge.instance_of' + +NP_STATUS_STARTED='started' +NP_STATUS_COMPLETED='completed' +NP_STATUS_FAILED='failed' +NP_STATUS_CANCELLED='cancelled' +NP_STATUS_TIMED_OUT='timed_out' +NP_STATUS_SKIPPED='skipped' +NP_STATUS_WAITING='waiting' + +NP_FACET_ERROR='tracing.error' +NP_FACET_TIMING='tracing.timing' +NP_FACET_INPUT='tracing.input' +NP_FACET_OUTPUT='tracing.output' +NP_FACET_BINDING='tracing.binding' +NP_FACET_DECISION='tracing.decision' +NP_FACET_RETRY='tracing.retry' +NP_FACET_SIGNAL='tracing.signal' +NP_FACET_EXTERNAL_LINKS='tracing.externalLinks' +NP_FACET_PLAN='tracing.plan' +NP_FACET_ACTOR='tracing.actor' +NP_FACET_DROPPED='tracing.dropped' +NP_FACET_ENGINE_STATUS='tracing.engineStatus' +NP_FACET_AFFORDANCES='tracing.affordances' +NP_FACET_EXPLAIN='tracing.explain' +NP_FACET_PROGRESS='tracing.progress' + +NP_CORE_FACETS="$NP_FACET_ERROR $NP_FACET_TIMING $NP_FACET_INPUT $NP_FACET_OUTPUT \ +$NP_FACET_BINDING $NP_FACET_DECISION $NP_FACET_RETRY $NP_FACET_SIGNAL \ +$NP_FACET_EXTERNAL_LINKS $NP_FACET_PLAN $NP_FACET_ACTOR $NP_FACET_DROPPED \ +$NP_FACET_ENGINE_STATUS $NP_FACET_AFFORDANCES $NP_FACET_EXPLAIN $NP_FACET_PROGRESS" + +NP_RESERVED_FACET_PREFIX='tracing.' +NP_RESERVED_LABEL_PREFIX='tracing.io/' + +# The context carrier: ONE field whose value packs version, trace and run. +NP_CARRIER_KEY='np-trace' +NP_CARRIER_VERSION='1' +NP_CARRIER_DELIMITER='|' + +np__is_terminal_status() { + case "${1:-}" in + completed | failed | cancelled | timed_out | skipped) return 0 ;; + *) return 1 ;; + esac +} + +# ---- src/state.sh ---- +# state.sh — the on-disk node registry. State lives on disk rather than in +# shell memory so handles survive process boundaries: in CI every pipeline step +# is a fresh shell. + +# Create the state tree. If it cannot be created or written — a read-only +# filesystem, a full disk, a bad NP_TRACE_DIR — the SDK degrades to a REAL +# no-op rather than half-working: a half-initialised SDK whose next write fails +# would take down a caller running under `set -e`, which is exactly the failure +# mode tracing must never cause. +np__state_init() { + if [ -z "${NP_TRACE_DIR:-}" ]; then + NP_TRACE_DIR="${TMPDIR:-/tmp}/nptrace.$$" + fi + export NP_TRACE_DIR + if ! mkdir -p "$NP_TRACE_DIR/nodes" "$NP_TRACE_DIR/staged" \ + "$NP_TRACE_DIR/spool" "$NP_TRACE_DIR/failed" 2>/dev/null; then + NP_TRACE_ENABLED=0 + return 0 + fi + # Prove the tree is actually writable before trusting it. + if ! printf '0' > "$NP_TRACE_DIR/seq.probe" 2>/dev/null; then + NP_TRACE_ENABLED=0 + return 0 + fi + rm -f "$NP_TRACE_DIR/seq.probe" 2>/dev/null || : + if [ ! -f "$NP_TRACE_DIR/seq" ]; then + printf '0' > "$NP_TRACE_DIR/seq" 2>/dev/null || : + fi + return 0 +} + +# Allocate the next handle. Handles are opaque by contract: consumers never +# parse them. +np__handle_new() { + _hn_seq=$(cat "$NP_TRACE_DIR/seq" 2>/dev/null || printf '0') + case "$_hn_seq" in + '' | *[!0-9]*) _hn_seq=0 ;; + esac + _hn_seq=$((_hn_seq + 1)) + printf '%s' "$_hn_seq" > "$NP_TRACE_DIR/seq" + _hn_handle="n$_hn_seq" + : > "$NP_TRACE_DIR/nodes/$_hn_handle" + printf '%s' "$_hn_handle" +} + +# THE rule the whole public surface rests on: an argument is a handle iff it +# has the allocator's shape AND names an existing node file. The shape check +# comes first so a caller-supplied string can never traverse out of nodes/. +np__is_handle() { + case "${1:-}" in + n) return 1 ;; + n*) case "${1#n}" in '' | *[!0-9]*) return 1 ;; esac ;; + *) return 1 ;; + esac + [ -f "$NP_TRACE_DIR/nodes/$1" ] +} + +np__node_set() { + _ns_file="$NP_TRACE_DIR/nodes/$1" + [ -f "$_ns_file" ] || return 0 + # Drop any prior value for this key, then append the new one. The trailing + # '=' in the match means a key that is a prefix of another never collides. + if grep -q "^$2=" "$_ns_file" 2>/dev/null; then + grep -v "^$2=" "$_ns_file" > "$_ns_file.tmp" 2>/dev/null || : > "$_ns_file.tmp" + mv "$_ns_file.tmp" "$_ns_file" + fi + printf '%s=%s\n' "$2" "$3" >> "$_ns_file" + return 0 +} + +np__node_get() { + _ng_file="$NP_TRACE_DIR/nodes/$1" + [ -f "$_ng_file" ] || return 0 + # Strip only the leading "key=", so a value containing '=' survives intact. + sed -n "s/^$2=//p" "$_ng_file" 2>/dev/null | head -n 1 + return 0 +} + +# Ambient resolution, exactly two levels. There is deliberately no third, +# session-wide level: that is where concurrent writers race. +# +# 1. NP_TRACE_CURRENT — explicit, and what you export to cross a CI step. +# 2. current.$$ — auto-maintained within one process tree. POSIX $$ +# does not change in a subshell, so a handle created +# inside $(...) is visible to the caller. +np__ambient() { + if [ -n "${NP_TRACE_CURRENT:-}" ]; then + printf '%s' "$NP_TRACE_CURRENT" + return 0 + fi + cat "$NP_TRACE_DIR/current.$$" 2>/dev/null || printf '' + return 0 +} + +np__ambient_set() { + printf '%s' "$1" > "$NP_TRACE_DIR/current.$$" 2>/dev/null || return 0 + return 0 +} + +np__ambient_clear() { + # Only clear when the cleared handle IS current, so terminalizing an outer + # node cannot silently retarget an inner one. + if [ "$(np__ambient)" = "$1" ]; then + rm -f "$NP_TRACE_DIR/current.$$" 2>/dev/null || : + if [ -n "${NP_TRACE_CURRENT:-}" ] && [ "$NP_TRACE_CURRENT" = "$1" ]; then + NP_TRACE_CURRENT='' + fi + fi + return 0 +} + +# Every node-scoped public function starts here: use $1 when it is a handle, +# otherwise fall back to the ambient node. +np__resolve_handle() { + if np__is_handle "${1:-}"; then + printf '%s' "$1" + else + np__ambient + fi + return 0 +} + +# ---- src/spool.sh ---- +# spool.sh — the emit hot path. Every emit is a LOCAL FILE WRITE: the network +# is never touched here, which is what makes API downtime invisible to the +# caller. The spool file's NAME is the event id, so re-POSTing after a crash is +# idempotent — that is recover() for free. + +# np__spool -> prints the event id +np__spool() { + _sp_id=$(np__uuidv7) + _sp_env=$(np__json_obj_raw \ + id "$(np__json_str "$_sp_id")" \ + time "$(np__json_str "$(np__iso8601)")" \ + type "$(np__json_str "$1")" \ + nrn "$(if [ -n "$2" ]; then np__json_str "$2"; fi)" \ + producer "$(np__json_str "${NP_TRACE_PRODUCER:-}")" \ + data "$3") + + _sp_tmp="$NP_TRACE_DIR/spool/$_sp_id.json.tmp" + _sp_final="$NP_TRACE_DIR/spool/$_sp_id.json" + printf '%s' "$_sp_env" > "$_sp_tmp" 2>/dev/null || return 0 + # Create-then-rename: a concurrent flush never sees a half-written envelope. + mv "$_sp_tmp" "$_sp_final" 2>/dev/null || return 0 + printf '%s' "$_sp_id" + return 0 +} + +np__spool_count() { + _sc_n=0 + for _sc_f in "$NP_TRACE_DIR/spool"/*.json; do + [ -f "$_sc_f" ] || continue + _sc_n=$((_sc_n + 1)) + done + printf '%s' "$_sc_n" + return 0 +} + +# ---- src/http.sh ---- +# http.sh — the only module that touches the network. Every request is bounded +# by a connect AND a total timeout, so an unreachable or hanging API can never +# stall the caller. + +NP_TRACE_CONNECT_TIMEOUT="${NP_TRACE_CONNECT_TIMEOUT:-3}" +NP_TRACE_MAX_TIME="${NP_TRACE_MAX_TIME:-10}" +NP_TRACE_DEFAULT_BASE_URL='https://api.nullplatform.com/tracing' +NP_TRACE_DEFAULT_AUTH_URL='https://api.nullplatform.com' + +np__drop() { + printf '%s\t%s\t%s\n' "$(np__iso8601)" "$1" "$2" >> "$NP_TRACE_DIR/drops.log" 2>/dev/null || : + if [ -n "${NP_TRACE_ON_DROP:-}" ]; then + "$NP_TRACE_ON_DROP" "$1" "$2" 2>/dev/null || : + fi + if [ -n "${NP_TRACE_DEBUG:-}" ]; then + printf 'np-trace drop: %s (%s)\n' "$1" "$2" >&2 + fi + return 0 +} + +# Suppress xtrace for a credential-handling region, remembering whether it was +# on. CI scripts routinely `set -x`, and shell options are global — so without +# this a sourced SDK function would print the bearer token into the build log +# even though it never reaches curl's argv. Every credential path is bracketed +# by np__secret_begin / np__secret_end. +np__secret_begin() { + case "$-" in + *x*) NP_TRACE_XTRACE=1; set +x ;; + *) NP_TRACE_XTRACE='' ;; + esac +} + +np__secret_end() { + if [ -n "${NP_TRACE_XTRACE:-}" ]; then + NP_TRACE_XTRACE='' + set -x + fi + return 0 +} + +# A bearer token. A pre-issued NP_TRACE_TOKEN wins; otherwise exchange the api +# key, caching until shortly before expiry. Called LAZILY, at first flush — +# never at init, so a down auth endpoint cannot delay pipeline startup. +np__token() { + np__secret_begin + if [ -n "${NP_TRACE_TOKEN:-}" ]; then + printf '%s' "$NP_TRACE_TOKEN" + np__secret_end + return 0 + fi + np__token_exchange + np__secret_end + return 0 +} + +# The api-key exchange. Always called from inside a secret region. +np__token_exchange() { + if [ -z "${NP_TRACE_API_KEY:-}" ]; then + printf '' + return 0 + fi + + _tk_cache="$NP_TRACE_DIR/token" + if [ -f "$_tk_cache" ]; then + _tk_exp=$(sed -n '1p' "$_tk_cache" 2>/dev/null) + _tk_val=$(sed -n '2p' "$_tk_cache" 2>/dev/null) + case "$_tk_exp" in + '' | *[!0-9]*) _tk_exp=0 ;; + esac + if [ -n "$_tk_val" ] && [ "$_tk_exp" -gt "$(date +%s)" ]; then + printf '%s' "$_tk_val" + return 0 + fi + fi + + _tk_body=$(curl -sS -X POST \ + --connect-timeout "$NP_TRACE_CONNECT_TIMEOUT" --max-time "$NP_TRACE_MAX_TIME" \ + -H 'Content-Type: application/json' \ + -d "$(np__json_obj apiKey "$NP_TRACE_API_KEY")" \ + "${NP_TRACE_AUTH_URL:-$NP_TRACE_DEFAULT_AUTH_URL}/token" 2>/dev/null) || _tk_body='' + + _tk_new=$(printf '%s' "$_tk_body" | + sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p') + if [ -z "$_tk_new" ]; then + np__drop 'auth' 'token exchange failed' + printf '' + return 0 + fi + ( umask 077; printf '%s\n%s\n' "$(( $(date +%s) + 3540 ))" "$_tk_new" > "$_tk_cache" ) + printf '%s' "$_tk_new" + return 0 +} + +# The auth header goes to curl via --config from a mode-600 file, NEVER as -H +# in argv: CI runs with `set -x`, and an argv-borne header prints the token +# straight into the build log. +np__auth_config() { + np__secret_begin + _ac_file="$NP_TRACE_DIR/curlcfg.$$" + ( umask 077; printf 'header = "Authorization: Bearer %s"\n' "$(np__token)" > "$_ac_file" ) + np__secret_end + printf '%s' "$_ac_file" + return 0 +} + +# POST one spool file. Prints the HTTP status code, or 000 on a network failure. +np__post_event() { + _pe_cfg=$(np__auth_config) + _pe_code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + --config "$_pe_cfg" \ + --connect-timeout "$NP_TRACE_CONNECT_TIMEOUT" --max-time "$NP_TRACE_MAX_TIME" \ + -H 'Content-Type: application/json' \ + --data-binary "@$1" \ + "${NP_TRACE_BASE_URL:-$NP_TRACE_DEFAULT_BASE_URL}/events" 2>/dev/null) || _pe_code='000' + rm -f "$_pe_cfg" 2>/dev/null || : + case "$_pe_code" in + '' | *[!0-9]*) _pe_code='000' ;; + esac + printf '%s' "$_pe_code" + return 0 +} + +# ---- src/flush.sh ---- +# flush.sh — the spool drain. Bounded by a wall-clock budget so a dead API can +# never hang process exit; every path returns 0. + +NP_TRACE_FLUSH_TIMEOUT="${NP_TRACE_FLUSH_TIMEOUT:-10}" +NP_TRACE_MAX_RETRIES="${NP_TRACE_MAX_RETRIES:-3}" + +np__attempts_of() { + _ao_n=$(cat "$1.attempts" 2>/dev/null || printf '0') + case "$_ao_n" in + '' | *[!0-9]*) _ao_n=0 ;; + esac + printf '%s' "$_ao_n" +} + +np__fail_event() { + mv "$1" "$NP_TRACE_DIR/failed/" 2>/dev/null || rm -f "$1" 2>/dev/null || : + rm -f "$1.attempts" 2>/dev/null || : + np__drop "${1##*/}" "$2" + return 0 +} + +np_trace_flush() { + [ -n "${NP_TRACE_DIR:-}" ] || return 0 + [ -d "$NP_TRACE_DIR/spool" ] || return 0 + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _fl_deadline=$(( $(date +%s) + NP_TRACE_FLUSH_TIMEOUT )) + + for _fl_file in "$NP_TRACE_DIR/spool"/*.json; do + [ -f "$_fl_file" ] || continue + if [ "$(date +%s)" -ge "$_fl_deadline" ]; then + # Budget spent. Remaining events stay on disk for the next flush or a + # later np_trace_recover; the process exits on time regardless. This is + # the guarantee that a dead API cannot hang a build. + return 0 + fi + + _fl_code=$(np__post_event "$_fl_file") + case "$_fl_code" in + 201 | 200) + # 200 is an idempotent re-POST of an already-accepted event. + rm -f "$_fl_file" "$_fl_file.attempts" 2>/dev/null || : + ;; + 400) + # A contract violation. Never retried — retrying cannot change it. + np__fail_event "$_fl_file" "rejected 400" + ;; + 401 | 403) + rm -f "$NP_TRACE_DIR/token" 2>/dev/null || : + np__fail_event "$_fl_file" "unauthorized $_fl_code" + ;; + *) + _fl_n=$(( $(np__attempts_of "$_fl_file") + 1 )) + if [ "$_fl_n" -gt "$NP_TRACE_MAX_RETRIES" ]; then + np__fail_event "$_fl_file" "gave up after $_fl_n attempts (last status $_fl_code)" + else + printf '%s' "$_fl_n" > "$_fl_file.attempts" 2>/dev/null || : + fi + ;; + esac + done + return 0 +} + +np_trace_shutdown() { + np_trace_flush + if [ -n "${NP_TRACE_DIR:-}" ] && [ "${NP_TRACE_KEEP_STATE:-0}" != '1' ]; then + rm -rf "$NP_TRACE_DIR" 2>/dev/null || : + fi + return 0 +} + +# Re-deliver a previous process's leftover spool. Idempotent by construction: +# the spool file name IS the event id, so the API answers a re-POST with +# 200 duplicate. +np_trace_recover() { + np_trace_flush + return 0 +} + +np__install_trap() { + if [ -z "${NP_TRACE_NO_TRAP:-}" ]; then + trap 'np_trace_flush' EXIT + trap 'np_trace_flush' INT + trap 'np_trace_flush' TERM + fi + return 0 +} + +# ---- src/propagation.sh ---- +# --------------------------------------------------------------------------- +# Propagation +# +# Cross-process trace context, wire-identical to the Go and JS SDKs: a single +# carrier value packing "||". The '|' delimiter is +# reserved, so the value splits unambiguously even though a run_id may itself +# contain '~' and '@'. +# +# The carrier travels in the NP_TRACE environment variable. Note that this is +# deliberately OUTSIDE the NP_TRACE_* configuration namespace the SDK reads for +# its own settings: NP_TRACE is context handed to us by a caller, not something +# a user configures. +# --------------------------------------------------------------------------- + +# np_trace_inject [handle] +# +# Print the carrier value for a handle (defaults to the ambient node), for +# handing to a child process. Prints nothing when there is no node to inject, +# so `NP_TRACE=$(np_trace_inject)` is always safe. +np_trace_inject() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _ij_h=$(np__resolve_handle "${1:-}") + np__is_handle "$_ij_h" || return 0 + printf '%s%s%s%s%s' \ + "$NP_CARRIER_VERSION" "$NP_CARRIER_DELIMITER" \ + "$(np__node_get "$_ij_h" trace_id)" "$NP_CARRIER_DELIMITER" \ + "$(np__node_get "$_ij_h" run_id)" + return 0 +} + +# np_trace_extract [carrier] +# +# Parse a carrier value (defaults to $NP_TRACE) and print " ". +# Returns 1 when there is no usable context, so callers can branch: +# +# if ctx=$(np_trace_extract); then set -- $ctx; fi +# +# When only a trace id is present it is used for both, matching the Go SDK, so +# the result is always a usable pair. +np_trace_extract() { + _ex_raw=${1-${NP_TRACE:-}} + [ -n "$_ex_raw" ] || return 1 + + case "$_ex_raw" in + "$NP_CARRIER_VERSION$NP_CARRIER_DELIMITER"*) ;; + *) return 1 ;; + esac + _ex_rest=${_ex_raw#*"$NP_CARRIER_DELIMITER"} + + # trace_id is up to the next delimiter; run_id is the whole remainder, which + # may itself contain '~' and '@' but never a delimiter. + case "$_ex_rest" in + *"$NP_CARRIER_DELIMITER"*) + _ex_trace=${_ex_rest%%"$NP_CARRIER_DELIMITER"*} + _ex_run=${_ex_rest#*"$NP_CARRIER_DELIMITER"} + ;; + *) + _ex_trace=$_ex_rest + _ex_run=$_ex_rest + ;; + esac + [ -n "$_ex_trace" ] || return 1 + [ -n "$_ex_run" ] || _ex_run=$_ex_trace + + printf '%s %s' "$_ex_trace" "$_ex_run" + return 0 +} + +# np_trace_adopt [carrier] +# +# Attach to an upstream node and return a handle standing in for it, so work +# started here nests UNDERNEATH it: +# +# parent=$(np_trace_adopt) || parent=$(np_trace_run --run-id "$(np_trace_occurrence)") +# step=$(np_trace_step "$parent" build) +# +# The adopted node belongs to whoever created it — typically the np CLI, which +# exports NP_TRACE per workflow step. We hold its ids so children derive +# correctly, but must never speak for it: it is marked foreign, so it emits no +# node event of its own and the terminal verbs refuse to close it. Children +# hanging off it still emit their own containment edges, which IS ours to say. +# +# Returns 1 when there is no upstream context, leaving the caller to open a root +# run instead. +np_trace_adopt() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 1 + _ad_ctx=$(np_trace_extract "${1-${NP_TRACE:-}}") || return 1 + _ad_trace=${_ad_ctx%% *} + _ad_run=${_ad_ctx#* } + + if ! _ad_why=$(np__trace_id_violation "$_ad_trace"); then + np__drop 'adopt' "trace_id $_ad_why" + return 1 + fi + # An upstream run_id is commonly a DERIVED path (parent~key@attempt.iteration) + # rather than a named id — the np CLI hands us the step it is running. Accept + # either: parse it as a node path first, and only fall back to the named-id + # rules when it has no delimiter. + if ! np__parse_node_id "$_ad_run" >/dev/null 2>&1; then + if ! _ad_why=$(np__named_id_violation "$_ad_run"); then + np__drop 'adopt' "run_id $_ad_why" + return 1 + fi + fi + + _ad_h=$(np__handle_new) + np__node_set "$_ad_h" kind run + np__node_set "$_ad_h" trace_id "$_ad_trace" + np__node_set "$_ad_h" run_id "$_ad_run" + np__node_set "$_ad_h" nrn "${NP_TRACE_NRN:-}" + np__node_set "$_ad_h" foreign 1 + # started=1 suppresses the lazy `started` emit; closed=0 keeps it usable as a + # parent for the whole script. + np__node_set "$_ad_h" started 1 + np__node_set "$_ad_h" closed 0 + np__ambient_set "$_ad_h" + printf '%s' "$_ad_h" + return 0 +} + +# True when a handle stands in for a node owned by another process. +np__is_foreign() { + [ "$(np__node_get "$1" foreign)" = '1' ] +} + +# ---- src/api.sh ---- +# api.sh — the public producer surface. Every function here returns 0, always: +# tracing must never fail the caller. +# +# Every node-scoped function takes an OPTIONAL leading handle. This is one +# function with a defaulted argument, not two ways to say the same thing: when +# the first argument is not a handle it falls back to the innermost open node. + +np_trace_init() { + while [ "$#" -gt 0 ]; do + case "$1" in + --producer) NP_TRACE_PRODUCER=${2:-}; shift 2 ;; + --base-url) NP_TRACE_BASE_URL=${2:-}; shift 2 ;; + --auth-url) NP_TRACE_AUTH_URL=${2:-}; shift 2 ;; + --api-key) NP_TRACE_API_KEY=${2:-}; shift 2 ;; + --token) NP_TRACE_TOKEN=${2:-}; shift 2 ;; + --nrn) NP_TRACE_NRN=${2:-}; shift 2 ;; + --enabled) NP_TRACE_ENABLED=${2:-1}; shift 2 ;; + --no-trap) NP_TRACE_NO_TRAP=1; shift ;; + *) shift ;; + esac + done + NP_TRACE_ENABLED="${NP_TRACE_ENABLED:-1}" + np__state_init + # No network call here, deliberately: a down auth endpoint must never delay + # the start of a pipeline. The token is fetched lazily, at first flush. + np__install_trap + return 0 +} + +# --------------------------------------------------------------------------- +# Emission +# --------------------------------------------------------------------------- + +# Emit the node event for a handle at the given status, carrying whatever +# context is currently staged. +np__emit_node() { + _en_h=$1 + _en_status=$2 + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + + _en_labels=$(np__node_get "$_en_h" labels) + _en_facets=$(np__node_get "$_en_h" facets) + _en_key=$(np__node_get "$_en_h" key) + _en_schema=$(np__node_get "$_en_h" schema_url) + + if [ -n "$_en_key" ]; then + _en_data=$(np__json_obj_raw \ + trace_id "$(np__json_str "$(np__node_get "$_en_h" trace_id)")" \ + run_id "$(np__json_str "$(np__node_get "$_en_h" run_id)")" \ + key "$(np__json_str "$_en_key")" \ + attempt "$(np__node_get "$_en_h" attempt)" \ + iteration "$(np__node_get "$_en_h" iteration)" \ + status "$(np__json_str "$_en_status")" \ + labels "$_en_labels" \ + facets "$_en_facets" \ + schema_url "$(if [ -n "$_en_schema" ]; then np__json_str "$_en_schema"; fi)") + else + _en_data=$(np__json_obj_raw \ + trace_id "$(np__json_str "$(np__node_get "$_en_h" trace_id)")" \ + run_id "$(np__json_str "$(np__node_get "$_en_h" run_id)")" \ + status "$(np__json_str "$_en_status")" \ + labels "$_en_labels" \ + facets "$_en_facets" \ + schema_url "$(if [ -n "$_en_schema" ]; then np__json_str "$_en_schema"; fi)") + fi + + np__spool "$NP_TYPE_NODE_RUN" "$(np__node_get "$_en_h" nrn)" "$_en_data" >/dev/null + return 0 +} + +# A run ref for a handle — the self-describing address used on edge endpoints. +np__ref_of() { + np__json_obj \ + type run \ + trace_id "$(np__node_get "$1" trace_id)" \ + run_id "$(np__node_get "$1" run_id)" +} + +np__emit_parent_edge() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _pe_data=$(np__json_obj_raw from "$(np__ref_of "$1")" to "$(np__ref_of "$2")") + np__spool "$NP_TYPE_EDGE_PARENT" "$(np__node_get "$1" nrn)" "$_pe_data" >/dev/null + return 0 +} + +# Force the lazy `started`. Idempotent. +# +# Shell has no microtask, so `started` is emitted at the first event that must +# follow it — a terminal, a child open, an explicit call, or flush. Context +# staged before that lands on `started`; context staged after lands on the +# terminal. Same observable semantics as the JS and Go SDKs, without a timer. +np_trace_start() { + _st_h=$(np__resolve_handle "${1:-}") + np__is_handle "$_st_h" || return 0 + if [ "$(np__node_get "$_st_h" started)" = '1' ]; then + return 0 + fi + np__node_set "$_st_h" started 1 + np__emit_node "$_st_h" "$NP_STATUS_STARTED" + return 0 +} + +# --------------------------------------------------------------------------- +# Nodes +# --------------------------------------------------------------------------- + +np_trace_run() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _rn_trace='' + _rn_run='' + _rn_nrn="${NP_TRACE_NRN:-}" + while [ "$#" -gt 0 ]; do + case "$1" in + --trace-id) _rn_trace=${2:-}; shift 2 ;; + --run-id) _rn_run=${2:-}; shift 2 ;; + --nrn) _rn_nrn=${2:-}; shift 2 ;; + *) shift ;; + esac + done + # A lone root run's trace_id defaults to its run_id, and vice versa. + [ -n "$_rn_trace" ] || _rn_trace=$_rn_run + [ -n "$_rn_run" ] || _rn_run=$_rn_trace + + if ! _rn_why=$(np__trace_id_violation "$_rn_trace"); then + np__drop 'run' "trace_id $_rn_why" + return 0 + fi + if ! _rn_why=$(np__named_id_violation "$_rn_run"); then + np__drop 'run' "run_id $_rn_why" + return 0 + fi + + _rn_h=$(np__handle_new) + np__node_set "$_rn_h" kind run + np__node_set "$_rn_h" trace_id "$_rn_trace" + np__node_set "$_rn_h" run_id "$_rn_run" + np__node_set "$_rn_h" nrn "$_rn_nrn" + np__node_set "$_rn_h" auto_started_at "$(np__iso8601)" + np__node_set "$_rn_h" started 0 + np__node_set "$_rn_h" closed 0 + np__ambient_set "$_rn_h" + printf '%s' "$_rn_h" + return 0 +} + +# np_trace_step [handle] [--attempt N] [--iteration N] +np_trace_step() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _sp_parent=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + _sp_key=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + _sp_attempt=0 + _sp_iteration=0 + while [ "$#" -gt 0 ]; do + case "$1" in + --attempt) _sp_attempt=${2:-0}; shift 2 ;; + --iteration) _sp_iteration=${2:-0}; shift 2 ;; + *) shift ;; + esac + done + + if ! np__is_handle "$_sp_parent"; then + np__drop 'step' 'no parent node in scope' + return 0 + fi + if ! _sp_why=$(np__key_violation "$_sp_key"); then + np__drop 'step' "key $_sp_why" + return 0 + fi + case "$_sp_attempt$_sp_iteration" in + '' | *[!0-9]*) np__drop 'step' 'attempt and iteration must be integers'; return 0 ;; + esac + + # Opening a child forces the parent's started: a parent edge must not point + # at a node the read model has never seen. + np_trace_start "$_sp_parent" + + _sp_id=$(np__derive_child_id "$(np__node_get "$_sp_parent" run_id)" \ + "$_sp_key" "$_sp_attempt" "$_sp_iteration") + + _sp_h=$(np__handle_new) + np__node_set "$_sp_h" kind step + np__node_set "$_sp_h" trace_id "$(np__node_get "$_sp_parent" trace_id)" + np__node_set "$_sp_h" run_id "$_sp_id" + np__node_set "$_sp_h" nrn "$(np__node_get "$_sp_parent" nrn)" + np__node_set "$_sp_h" key "$_sp_key" + np__node_set "$_sp_h" attempt "$_sp_attempt" + np__node_set "$_sp_h" iteration "$_sp_iteration" + np__node_set "$_sp_h" parent "$_sp_parent" + np__node_set "$_sp_h" auto_started_at "$(np__iso8601)" + np__node_set "$_sp_h" started 0 + np__node_set "$_sp_h" closed 0 + + np_trace_start "$_sp_h" + np__emit_parent_edge "$_sp_parent" "$_sp_h" + np__ambient_set "$_sp_h" + printf '%s' "$_sp_h" + return 0 +} + +# A named child run — a new scope under the same trace. +np_trace_child() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _ch_parent=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + _ch_run='' + while [ "$#" -gt 0 ]; do + case "$1" in + --run-id) _ch_run=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if ! np__is_handle "$_ch_parent"; then + np__drop 'child' 'no parent node in scope' + return 0 + fi + if ! _ch_why=$(np__named_id_violation "$_ch_run"); then + np__drop 'child' "run_id $_ch_why" + return 0 + fi + np_trace_start "$_ch_parent" + _ch_h=$(np_trace_run --trace-id "$(np__node_get "$_ch_parent" trace_id)" \ + --run-id "$_ch_run" \ + --nrn "$(np__node_get "$_ch_parent" nrn)") + np__is_handle "$_ch_h" || return 0 + np__node_set "$_ch_h" parent "$_ch_parent" + np_trace_start "$_ch_h" + np__emit_parent_edge "$_ch_parent" "$_ch_h" + np__ambient_set "$_ch_h" + printf '%s' "$_ch_h" + return 0 +} + +# --------------------------------------------------------------------------- +# Staging context +# --------------------------------------------------------------------------- + +# Merge a pre-formed `"key":value` fragment into the node's staged labels. +np__stage_label() { + _sl_cur=$(np__node_get "$1" labels) + if [ -z "$_sl_cur" ] || [ "$_sl_cur" = '{}' ]; then + np__node_set "$1" labels "{$2}" + else + np__node_set "$1" labels "${_sl_cur%\}},$2}" + fi + return 0 +} + +np__stage_facet() { + _sf_cur=$(np__node_get "$1" facets) + _sf_entry="$(np__json_str "$2"):$3" + if [ -z "$_sf_cur" ] || [ "$_sf_cur" = '{}' ]; then + np__node_set "$1" facets "{$_sf_entry}" + else + # Last write wins per namespace: drop any prior entry for this facet. + np__node_set "$1" facets "${_sf_cur%\}},$_sf_entry}" + fi + return 0 +} + +# Staged context normally rides the node's NEXT lifecycle emit. A FOREIGN +# (adopted) node never has one here — its owner closes it in another process — +# so anything staged on it would die in local state. Re-emit `started` with the +# full current bag instead (additive, the same shape the JS SDK's +# late-enrichment flush produces): the fold keeps the node's real outcome (the +# owner's terminal is later by time) and gains the facts this process observed. +np__flush_foreign() { + [ "$(np__node_get "$1" foreign)" = '1' ] || return 0 + np__emit_node "$1" "$NP_STATUS_STARTED" + return 0 +} + +# np_trace_labels [handle] key=value ... +np_trace_labels() { + _lb_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_lb_h" || return 0 + for _lb_pair in "$@"; do + case "$_lb_pair" in + *=*) ;; + *) continue ;; + esac + _lb_k=${_lb_pair%%=*} + _lb_v=${_lb_pair#*=} + # An absent optional is omitted, never recorded as the string "null". + if [ -n "$_lb_k" ] && [ -n "$_lb_v" ]; then + np__stage_label "$_lb_h" "$(np__json_str "$_lb_k"):$(np__json_str "$_lb_v")" + fi + done + np__flush_foreign "$_lb_h" + return 0 +} + +# np_trace_facet [handle] — your own namespace. +np_trace_facet() { + _fc_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_fc_h" || return 0 + if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then + return 0 + fi + np__stage_facet "$_fc_h" "$1" "$2" + np__flush_foreign "$_fc_h" + return 0 +} + +np_trace_schema() { + _sc_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_sc_h" || return 0 + np__node_set "$_sc_h" schema_url "${1:-}" + return 0 +} + +np_trace_explain() { + _ex_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_ex_h" || return 0 + _ex_title='' + _ex_what='' + _ex_why='' + _ex_impact='' + _ex_next='' + _ex_sev='' + while [ "$#" -gt 0 ]; do + case "$1" in + --title) _ex_title=${2:-}; shift 2 ;; + --what) _ex_what=${2:-}; shift 2 ;; + --why) _ex_why=${2:-}; shift 2 ;; + --impact) _ex_impact=${2:-}; shift 2 ;; + --next) _ex_next=${2:-}; shift 2 ;; + --severity) _ex_sev=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_ex_title" ]; then + np__drop 'explain' 'title is required' + return 0 + fi + np__stage_facet "$_ex_h" "$NP_FACET_EXPLAIN" \ + "$(np__json_obj title "$_ex_title" severity "$_ex_sev" what "$_ex_what" \ + why "$_ex_why" impact "$_ex_impact" next "$_ex_next")" + np__flush_foreign "$_ex_h" + return 0 +} + +np_trace_error() { + _er_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_er_h" || return 0 + _er_msg='' + _er_code='' + _er_stack='' + while [ "$#" -gt 0 ]; do + case "$1" in + --message) _er_msg=${2:-}; shift 2 ;; + --code) _er_code=${2:-}; shift 2 ;; + --stack-trace) _er_stack=${2:-}; shift 2 ;; + *) + if [ -z "$_er_msg" ]; then + _er_msg=$1 + fi + shift + ;; + esac + done + [ -n "$_er_msg" ] || return 0 + np__stage_facet "$_er_h" "$NP_FACET_ERROR" \ + "$(np__json_obj message "$_er_msg" code "$_er_code" stack_trace "$_er_stack")" + np__flush_foreign "$_er_h" + return 0 +} + +np_trace_timing() { + _tm_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_tm_h" || return 0 + while [ "$#" -gt 0 ]; do + case "$1" in + --started-at) np__node_set "$_tm_h" started_at "${2:-}"; shift 2 ;; + --ended-at) np__node_set "$_tm_h" ended_at "${2:-}"; shift 2 ;; + *) shift ;; + esac + done + return 0 +} + +# Stamp the auto timing facet, letting any manual override win per field. +np__stage_timing() { + _sg_started=$(np__node_get "$1" started_at) + _sg_ended=$(np__node_get "$1" ended_at) + [ -n "$_sg_started" ] || _sg_started=$(np__node_get "$1" auto_started_at) + [ -n "$_sg_ended" ] || _sg_ended=$2 + np__stage_facet "$1" "$NP_FACET_TIMING" \ + "$(np__json_obj started_at "$_sg_started" ended_at "$_sg_ended")" + return 0 +} + +# --------------------------------------------------------------------------- +# Lifecycle terminals +# --------------------------------------------------------------------------- + +# The shared terminal path. $1 = handle, $2 = status. +np__terminalize() { + np__is_handle "$1" || return 0 + if [ "$(np__node_get "$1" closed)" = '1' ]; then + return 0 + fi + # An adopted node belongs to the process that created it. Its owner decides + # its outcome; emitting a terminal here would assert a state we did not + # observe, and would race the owner's own terminal event. + if np__is_foreign "$1"; then + np__drop 'terminal' 'refusing to close an adopted node' + return 0 + fi + np_trace_start "$1" + np__stage_timing "$1" "$(np__iso8601)" + np__node_set "$1" closed 1 + np__emit_node "$1" "$2" + np__ambient_clear "$1" + # Restore the parent as ambient so a sibling opened next lands correctly. + _tz_parent=$(np__node_get "$1" parent) + if [ -n "$_tz_parent" ] && np__is_handle "$_tz_parent"; then + if [ "$(np__node_get "$_tz_parent" closed)" != '1' ]; then + np__ambient_set "$_tz_parent" + fi + fi + return 0 +} + +np_trace_complete() { + np__terminalize "$(np__resolve_handle "${1:-}")" "$NP_STATUS_COMPLETED" + return 0 +} + +# An idempotent completing close. +np_trace_end() { + np_trace_complete "$@" + return 0 +} + +np_trace_fail() { + _fa_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + # Refuse a foreign fail WHOLE, before the message stages: half-applying it + # (error facet emitted via the foreign flush, close refused) would smear an + # unowned outcome onto the node. Recording an observed fact on a foreign + # node is np_trace_error, deliberately. + if np__is_foreign "$_fa_h"; then + np__drop 'terminal' 'refusing to close an adopted node' + return 0 + fi + if [ -n "${1:-}" ]; then + np_trace_error "$_fa_h" --message "$1" + fi + # fail cascades to still-open child steps; complete deliberately does not — + # auto-completing an open child would assert a success the SDK cannot vouch + # for, and back-date its duration. + np__cascade_fail "$_fa_h" "${1:-}" + np__terminalize "$_fa_h" "$NP_STATUS_FAILED" + return 0 +} + +# True when $1 is a descendant of $2, by walking the parent chain upward. +# Deliberately NOT recursive: POSIX sh has no `local`, so a recursive walk +# clobbers its caller's loop variables — which silently skipped intermediate +# nodes in the cascade. +np__is_descendant_of() { + _dz_cur=$(np__node_get "$1" parent) + _dz_guard=0 + while [ -n "$_dz_cur" ] && [ "$_dz_guard" -lt 64 ]; do + if [ "$_dz_cur" = "$2" ]; then + return 0 + fi + _dz_cur=$(np__node_get "$_dz_cur" parent) + _dz_guard=$((_dz_guard + 1)) + done + return 1 +} + +# Fail every still-open descendant. One flat pass over the registry, deepest +# first, so a node is closed before anything reads it as a parent. +np__cascade_fail() { + _cf_depth=64 + while [ "$_cf_depth" -ge 0 ]; do + for _cf_file in "$NP_TRACE_DIR/nodes"/*; do + [ -f "$_cf_file" ] || continue + _cf_h=${_cf_file##*/} + [ "$_cf_h" = "$1" ] && continue + [ "$(np__node_get "$_cf_h" closed)" = '1' ] && continue + np__is_descendant_of "$_cf_h" "$1" || continue + [ "$(np__depth_of "$_cf_h")" -eq "$_cf_depth" ] || continue + if [ -n "$2" ]; then + np_trace_error "$_cf_h" --message "$2" + fi + np__terminalize "$_cf_h" "$NP_STATUS_FAILED" + done + _cf_depth=$((_cf_depth - 1)) + done + return 0 +} + +# How many parent links sit above this node. +np__depth_of() { + _do_cur=$(np__node_get "$1" parent) + _do_n=0 + while [ -n "$_do_cur" ] && [ "$_do_n" -lt 64 ]; do + _do_n=$((_do_n + 1)) + _do_cur=$(np__node_get "$_do_cur" parent) + done + printf '%s' "$_do_n" +} + +np_trace_skip() { + _sk_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_sk_h" || return 0 + if [ -n "${1:-}" ]; then + np__stage_facet "$_sk_h" "$NP_FACET_DROPPED" "$(np__json_obj reason "$1")" + fi + np__terminalize "$_sk_h" "$NP_STATUS_SKIPPED" + return 0 +} + +np_trace_cancel() { + _cn_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__terminalize "$_cn_h" "$NP_STATUS_CANCELLED" + return 0 +} + +np_trace_timeout() { + np__terminalize "$(np__resolve_handle "${1:-}")" "$NP_STATUS_TIMED_OUT" + return 0 +} + +# Non-terminal: the node stays open. +np_trace_waiting() { + _wt_h=$(np__resolve_handle "${1:-}") + np__is_handle "$_wt_h" || return 0 + np_trace_start "$_wt_h" + np__emit_node "$_wt_h" "$NP_STATUS_WAITING" + return 0 +} + +# ---- src/cli.sh ---- +# cli.sh — argv to function shim (Phase 2). From 8170c7a508490be994914c1cf5fd42c1ca26ed77 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 00:13:30 -0300 Subject: [PATCH 02/52] feat(k8s): trace waits as sub-steps with live progress, cover every wait loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phases inside a script can now be their own step in the trace: np_scope_step_begin/end/timeout open a keyed sub-step under the platform step, log errors and heartbeats attach to the innermost open node, a sub-step still open when the shell dies inherits the shell's outcome, and a wait that hits its deadline closes as timed_out rather than failed. Every wait loop is instrumented — wait_for_alb and wait_deployment_active upgraded from bare heartbeats to sub-steps (rollout heartbeats now carry desired/launched/ready/available/updated replica counts as labels), and wait_on_balancer, verify_ingress_reconciliation and verify_http_route_reconciliation gain sub-steps and heartbeats. scheduled_task/logging picks up the same tracing hooks, so scheduled-task scopes trace like k8s ones. The blue-green switch needs no iteration steps: it is one-shot, and switch_traffic.yaml's named steps already are the plan. --- .../verify_http_route_reconciliation | 16 ++ k8s/deployment/verify_ingress_reconciliation | 19 ++ k8s/deployment/wait_deployment_active | 18 +- k8s/logging | 138 +++++++++++-- k8s/scope/networking/wait_for_alb | 19 +- k8s/scope/wait_on_balancer | 18 ++ k8s/utils/tests/trace_logging.bats | 102 +++++++++ scheduled_task/logging | 194 ++++++++++++++++++ 8 files changed, 501 insertions(+), 23 deletions(-) diff --git a/k8s/deployment/verify_http_route_reconciliation b/k8s/deployment/verify_http_route_reconciliation index 5e71e88c..6962efc6 100644 --- a/k8s/deployment/verify_http_route_reconciliation +++ b/k8s/deployment/verify_http_route_reconciliation @@ -11,6 +11,13 @@ elapsed=0 log debug "🔍 Verifying HTTPRoute reconciliation..." log debug "📋 HTTPRoute: $HTTPROUTE_NAME | Namespace: $K8S_NAMESPACE | Timeout: ${MAX_WAIT_SECONDS}s" +# The reconciliation wait is its own SUB-STEP in the trace. (Guarded: +# overrides may reuse this script without k8s/logging loaded.) +if command -v np_scope_step_begin >/dev/null 2>&1; then + np_scope_step_begin verify-httproute --title "Verify HTTPRoute reconciliation ($HTTPROUTE_NAME)" + np_scope_wait_heartbeat "httproute-reconciliation" 0 "$MAX_WAIT_SECONDS" "pending" +fi + while [ $elapsed -lt $MAX_WAIT_SECONDS ]; do sleep $CHECK_INTERVAL @@ -43,6 +50,9 @@ while [ $elapsed -lt $MAX_WAIT_SECONDS ]; do if [ "$accepted_status" == "True" ] && [ "$resolved_status" == "True" ]; then log info "✅ HTTPRoute successfully reconciled (Accepted: True, ResolvedRefs: True)" + if command -v np_scope_step_end >/dev/null 2>&1; then + np_scope_step_end 0 + fi return 0 fi @@ -106,6 +116,9 @@ while [ $elapsed -lt $MAX_WAIT_SECONDS ]; do log debug "📝 HTTPRoute reconciling... (${elapsed}s/${MAX_WAIT_SECONDS}s)" echo "$conditions" | jq -r '.[] | " - \(.type): \(.status) (\(.reason))"' + if [ $((elapsed % 30)) -eq 0 ] && command -v np_scope_wait_heartbeat >/dev/null 2>&1; then + np_scope_wait_heartbeat "httproute-reconciliation" "$elapsed" "$MAX_WAIT_SECONDS" "reconciling" + fi elapsed=$((elapsed + CHECK_INTERVAL)) done @@ -120,4 +133,7 @@ echo "$httproute_json" | jq -r '.status.parents[0].conditions[] | " - \(.type) log error "🔧 How to fix:" log error " - Check Gateway controller logs" log error " - Verify Gateway and Istio configuration" +if command -v np_scope_step_timeout >/dev/null 2>&1; then + np_scope_step_timeout "HTTPRoute $HTTPROUTE_NAME not reconciled after ${MAX_WAIT_SECONDS}s" +fi exit 1 diff --git a/k8s/deployment/verify_ingress_reconciliation b/k8s/deployment/verify_ingress_reconciliation index 759767b5..a75f47dc 100644 --- a/k8s/deployment/verify_ingress_reconciliation +++ b/k8s/deployment/verify_ingress_reconciliation @@ -247,10 +247,20 @@ validate_alb_config() { fi } +# The reconciliation wait is its own SUB-STEP in the trace. (Guarded: +# overrides may reuse this script without k8s/logging loaded.) +if command -v np_scope_step_begin >/dev/null 2>&1; then + np_scope_step_begin verify-ingress --title "Verify ingress reconciliation ($INGRESS_NAME)" + np_scope_wait_heartbeat "ingress-reconciliation" 0 "$MAX_WAIT_SECONDS" "pending" +fi + while [ $elapsed -lt $MAX_WAIT_SECONDS ]; do if [ "$ALB_RECONCILIATION_ENABLED" = "true" ]; then if validate_alb_config; then log info "✅ ALB configuration validated successfully" + if command -v np_scope_step_end >/dev/null 2>&1; then + np_scope_step_end 0 + fi return 0 fi log debug "📝 ALB validation incomplete, checking Kubernetes events..." @@ -281,6 +291,9 @@ while [ $elapsed -lt $MAX_WAIT_SECONDS ]; do log debug "📝 Ingress reported as reconciled, but ALB validation has not passed yet" else log info "✅ Ingress successfully reconciled" + if command -v np_scope_step_end >/dev/null 2>&1; then + np_scope_step_end 0 + fi return 0 fi fi @@ -311,6 +324,9 @@ while [ $elapsed -lt $MAX_WAIT_SECONDS ]; do fi log debug "📝 Waiting for ALB reconciliation... (${elapsed}s/${MAX_WAIT_SECONDS}s)" + if [ $((elapsed % 30)) -eq 0 ] && command -v np_scope_wait_heartbeat >/dev/null 2>&1; then + np_scope_wait_heartbeat "ingress-reconciliation" "$elapsed" "$MAX_WAIT_SECONDS" "reconciling" + fi sleep $CHECK_INTERVAL elapsed=$((elapsed + CHECK_INTERVAL)) done @@ -329,4 +345,7 @@ events_json=$(kubectl get events -n "$K8S_NAMESPACE" \ -o json) echo "$events_json" | jq -r '.items | sort_by(.lastTimestamp) | .[] | " [\(.type)] \(.reason): \(.message)"' | tail -10 +if command -v np_scope_step_timeout >/dev/null 2>&1; then + np_scope_step_timeout "ingress $INGRESS_NAME not reconciled after ${MAX_WAIT_SECONDS}s" +fi exit 1 diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index e6cc062c..d228a08e 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -95,6 +95,14 @@ log debug "📋 Namespace: $K8S_NAMESPACE" log debug "📋 Timeout: ${TIMEOUT}s (max $MAX_ITERATIONS iterations)" log debug "" +# The rollout wait is its own SUB-STEP in the trace; heartbeats carry the live +# replica counts. (Guarded: overrides may reuse this script without k8s/logging +# loaded.) +if command -v np_scope_step_begin >/dev/null 2>&1; then + np_scope_step_begin wait-deployment-active --title "Wait for deployment rollout ($K8S_DEPLOYMENT_NAME)" + np_scope_wait_heartbeat "deployment-active" 0 "$TIMEOUT" "starting" +fi + while true; do ((++iteration)) if [ $iteration -gt $MAX_ITERATIONS ]; then @@ -104,6 +112,9 @@ while true; do source "$SERVICE_PATH/deployment/print_failed_deployment_hints" + if command -v np_scope_step_timeout >/dev/null 2>&1; then + np_scope_step_timeout "deployment '$K8S_DEPLOYMENT_NAME' not active after ${TIMEOUT}s" + fi exit 1 fi @@ -144,6 +155,9 @@ while true; do if [ "$desired" = "$current" ] && [ "$desired" = "$updated" ] && [ "$desired" = "$ready" ] && [ "$desired" -gt 0 ]; then log debug "" log info "✅ All pods in deployment '$K8S_DEPLOYMENT_NAME' are available and ready!" + if command -v np_scope_step_end >/dev/null 2>&1; then + np_scope_step_end 0 + fi break fi @@ -151,7 +165,9 @@ while true; do elapsed_s=$(( iteration * 10 )) log info "⏳ Still waiting — Ready: $ready/$desired, Available: $current/$desired (attempt $iteration/$MAX_ITERATIONS, ${elapsed_s}s elapsed)" if command -v np_scope_wait_heartbeat >/dev/null 2>&1; then - np_scope_wait_heartbeat "deployment-active" "$elapsed_s" "$TIMEOUT" "ready-$ready-of-$desired" + np_scope_wait_heartbeat "deployment-active" "$elapsed_s" "$TIMEOUT" "progressing" \ + "wait.desired=$desired" "wait.launched=$launched" \ + "wait.ready=$ready" "wait.available=$current" "wait.updated=$updated" fi fi diff --git a/k8s/logging b/k8s/logging index 18db608f..17d808b8 100644 --- a/k8s/logging +++ b/k8s/logging @@ -59,21 +59,42 @@ log() { # • every `log error` lands as a tracing.error facet ON that step # • an uncaught command failure surfaces the failing command itself # • long waits report live progress (np_scope_wait_heartbeat) +# • a phase inside a script can be its own SUB-STEP in the trace +# (np_scope_step_begin / np_scope_step_end), keyed and — for things like +# progressive traffic switches — iterated, exactly like native scopes # # All of it is best-effort: no SDK file, no NP_API_KEY, or no NP_TRACE means # plain logging, byte-identical to before. A down tracing API never blocks the # workflow: every emit is a local file write, and flushes are hard-bounded. # ============================================================================= -# Record an observed failure on the step this fragment runs inside. Adoption is -# per-call on purpose: NP_TRACE changes as the CLI moves between steps, so the -# error must attach to the step CURRENT at the moment it happened. +# Resolve the node observations attach to: the open sub-step when one belongs +# to the CURRENT platform step, else a fresh adoption of NP_TRACE. Adoption is +# per-call on purpose: NP_TRACE changes as the CLI moves between steps, so an +# observation must attach to the step current at the moment it happened — and +# a sub-step opened under a PREVIOUS platform step must never swallow it. +_np_scopes_node() { + command -v np_trace_adopt >/dev/null 2>&1 || return 1 + [ -n "${NP_TRACE:-}" ] || return 1 + if [ -n "${_NP_SCOPES_SUBSTEP:-}" ]; then + if [ "${_NP_SCOPES_SUBSTEP_UNDER:-}" = "$NP_TRACE" ]; then + printf '%s' "$_NP_SCOPES_SUBSTEP" + return 0 + fi + # The platform moved on with a sub-step still open — stale; forget it. + _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" + fi + local _nd_node + _nd_node=$(np_trace_adopt 2>/dev/null) || return 1 + [ -n "$_nd_node" ] || return 1 + printf '%s' "$_nd_node" + return 0 +} + +# Record an observed failure on the current step (or open sub-step). _np_scopes_trace_error() { - command -v np_trace_adopt >/dev/null 2>&1 || return 0 - [ -n "${NP_TRACE:-}" ] || return 0 local _lt_node - _lt_node=$(np_trace_adopt 2>/dev/null) || return 0 - [ -n "$_lt_node" ] || return 0 + _lt_node=$(_np_scopes_node) || return 0 np_trace_error "$_lt_node" --message "$1" ${2:+--code "$2"} # Remember which step already carries a real message, so the exit trap does # not shadow it with a generic one. @@ -81,22 +102,100 @@ _np_scopes_trace_error() { return 0 } -# np_scope_wait_heartbeat [state] +# np_scope_step_begin [--iteration N] [--attempt N] [--title ] # -# Mark the current step `waiting` and record progress, flushed immediately so -# the wait is visible LIVE, not after the workflow ends. The flush is bounded -# to 2s per beat: with the API down, a 30s-cadence wait loses at most ~6% of -# its poll budget — the deadline is wall-clock, so the timeout is never -# extended. Always defined; a no-op when the workflow is untraced. -np_scope_wait_heartbeat() { +# Open a keyed sub-step under the step the platform is running — a phase that +# deserves its own line in the trace (a long wait, one traffic increment). The +# key names WHAT the phase is (`[A-Za-z0-9_.-]+`); --iteration distinguishes +# repeats of the same phase (traffic at 10, then 50, then 100). One sub-step +# open at a time: while open, `log error` and heartbeats attach to it, and +# np_scope_step_end closes it. Always defined; a no-op when untraced. +np_scope_step_begin() { command -v np_trace_adopt >/dev/null 2>&1 || return 0 [ -n "${NP_TRACE:-}" ] || return 0 + local _sb_key="${1:-}" _sb_title="" _sb_parent _sb_h + shift || true + local _sb_args=() + while [ "$#" -gt 0 ]; do + case "$1" in + --title) _sb_title="${2:-}"; shift 2 ;; + *) _sb_args+=("$1"); shift ;; + esac + done + # A still-open sub-step would silently become the parent — close it instead: + # phases at this level are sequential, and a dangling open one is a bug here, + # not a hierarchy. + [ -n "${_NP_SCOPES_SUBSTEP:-}" ] && np_scope_step_end 0 + _sb_parent=$(np_trace_adopt 2>/dev/null) || return 0 + [ -n "$_sb_parent" ] || return 0 + _sb_h=$(np_trace_step "$_sb_parent" "$_sb_key" ${_sb_args[@]+"${_sb_args[@]}"}) + [ -n "$_sb_h" ] || return 0 + [ -n "$_sb_title" ] && np_trace_explain "$_sb_h" --title "$_sb_title" + _NP_SCOPES_SUBSTEP="$_sb_h" + _NP_SCOPES_SUBSTEP_UNDER="$NP_TRACE" + return 0 +} + +# np_scope_step_end [rc] [message] +# +# Close the open sub-step: completed on rc 0, failed (with the message) on +# anything else. Flushed immediately so the phase transition is visible live. +np_scope_step_end() { + [ -n "${_NP_SCOPES_SUBSTEP:-}" ] || return 0 + local _se_h="$_NP_SCOPES_SUBSTEP" _se_rc="${1:-0}" + _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" + if [ "$_se_rc" -eq 0 ] 2>/dev/null; then + np_trace_complete "$_se_h" + else + # Only fabricate a generic message when the step does not already carry a + # real one (from `log error` or the exit trap) — never shadow the actual + # failure with "phase exited with status 1". + if [ -n "${2:-}" ]; then + np_trace_fail "$_se_h" "$2" + elif [ "${_NP_SCOPES_ERRED_ON:-}" != "${NP_TRACE:-}" ]; then + np_trace_fail "$_se_h" "phase exited with status $_se_rc" + else + np_trace_fail "$_se_h" + fi + _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" + fi + NP_TRACE_FLUSH_TIMEOUT=2 np_trace_flush + return 0 +} + +# np_scope_step_timeout [message] +# +# Close the open sub-step as timed_out — the truthful terminal state for a +# wait that hit its deadline (distinct from failed: nothing broke, time ran +# out). The message, when given, is recorded as the step's error detail. +np_scope_step_timeout() { + [ -n "${_NP_SCOPES_SUBSTEP:-}" ] || return 0 + local _st_h="$_NP_SCOPES_SUBSTEP" + _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" + [ -n "${1:-}" ] && np_trace_error "$_st_h" --message "$1" + np_trace_timeout "$_st_h" + NP_TRACE_FLUSH_TIMEOUT=2 np_trace_flush + return 0 +} + +# np_scope_wait_heartbeat [state] [k=v ...] +# +# Mark the current step (or open sub-step) `waiting` and record progress, +# flushed immediately so the wait is visible LIVE, not after the workflow +# ends. Extra k=v pairs carry structured progress — replica counts, ALB state +# — as labels on the node. The flush is bounded to 2s per beat: with the API +# down, a 30s-cadence wait loses at most ~6% of its poll budget — the deadline +# is wall-clock, so the timeout is never extended. Always defined; a no-op +# when the workflow is untraced. +np_scope_wait_heartbeat() { local _hb_node - _hb_node=$(np_trace_adopt 2>/dev/null) || return 0 - [ -n "$_hb_node" ] || return 0 + _hb_node=$(_np_scopes_node) || return 0 + local _hb_what="${1:-}" _hb_elapsed="${2:-0}" _hb_timeout="${3:-0}" _hb_state="${4:-}" + shift 4 2>/dev/null || shift $# np_trace_labels "$_hb_node" \ - "wait.what=${1:-}" "wait.elapsed_s=${2:-0}" "wait.timeout_s=${3:-0}" \ - ${4:+"wait.state=$4"} + "wait.what=$_hb_what" "wait.elapsed_s=$_hb_elapsed" "wait.timeout_s=$_hb_timeout" \ + ${_hb_state:+"wait.state=$_hb_state"} \ + ${1+"$@"} np_trace_waiting "$_hb_node" NP_TRACE_FLUSH_TIMEOUT=2 np_trace_flush return 0 @@ -115,6 +214,9 @@ _np_scopes_on_exit() { _np_scopes_trace_error \ "${_NP_SCOPES_LAST_ERR:-workflow shell exited with status $_ex_rc}" || true fi + # A sub-step still open when the shell dies inherits the shell's outcome — + # closed AFTER the error report above so the failure detail lands on it. + [ -n "${_NP_SCOPES_SUBSTEP:-}" ] && np_scope_step_end "$_ex_rc" || true np_trace_flush } diff --git a/k8s/scope/networking/wait_for_alb b/k8s/scope/networking/wait_for_alb index 7cbb4f4f..48abf572 100644 --- a/k8s/scope/networking/wait_for_alb +++ b/k8s/scope/networking/wait_for_alb @@ -49,10 +49,12 @@ polls_since_heartbeat=0 heartbeats_emitted=0 log info "⏳ Waiting up to ${TIMEOUT_SECONDS}s for ALB '$ALB_NAME' to become active..." -# Flip the step to `waiting` in the trace right away — an operator watching the -# provision sees WHAT it is blocked on, live, not after the workflow ends. -# (Guarded: overrides may reuse this script without k8s/logging loaded.) -if command -v np_scope_wait_heartbeat >/dev/null 2>&1; then +# The wait is its own SUB-STEP in the trace, flipped to `waiting` right away — +# an operator watching the provision sees WHAT it is blocked on, live, not +# after the workflow ends. (Guarded: overrides may reuse this script without +# k8s/logging loaded.) +if command -v np_scope_step_begin >/dev/null 2>&1; then + np_scope_step_begin wait-alb-active --title "Wait for ALB '$ALB_NAME' to become active" np_scope_wait_heartbeat "alb-active" 0 "$TIMEOUT_SECONDS" "pending" fi @@ -102,9 +104,18 @@ if [ "$state" != "active" ]; then log error "🔧 How to fix:" log error " • Check controller logs: kubectl -n kube-system logs deploy/aws-load-balancer-controller" log error " • Verify ALB quota: aws service-quotas get-service-quota --service-code elasticloadbalancing --quota-code L-53DA6B97" + if command -v np_scope_step_timeout >/dev/null 2>&1; then + np_scope_step_timeout "ALB '$ALB_NAME' not active after ${TIMEOUT_SECONDS}s (last state: ${state:-pending})" + fi exit 1 fi +# The wait phase is over and the ALB is active — close its sub-step; the audit +# tagging below is not part of the wait. +if command -v np_scope_step_end >/dev/null 2>&1; then + np_scope_step_end 0 +fi + # Audit tags — only on the scope that triggered the autocreate, so the cloud # carries the lineage of which scope created which ALB. Failure is non-fatal: # the provider registration (the authoritative source) already succeeded; the diff --git a/k8s/scope/wait_on_balancer b/k8s/scope/wait_on_balancer index bde5cfec..72674d8e 100644 --- a/k8s/scope/wait_on_balancer +++ b/k8s/scope/wait_on_balancer @@ -18,6 +18,13 @@ case "$DNS_TYPE" in log debug "📋 Checking ExternalDNS record creation for domain: $SCOPE_DOMAIN" + # The DNS wait is its own SUB-STEP in the trace, heartbeating every ~30s. + # (Guarded: overrides may reuse this script without k8s/logging loaded.) + if command -v np_scope_step_begin >/dev/null 2>&1; then + np_scope_step_begin wait-dns-endpoint --title "Wait for ExternalDNS to process $DNS_ENDPOINT_NAME" + np_scope_wait_heartbeat "dns-endpoint" 0 "$((MAX_ITERATIONS * 10))" "pending" + fi + while true; do iteration=$((iteration + 1)) if [ $iteration -gt $MAX_ITERATIONS ]; then @@ -31,6 +38,9 @@ case "$DNS_TYPE" in log error " • Check DNSEndpoint resources: kubectl get dnsendpoint -A" log error " • Check ExternalDNS logs: kubectl logs -n external-dns -l app=external-dns --tail=50" log error "" + if command -v np_scope_step_timeout >/dev/null 2>&1; then + np_scope_step_timeout "DNSEndpoint $DNS_ENDPOINT_NAME not processed after $((MAX_ITERATIONS * 10))s" + fi exit 1 fi @@ -44,10 +54,18 @@ case "$DNS_TYPE" in break fi + if [ $((iteration % 3)) -eq 0 ] && command -v np_scope_wait_heartbeat >/dev/null 2>&1; then + np_scope_wait_heartbeat "dns-endpoint" "$((iteration * 10))" "$((MAX_ITERATIONS * 10))" "pending" + fi + log debug "📋 DNSEndpoint not yet processed, waiting 10s..." sleep 10 done + if command -v np_scope_step_end >/dev/null 2>&1; then + np_scope_step_end 0 + fi + log info "" log info "✨ ExternalDNS setup completed successfully" ;; diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index f2336a24..f2f16ffb 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -141,3 +141,105 @@ hello" ] run "$BASH" -c "source '$LOGGING'; log error 'to-stderr' 2>&1 >/dev/null" [[ "$output" == *"to-stderr"* ]] } + +# --- sub-steps -------------------------------------------------------------- + +@test "step_begin opens a keyed sub-step under the platform step" { + # The title rides the next lifecycle emit (own-node enrichment is bagged, + # not emitted eagerly), so close the step to see it on the wire. + run_logged 'np_scope_step_begin wait-alb-active --title "Wait for the ALB"; np_scope_step_end 0' + [ "$status" -eq 0 ] + echo "$output" | grep -q '"run_id":"scope-provision-42~apply-manifests@0.0~wait-alb-active@0.0"' + echo "$output" | grep -q '"status":"started"' + echo "$output" | grep -q 'Wait for the ALB' +} + +@test "step_end 0 completes the sub-step" { + run_logged 'np_scope_step_begin wait-alb-active; np_scope_step_end 0' + [ "$status" -eq 0 ] + echo "$output" | grep '"status":"completed"' | grep -q 'wait-alb-active@0.0' +} + +@test "step_end with a non-zero rc fails the sub-step with the message" { + run_logged 'np_scope_step_begin wait-alb-active; np_scope_step_end 1 "ALB never came up"' + [ "$status" -eq 0 ] + echo "$output" | grep '"status":"failed"' | grep -q 'wait-alb-active@0.0' + echo "$output" | grep -q 'ALB never came up' +} + +@test "step_timeout closes the sub-step as timed_out, not failed" { + run_logged 'np_scope_step_begin wait-alb-active; np_scope_step_timeout "deadline hit"' + [ "$status" -eq 0 ] + echo "$output" | grep '"status":"timed_out"' | grep -q 'wait-alb-active@0.0' + echo "$output" | grep -q 'deadline hit' + ! echo "$output" | grep '"status":"failed"' | grep -q 'wait-alb-active@0.0' +} + +@test "while a sub-step is open, log error attaches to IT, not the platform step" { + # The facet rides the step's closing emit — and because a real message was + # already recorded, the close adds no generic shadow next to it. + run_logged 'np_scope_step_begin wait-alb-active; log error "quota exceeded"; np_scope_step_end 1' + [ "$status" -eq 0 ] + echo "$output" | grep '"quota exceeded"' | grep -q 'wait-alb-active@0.0' + ! echo "$output" | grep -q 'phase exited with status' +} + +@test "while a sub-step is open, heartbeats attach to it" { + run_logged 'np_scope_step_begin wait-alb-active; np_scope_wait_heartbeat "alb-active" 30 300 "pending"' + [ "$status" -eq 0 ] + echo "$output" | grep '"status":"waiting"' | grep -q 'wait-alb-active@0.0' +} + +@test "heartbeat extra k=v pairs land as labels" { + run_logged 'np_scope_wait_heartbeat "deployment-active" 20 600 "progressing" "wait.ready=2" "wait.desired=5"' + [ "$status" -eq 0 ] + echo "$output" | grep -q '"wait.ready":"2"' + echo "$output" | grep -q '"wait.desired":"5"' +} + +@test "a sub-step left open when the platform moves on is forgotten, not reused" { + run_logged ' + np_scope_step_begin wait-alb-active + export NP_TRACE="1|trace-9|scope-provision-42~wait-for-alb@0.0" + log error "late failure" + ' + [ "$status" -eq 0 ] + # the error lands on the NEW platform step, not the stale sub-step + echo "$output" | grep '"late failure"' | grep -q 'scope-provision-42~wait-for-alb@0.0"' +} + +@test "a sub-step still open when the shell dies inherits the failure" { + run "$BASH" -c " + ( source '$LOGGING'; np_scope_step_begin wait-alb-active; log error 'the real reason'; exit 3 ) || true + source '$LOGGING'; trap - EXIT ERR + for f in \"\$NP_TRACE_DIR\"/spool/*.json \"\$NP_TRACE_DIR\"/failed/*.json; do + [ -f \"\$f\" ] && cat \"\$f\" && echo + done + true + " + [ "$status" -eq 0 ] + echo "$output" | grep '"status":"failed"' | grep -q 'wait-alb-active@0.0' + echo "$output" | grep '"the real reason"' | grep -q 'wait-alb-active@0.0' + # no generic 'phase exited' shadow next to the real message + ! echo "$output" | grep -q 'phase exited with status' +} + +@test "opening a second sub-step completes the first — phases are sequential" { + run_logged 'np_scope_step_begin phase-one; np_scope_step_begin phase-two; np_scope_step_end 0' + [ "$status" -eq 0 ] + echo "$output" | grep '"status":"completed"' | grep -q 'phase-one@0.0' + echo "$output" | grep '"status":"completed"' | grep -q 'phase-two@0.0' +} + +@test "step functions are defined no-ops when the workflow is untraced" { + unset NP_TRACE + run "$BASH" -c "source '$LOGGING'; np_scope_step_begin x; np_scope_step_end 1; np_scope_step_timeout; echo rc=\$?" + [ "$status" -eq 0 ] + [[ "$output" == *"rc=0"* ]] +} + +@test "step_end without an open sub-step is a no-op" { + run_logged 'np_scope_step_end 1 "nothing open"' + [ "$status" -eq 0 ] + ! echo "$output" | grep -q 'nothing open' +} diff --git a/scheduled_task/logging b/scheduled_task/logging index d0df55d7..17d808b8 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -38,4 +38,198 @@ log() { echo "$message" fi fi + + # Every `log error`, anywhere in any script, also lands on the TRACE — as a + # tracing.error facet on the workflow step it happened in. This is what makes + # a failed custom scope read like a failed native one: not just "step exited + # 1", but the actual message. A no-op unless the platform launched this + # workflow traced (see the tracing section below). + if [ "$msg_num" -ge 3 ]; then + _np_scopes_trace_error "$message" || true + fi } + +# ============================================================================= +# Tracing — best-effort, structural +# +# When the platform launches a workflow, NP_TRACE carries the trace context of +# the step each fragment runs inside (the np CLI re-points it per step). With +# the vendored shell SDK sourced: +# +# • every `log error` lands as a tracing.error facet ON that step +# • an uncaught command failure surfaces the failing command itself +# • long waits report live progress (np_scope_wait_heartbeat) +# • a phase inside a script can be its own SUB-STEP in the trace +# (np_scope_step_begin / np_scope_step_end), keyed and — for things like +# progressive traffic switches — iterated, exactly like native scopes +# +# All of it is best-effort: no SDK file, no NP_API_KEY, or no NP_TRACE means +# plain logging, byte-identical to before. A down tracing API never blocks the +# workflow: every emit is a local file write, and flushes are hard-bounded. +# ============================================================================= + +# Resolve the node observations attach to: the open sub-step when one belongs +# to the CURRENT platform step, else a fresh adoption of NP_TRACE. Adoption is +# per-call on purpose: NP_TRACE changes as the CLI moves between steps, so an +# observation must attach to the step current at the moment it happened — and +# a sub-step opened under a PREVIOUS platform step must never swallow it. +_np_scopes_node() { + command -v np_trace_adopt >/dev/null 2>&1 || return 1 + [ -n "${NP_TRACE:-}" ] || return 1 + if [ -n "${_NP_SCOPES_SUBSTEP:-}" ]; then + if [ "${_NP_SCOPES_SUBSTEP_UNDER:-}" = "$NP_TRACE" ]; then + printf '%s' "$_NP_SCOPES_SUBSTEP" + return 0 + fi + # The platform moved on with a sub-step still open — stale; forget it. + _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" + fi + local _nd_node + _nd_node=$(np_trace_adopt 2>/dev/null) || return 1 + [ -n "$_nd_node" ] || return 1 + printf '%s' "$_nd_node" + return 0 +} + +# Record an observed failure on the current step (or open sub-step). +_np_scopes_trace_error() { + local _lt_node + _lt_node=$(_np_scopes_node) || return 0 + np_trace_error "$_lt_node" --message "$1" ${2:+--code "$2"} + # Remember which step already carries a real message, so the exit trap does + # not shadow it with a generic one. + _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" + return 0 +} + +# np_scope_step_begin [--iteration N] [--attempt N] [--title ] +# +# Open a keyed sub-step under the step the platform is running — a phase that +# deserves its own line in the trace (a long wait, one traffic increment). The +# key names WHAT the phase is (`[A-Za-z0-9_.-]+`); --iteration distinguishes +# repeats of the same phase (traffic at 10, then 50, then 100). One sub-step +# open at a time: while open, `log error` and heartbeats attach to it, and +# np_scope_step_end closes it. Always defined; a no-op when untraced. +np_scope_step_begin() { + command -v np_trace_adopt >/dev/null 2>&1 || return 0 + [ -n "${NP_TRACE:-}" ] || return 0 + local _sb_key="${1:-}" _sb_title="" _sb_parent _sb_h + shift || true + local _sb_args=() + while [ "$#" -gt 0 ]; do + case "$1" in + --title) _sb_title="${2:-}"; shift 2 ;; + *) _sb_args+=("$1"); shift ;; + esac + done + # A still-open sub-step would silently become the parent — close it instead: + # phases at this level are sequential, and a dangling open one is a bug here, + # not a hierarchy. + [ -n "${_NP_SCOPES_SUBSTEP:-}" ] && np_scope_step_end 0 + _sb_parent=$(np_trace_adopt 2>/dev/null) || return 0 + [ -n "$_sb_parent" ] || return 0 + _sb_h=$(np_trace_step "$_sb_parent" "$_sb_key" ${_sb_args[@]+"${_sb_args[@]}"}) + [ -n "$_sb_h" ] || return 0 + [ -n "$_sb_title" ] && np_trace_explain "$_sb_h" --title "$_sb_title" + _NP_SCOPES_SUBSTEP="$_sb_h" + _NP_SCOPES_SUBSTEP_UNDER="$NP_TRACE" + return 0 +} + +# np_scope_step_end [rc] [message] +# +# Close the open sub-step: completed on rc 0, failed (with the message) on +# anything else. Flushed immediately so the phase transition is visible live. +np_scope_step_end() { + [ -n "${_NP_SCOPES_SUBSTEP:-}" ] || return 0 + local _se_h="$_NP_SCOPES_SUBSTEP" _se_rc="${1:-0}" + _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" + if [ "$_se_rc" -eq 0 ] 2>/dev/null; then + np_trace_complete "$_se_h" + else + # Only fabricate a generic message when the step does not already carry a + # real one (from `log error` or the exit trap) — never shadow the actual + # failure with "phase exited with status 1". + if [ -n "${2:-}" ]; then + np_trace_fail "$_se_h" "$2" + elif [ "${_NP_SCOPES_ERRED_ON:-}" != "${NP_TRACE:-}" ]; then + np_trace_fail "$_se_h" "phase exited with status $_se_rc" + else + np_trace_fail "$_se_h" + fi + _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" + fi + NP_TRACE_FLUSH_TIMEOUT=2 np_trace_flush + return 0 +} + +# np_scope_step_timeout [message] +# +# Close the open sub-step as timed_out — the truthful terminal state for a +# wait that hit its deadline (distinct from failed: nothing broke, time ran +# out). The message, when given, is recorded as the step's error detail. +np_scope_step_timeout() { + [ -n "${_NP_SCOPES_SUBSTEP:-}" ] || return 0 + local _st_h="$_NP_SCOPES_SUBSTEP" + _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" + [ -n "${1:-}" ] && np_trace_error "$_st_h" --message "$1" + np_trace_timeout "$_st_h" + NP_TRACE_FLUSH_TIMEOUT=2 np_trace_flush + return 0 +} + +# np_scope_wait_heartbeat [state] [k=v ...] +# +# Mark the current step (or open sub-step) `waiting` and record progress, +# flushed immediately so the wait is visible LIVE, not after the workflow +# ends. Extra k=v pairs carry structured progress — replica counts, ALB state +# — as labels on the node. The flush is bounded to 2s per beat: with the API +# down, a 30s-cadence wait loses at most ~6% of its poll budget — the deadline +# is wall-clock, so the timeout is never extended. Always defined; a no-op +# when the workflow is untraced. +np_scope_wait_heartbeat() { + local _hb_node + _hb_node=$(_np_scopes_node) || return 0 + local _hb_what="${1:-}" _hb_elapsed="${2:-0}" _hb_timeout="${3:-0}" _hb_state="${4:-}" + shift 4 2>/dev/null || shift $# + np_trace_labels "$_hb_node" \ + "wait.what=$_hb_what" "wait.elapsed_s=$_hb_elapsed" "wait.timeout_s=$_hb_timeout" \ + ${_hb_state:+"wait.state=$_hb_state"} \ + ${1+"$@"} + np_trace_waiting "$_hb_node" + NP_TRACE_FLUSH_TIMEOUT=2 np_trace_flush + return 0 +} + +# A silent failure (no `log error` on the way down) must still be clear: the +# ERR trap remembers the last failing top-level command, and the EXIT trap +# reports it against the step that was current when the shell died. +_np_scopes_on_err() { + _NP_SCOPES_LAST_ERR="$BASH_COMMAND (exit ${1:-1})" +} + +_np_scopes_on_exit() { + local _ex_rc="${1:-0}" + if [ "$_ex_rc" -ne 0 ] && [ "${_NP_SCOPES_ERRED_ON:-}" != "${NP_TRACE:-}" ]; then + _np_scopes_trace_error \ + "${_NP_SCOPES_LAST_ERR:-workflow shell exited with status $_ex_rc}" || true + fi + # A sub-step still open when the shell dies inherits the shell's outcome — + # closed AFTER the error report above so the failure detail lands on it. + [ -n "${_NP_SCOPES_SUBSTEP:-}" ] && np_scope_step_end "$_ex_rc" || true + np_trace_flush +} + +_NP_SCOPES_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [ -z "${NP_TRACE_LOADED:-}" ] \ + && [ -f "$_NP_SCOPES_ROOT/nptrace.sh" ] \ + && [ -n "${NP_API_KEY:-}" ] \ + && [ -n "${NP_TRACE:-}" ]; then + # shellcheck source=/dev/null + . "$_NP_SCOPES_ROOT/nptrace.sh" + # --no-trap: the exit flush is ours, so the uncaught-failure report and the + # flush share ONE trap in a defined order. + np_trace_init --producer "nullplatform-scopes@1" --api-key "$NP_API_KEY" --no-trap + trap '_np_scopes_on_err $?' ERR + trap '_np_scopes_on_exit $?' EXIT +fi From f6e51cfae32699b5eb02ae44d69d41a2bbb87984 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 10:16:07 -0300 Subject: [PATCH 03/52] =?UTF-8?q?feat(k8s):=20SWM-grade=20lineage=20?= =?UTF-8?q?=E2=80=94=20what=20each=20step=20wrote,=20read=20and=20offers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Custom scopes now report the same data lineage the scope-workflow-manager does, by the same canonical dataset ids so the graphs join by value: • apply_templates turns kubectl apply output into produces edges for the kinds the platform lineage model knows (k8s-deployment, k8s-service, k8s-ingress) — one chokepoint covers every workflow • the namespace create records k8s-namespace: • manage_dns CREATE records dns-record: • wait_for_alb records the load-balancer: dependency • wait_deployment_active records the cross-flow join back to the build (consumes :), produces this deploy's log dataset with the concrete log query as its pointer, declares the deploy-log affordance the UI renders as "view logs", and reports rollout progress ({current, target, unit: instances}) per heartbeat Vendored nptrace.sh rebuilt from catalog-tracing-sh#6, which adds np_trace_produces/consumes/affordances/progress to the shell SDK. --- k8s/apply_templates | 10 +- k8s/deployment/wait_deployment_active | 27 ++++ k8s/logging | 78 +++++++++++ k8s/scope/build_context | 3 + k8s/scope/networking/dns/manage_dns | 7 + k8s/scope/networking/wait_for_alb | 6 + k8s/utils/tests/trace_logging.bats | 59 +++++++++ nptrace.sh | 182 ++++++++++++++++++++++++++ scheduled_task/logging | 78 +++++++++++ 9 files changed, 449 insertions(+), 1 deletion(-) diff --git a/k8s/apply_templates b/k8s/apply_templates index 3a5dfaa4..384f7890 100644 --- a/k8s/apply_templates +++ b/k8s/apply_templates @@ -31,7 +31,15 @@ while IFS= read -r TEMPLATE_FILE; do IGNORE_NOT_FOUND="--ignore-not-found=true" fi - if ! kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND; then + # Captured (and re-echoed) so applied resources can be recorded as + # lineage on the trace; the console output is unchanged. + if KUBECTL_OUT=$(kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND); then + [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" + if [[ "$ACTION" == "apply" ]] && command -v np_scope_k8s_applied >/dev/null 2>&1; then + np_scope_k8s_applied "${K8S_NAMESPACE:-}" "$KUBECTL_OUT" + fi + else + [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" log error " ❌ Failed to apply" fi fi diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index d228a08e..c7de39b0 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -156,6 +156,32 @@ while true; do log debug "" log info "✅ All pods in deployment '$K8S_DEPLOYMENT_NAME' are available and ready!" if command -v np_scope_step_end >/dev/null 2>&1; then + np_scope_progress "$ready" "$desired" instances + + # Lineage, by the same canonical ids the platform's own scopes use, + # so a custom scope's graph joins by value: + # • the deployment CONSUMED the build's asset — the cross-flow + # edge that links this deploy back to the build that made it + # • it PRODUCED this deploy's logs; the pointer is the concrete + # log query, and the affordance is the UI's "view logs" control + _wda_asset_url=$(echo "$CONTEXT" | jq -r '.asset.url // empty') + _wda_asset_type=$(echo "$CONTEXT" | jq -r '.asset.type // "docker-image"') + if [ -n "$_wda_asset_url" ]; then + np_scope_consumes "$_wda_asset_type:$_wda_asset_url" image "$_wda_asset_url" + fi + _wda_app_id=$(echo "$CONTEXT" | jq -r '.application.id // empty') + _wda_scope_id=$(echo "$CONTEXT" | jq -r '.scope.id // empty') + _wda_created=$(echo "$CONTEXT" | jq -r '.deployment.created_at // empty') + _wda_start_ms=$(date -d "$_wda_created" +%s000 2>/dev/null || echo "") + if [ -n "$_wda_app_id" ] && [ -n "$_wda_scope_id" ]; then + _wda_log_uri="application/$_wda_app_id/log?scope=$_wda_scope_id&type=application" + [ -n "$_wda_start_ms" ] && _wda_log_uri="$_wda_log_uri&start_time=$_wda_start_ms" + np_scope_produces \ + "deployment-log:$_wda_app_id/$_wda_scope_id/$DEPLOYMENT_ID" \ + log "$_wda_log_uri" + np_scope_affordance "{\"kind\":\"deploy-log\",\"application_id\":\"$_wda_app_id\",\"scope_id\":\"$_wda_scope_id\",\"type\":\"application\"${_wda_start_ms:+,\"start_time\":$_wda_start_ms}}" + fi + np_scope_step_end 0 fi break @@ -168,6 +194,7 @@ while true; do np_scope_wait_heartbeat "deployment-active" "$elapsed_s" "$TIMEOUT" "progressing" \ "wait.desired=$desired" "wait.launched=$launched" \ "wait.ready=$ready" "wait.available=$current" "wait.updated=$updated" + np_scope_progress "$ready" "$desired" instances fi fi diff --git a/k8s/logging b/k8s/logging index 17d808b8..dc6fd3e0 100644 --- a/k8s/logging +++ b/k8s/logging @@ -201,6 +201,84 @@ np_scope_wait_heartbeat() { return 0 } +# np_scope_produces [ ] +# np_scope_consumes [ ] +# +# Data lineage: declare what the current step (or open sub-step) WROTE or +# READ, by CANONICAL dataset id — the exact grammar the scope-workflow-manager +# uses, so a custom scope's lineage joins the platform's by value: +# +# k8s-namespace: k8s-deployment:/ +# k8s-service:/ k8s-ingress:/ +# dns-record: load-balancer: +# docker-image: deployment-log:// +# +# Always defined; a no-op when the workflow is untraced. +np_scope_produces() { + command -v np_trace_produces >/dev/null 2>&1 || return 0 + local _pd_node + _pd_node=$(_np_scopes_node) || return 0 + np_trace_produces "$_pd_node" "$1" ${2:+--name "$2"} ${3:+--uri "$3"} + return 0 +} + +np_scope_consumes() { + command -v np_trace_consumes >/dev/null 2>&1 || return 0 + local _cd_node + _cd_node=$(_np_scopes_node) || return 0 + np_trace_consumes "$_cd_node" "$1" ${2:+--name "$2"} ${3:+--uri "$3"} + return 0 +} + +# np_scope_affordance +# +# What the current step OFFERS a human to do — the UI renders it as a control +# (e.g. '{"kind":"deploy-log","application_id":"7","scope_id":"42",...}'). +np_scope_affordance() { + command -v np_trace_affordances >/dev/null 2>&1 || return 0 + local _ad_node + _ad_node=$(_np_scopes_node) || return 0 + np_trace_affordances "$_ad_node" "$1" + return 0 +} + +# np_scope_progress [unit] +# +# A converging phase's advance toward its target (instances 3 of 10). +np_scope_progress() { + command -v np_trace_progress >/dev/null 2>&1 || return 0 + local _pg_node + _pg_node=$(_np_scopes_node) || return 0 + np_trace_progress "$_pg_node" "$1" "$2" "${3:-}" + return 0 +} + +# np_scope_k8s_applied +# +# Turn `kubectl apply` output lines (`deployment.apps/name created`) into +# lineage on the current step, for the resource kinds the platform's lineage +# model knows. One chokepoint (apply_templates) covers every workflow. +np_scope_k8s_applied() { + command -v np_trace_produces >/dev/null 2>&1 || return 0 + local _ka_ns="$1" _ka_line _ka_kind _ka_name + [ -n "$_ka_ns" ] || return 0 + while IFS= read -r _ka_line; do + [ -n "$_ka_line" ] || continue + _ka_kind="${_ka_line%%/*}" + _ka_name="${_ka_line#*/}" + _ka_name="${_ka_name%% *}" + case "$_ka_kind" in + deployment.apps|deployment) + np_scope_produces "k8s-deployment:$_ka_ns/$_ka_name" workload "$_ka_name" ;; + service) + np_scope_produces "k8s-service:$_ka_ns/$_ka_name" service "$_ka_name" ;; + ingress.networking.k8s.io|ingress) + np_scope_produces "k8s-ingress:$_ka_ns/$_ka_name" ingress "$_ka_name" ;; + esac + done <<< "$2" + return 0 +} + # A silent failure (no `log error` on the way down) must still be clear: the # ERR trap remembers the last failing top-level command, and the EXIT trap # reports it against the step that was current when the shell died. diff --git a/k8s/scope/build_context b/k8s/scope/build_context index 0650f897..4d5c0bf8 100755 --- a/k8s/scope/build_context +++ b/k8s/scope/build_context @@ -126,6 +126,9 @@ if ! kubectl get namespace "$K8S_NAMESPACE" &> /dev/null; then kubectl apply -f - log info " ✅ Namespace '$K8S_NAMESPACE' created successfully" + if command -v np_scope_produces >/dev/null 2>&1; then + np_scope_produces "k8s-namespace:$K8S_NAMESPACE" namespace "$K8S_NAMESPACE" + fi else log error "" log error "💡 Possible causes:" diff --git a/k8s/scope/networking/dns/manage_dns b/k8s/scope/networking/dns/manage_dns index 6d7538c3..f52e3fd0 100755 --- a/k8s/scope/networking/dns/manage_dns +++ b/k8s/scope/networking/dns/manage_dns @@ -70,3 +70,10 @@ case "$DNS_TYPE" in esac log info "✅ DNS records managed successfully" + +# The DNS record joins the lineage under its real-world address, so any other +# operation naming the same FQDN links to it. Only a CREATE writes the record. +if [ "${ACTION:-}" = "CREATE" ] && [ -n "${SCOPE_DOMAIN:-}" ] \ + && command -v np_scope_produces >/dev/null 2>&1; then + np_scope_produces "dns-record:$SCOPE_DOMAIN" dns_record "$SCOPE_DOMAIN" +fi diff --git a/k8s/scope/networking/wait_for_alb b/k8s/scope/networking/wait_for_alb index 48abf572..aaf07730 100644 --- a/k8s/scope/networking/wait_for_alb +++ b/k8s/scope/networking/wait_for_alb @@ -110,6 +110,12 @@ if [ "$state" != "active" ]; then exit 1 fi +# The scope depends on this ALB: record the lineage under its ARN — the same +# address the platform's own scopes name — before closing the wait sub-step. +if [ -n "$alb_arn" ] && command -v np_scope_consumes >/dev/null 2>&1; then + np_scope_consumes "load-balancer:$alb_arn" load_balancer "$alb_arn" +fi + # The wait phase is over and the ALB is active — close its sub-step; the audit # tagging below is not part of the wait. if command -v np_scope_step_end >/dev/null 2>&1; then diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index f2f16ffb..b10a91c4 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -243,3 +243,62 @@ hello" ] [ "$status" -eq 0 ] ! echo "$output" | grep -q 'nothing open' } + +# --- lineage ---------------------------------------------------------------- + +@test "produces records the edge and the pointer on the current step" { + run_logged 'np_scope_produces "dns-record:api.example.com" dns_record "api.example.com"' + [ "$status" -eq 0 ] + echo "$output" | grep '"edge.produces"' | grep -q '"id":"dns-record:api.example.com"' + echo "$output" | grep -q '"tracing.binding":{"kind":"pointer","name":"dns_record","uri":"api.example.com"}' + # foreign re-emit carries the io facet on the adopted step + echo "$output" | grep '"tracing.output"' | grep -q 'apply-manifests@0.0' +} + +@test "consumes records the cross-flow image edge" { + run_logged 'np_scope_consumes "docker-image:registry.example.com/app:1.2" image "registry.example.com/app:1.2"' + [ "$status" -eq 0 ] + echo "$output" | grep '"edge.consumes"' | grep -q '"id":"docker-image:registry.example.com/app:1.2"' +} + +@test "lineage inside an open sub-step attaches to the sub-step" { + run_logged ' + np_scope_step_begin wait-alb-active + np_scope_consumes "load-balancer:arn:aws:elb:demo" load_balancer "arn:aws:elb:demo" + np_scope_step_end 0 + ' + [ "$status" -eq 0 ] + echo "$output" | grep '"edge.consumes"' | grep -q 'wait-alb-active@0.0' +} + +@test "kubectl apply output becomes workload/service/ingress lineage" { + run_logged ' + np_scope_k8s_applied "ns-42" "deployment.apps/d-1-2 created +service/s-1-2 configured +ingress.networking.k8s.io/i-1-2 created +secret/sec-1 created" + ' + [ "$status" -eq 0 ] + echo "$output" | grep -q '"id":"k8s-deployment:ns-42/d-1-2"' + echo "$output" | grep -q '"id":"k8s-service:ns-42/s-1-2"' + echo "$output" | grep -q '"id":"k8s-ingress:ns-42/i-1-2"' + # kinds outside the platform lineage model are not datasets + ! echo "$output" | grep -q 'sec-1' +} + +@test "affordance and progress land as their core facets" { + run_logged ' + np_scope_affordance "{\"kind\":\"deploy-log\",\"application_id\":\"7\"}" + np_scope_progress 3 10 instances + ' + [ "$status" -eq 0 ] + echo "$output" | grep -q '"tracing.affordances":\[{"kind":"deploy-log","application_id":"7"}\]' + echo "$output" | grep -q '"tracing.progress":{"current":3,"target":10,"unit":"instances"}' +} + +@test "lineage helpers are defined no-ops when untraced" { + unset NP_TRACE + run "$BASH" -c "source '$LOGGING'; np_scope_produces d:1 n u; np_scope_consumes d:2; np_scope_affordance '{\"kind\":\"x\"}'; np_scope_progress 1 2; np_scope_k8s_applied ns 'deployment.apps/x created'; echo rc=\$?" + [ "$status" -eq 0 ] + [[ "$output" == *"rc=0"* ]] +} diff --git a/nptrace.sh b/nptrace.sh index 4e4a006f..5967b177 100755 --- a/nptrace.sh +++ b/nptrace.sh @@ -1269,6 +1269,188 @@ np__stage_timing() { return 0 } +# --------------------------------------------------------------------------- +# Lineage — produces/consumes edges with io pointers +# --------------------------------------------------------------------------- + +# A dataset ref for an edge endpoint. The id is the CANONICAL dataset id — the +# exact string a producer and a consumer must both name for lineage to join +# them by value (an ARN, an FQDN, `:` for an asset) — never a +# synthesised id. +np__dataset_ref() { + np__json_obj type dataset id "$1" +} + +# The shared body of produces/consumes. +# $1 handle, $2 edge type, $3 io facet namespace, $4 io side key (io_output / +# io_input), $5 dataset id, $6 pointer name, $7 pointer uri. +# +# With a name+uri the io is declared ONCE as a pointer descriptor: it +# accumulates into the node's tracing.input/tracing.output facet AND becomes +# the edge's tracing.binding — the same single-source rule as the sibling +# SDKs. Bare (no pointer) records lineage only. +# +# On a FOREIGN (adopted) node this is an observed fact, exactly like +# np_trace_error: the edge is ours to say, and the staged io facet reaches the +# wire through the foreign re-emit. +np__io_edge() { + _ie_h=$1 + _ie_type=$2 + _ie_facet=$3 + _ie_side=$4 + _ie_id=$5 + _ie_name=${6:-} + _ie_uri=${7:-} + + _ie_binding='' + if [ -n "$_ie_name" ] && [ -n "$_ie_uri" ]; then + _ie_binding=$(np__json_obj kind pointer name "$_ie_name" uri "$_ie_uri") + # Append to the side's descriptor list; the facet is re-staged whole each + # time (last write wins per namespace), so the array only ever grows. + _ie_list=$(np__node_get "$_ie_h" "$_ie_side") + if [ -n "$_ie_list" ]; then + _ie_list="$_ie_list,$_ie_binding" + else + _ie_list="$_ie_binding" + fi + np__node_set "$_ie_h" "$_ie_side" "$_ie_list" + np__stage_facet "$_ie_h" "$_ie_facet" "[$_ie_list]" + fi + + # An edge must not point FROM a node the read model has never seen. + np_trace_start "$_ie_h" + + if [ -n "$_ie_binding" ]; then + _ie_data=$(np__json_obj_raw \ + from "$(np__ref_of "$_ie_h")" \ + to "$(np__dataset_ref "$_ie_id")" \ + facets "{$(np__json_str "$NP_FACET_BINDING"):$_ie_binding}") + else + _ie_data=$(np__json_obj_raw \ + from "$(np__ref_of "$_ie_h")" \ + to "$(np__dataset_ref "$_ie_id")") + fi + np__spool "$_ie_type" "$(np__node_get "$_ie_h" nrn)" "$_ie_data" >/dev/null + np__flush_foreign "$_ie_h" + return 0 +} + +# np_trace_produces [handle] [--name --uri ] +# +# Declare this node WROTE the dataset. `--name`/`--uri` record the io as a +# pointer descriptor (the artifact's address) on both the node and the edge. +np_trace_produces() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _pr_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_pr_h" || { np__drop 'produces' 'no node in scope'; return 0; } + _pr_id=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + if [ -z "$_pr_id" ]; then + np__drop 'produces' 'dataset id is required' + return 0 + fi + _pr_name='' + _pr_uri='' + while [ "$#" -gt 0 ]; do + case "$1" in + --name) _pr_name=${2:-}; shift 2 ;; + --uri) _pr_uri=${2:-}; shift 2 ;; + *) shift ;; + esac + done + np__io_edge "$_pr_h" "$NP_TYPE_EDGE_PRODUCES" "$NP_FACET_OUTPUT" io_output \ + "$_pr_id" "$_pr_name" "$_pr_uri" + return 0 +} + +# np_trace_consumes [handle] [--name --uri ] +# +# Declare this node READ the dataset; see np_trace_produces. +np_trace_consumes() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _cn_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_cn_h" || { np__drop 'consumes' 'no node in scope'; return 0; } + _cn_id=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + if [ -z "$_cn_id" ]; then + np__drop 'consumes' 'dataset id is required' + return 0 + fi + _cn_name='' + _cn_uri='' + while [ "$#" -gt 0 ]; do + case "$1" in + --name) _cn_name=${2:-}; shift 2 ;; + --uri) _cn_uri=${2:-}; shift 2 ;; + *) shift ;; + esac + done + np__io_edge "$_cn_h" "$NP_TYPE_EDGE_CONSUMES" "$NP_FACET_INPUT" io_input \ + "$_cn_id" "$_cn_name" "$_cn_uri" + return 0 +} + +# np_trace_affordances [handle] +# +# What this node OFFERS a human to do — a declared fact the UI renders as a +# control (view live logs, switch traffic). One affordance object +# ('{"kind":"deploy-log",...}') or a bare array of them; the wire form is +# always the array. +np_trace_affordances() { + _af_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_af_h" || return 0 + _af_body=${1:-} + case "$_af_body" in + \[*) ;; + \{*) _af_body="[$_af_body]" ;; + *) np__drop 'affordances' 'body must be a JSON object or array'; return 0 ;; + esac + np__stage_facet "$_af_h" "$NP_FACET_AFFORDANCES" "$_af_body" + np__flush_foreign "$_af_h" + return 0 +} + +# np_trace_progress [handle] [unit] +# +# How far a CONVERGING phase has advanced toward its declared target — +# instances 3 of 10, traffic 40 of 100. Non-negative integers; the optional +# unit names what is counted ("percent", "instances"). +np_trace_progress() { + _pg_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_pg_h" || return 0 + _pg_current=${1:-} + _pg_target=${2:-} + _pg_unit=${3:-} + case "$_pg_current$_pg_target" in + '' | *[!0-9]*) np__drop 'progress' 'current and target must be non-negative integers'; return 0 ;; + esac + [ -n "$_pg_current" ] && [ -n "$_pg_target" ] || { + np__drop 'progress' 'current and target must be non-negative integers' + return 0 + } + np__stage_facet "$_pg_h" "$NP_FACET_PROGRESS" \ + "$(np__json_obj_raw current "$_pg_current" target "$_pg_target" \ + unit "$(if [ -n "$_pg_unit" ]; then np__json_str "$_pg_unit"; fi)")" + np__flush_foreign "$_pg_h" + return 0 +} + # --------------------------------------------------------------------------- # Lifecycle terminals # --------------------------------------------------------------------------- diff --git a/scheduled_task/logging b/scheduled_task/logging index 17d808b8..dc6fd3e0 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -201,6 +201,84 @@ np_scope_wait_heartbeat() { return 0 } +# np_scope_produces [ ] +# np_scope_consumes [ ] +# +# Data lineage: declare what the current step (or open sub-step) WROTE or +# READ, by CANONICAL dataset id — the exact grammar the scope-workflow-manager +# uses, so a custom scope's lineage joins the platform's by value: +# +# k8s-namespace: k8s-deployment:/ +# k8s-service:/ k8s-ingress:/ +# dns-record: load-balancer: +# docker-image: deployment-log:// +# +# Always defined; a no-op when the workflow is untraced. +np_scope_produces() { + command -v np_trace_produces >/dev/null 2>&1 || return 0 + local _pd_node + _pd_node=$(_np_scopes_node) || return 0 + np_trace_produces "$_pd_node" "$1" ${2:+--name "$2"} ${3:+--uri "$3"} + return 0 +} + +np_scope_consumes() { + command -v np_trace_consumes >/dev/null 2>&1 || return 0 + local _cd_node + _cd_node=$(_np_scopes_node) || return 0 + np_trace_consumes "$_cd_node" "$1" ${2:+--name "$2"} ${3:+--uri "$3"} + return 0 +} + +# np_scope_affordance +# +# What the current step OFFERS a human to do — the UI renders it as a control +# (e.g. '{"kind":"deploy-log","application_id":"7","scope_id":"42",...}'). +np_scope_affordance() { + command -v np_trace_affordances >/dev/null 2>&1 || return 0 + local _ad_node + _ad_node=$(_np_scopes_node) || return 0 + np_trace_affordances "$_ad_node" "$1" + return 0 +} + +# np_scope_progress [unit] +# +# A converging phase's advance toward its target (instances 3 of 10). +np_scope_progress() { + command -v np_trace_progress >/dev/null 2>&1 || return 0 + local _pg_node + _pg_node=$(_np_scopes_node) || return 0 + np_trace_progress "$_pg_node" "$1" "$2" "${3:-}" + return 0 +} + +# np_scope_k8s_applied +# +# Turn `kubectl apply` output lines (`deployment.apps/name created`) into +# lineage on the current step, for the resource kinds the platform's lineage +# model knows. One chokepoint (apply_templates) covers every workflow. +np_scope_k8s_applied() { + command -v np_trace_produces >/dev/null 2>&1 || return 0 + local _ka_ns="$1" _ka_line _ka_kind _ka_name + [ -n "$_ka_ns" ] || return 0 + while IFS= read -r _ka_line; do + [ -n "$_ka_line" ] || continue + _ka_kind="${_ka_line%%/*}" + _ka_name="${_ka_line#*/}" + _ka_name="${_ka_name%% *}" + case "$_ka_kind" in + deployment.apps|deployment) + np_scope_produces "k8s-deployment:$_ka_ns/$_ka_name" workload "$_ka_name" ;; + service) + np_scope_produces "k8s-service:$_ka_ns/$_ka_name" service "$_ka_name" ;; + ingress.networking.k8s.io|ingress) + np_scope_produces "k8s-ingress:$_ka_ns/$_ka_name" ingress "$_ka_name" ;; + esac + done <<< "$2" + return 0 +} + # A silent failure (no `log error` on the way down) must still be clear: the # ERR trap remembers the last failing top-level command, and the EXIT trap # reports it against the step that was current when the shell died. From 5f8ed972c18a2edd3207809b5e7cfd33335eab71 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 10:21:33 -0300 Subject: [PATCH 04/52] chore(tracing): re-vendor nptrace.sh with the lint fix from catalog-tracing-sh#6 --- nptrace.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nptrace.sh b/nptrace.sh index 5967b177..d70cd373 100755 --- a/nptrace.sh +++ b/nptrace.sh @@ -1437,13 +1437,13 @@ np_trace_progress() { _pg_current=${1:-} _pg_target=${2:-} _pg_unit=${3:-} - case "$_pg_current$_pg_target" in - '' | *[!0-9]*) np__drop 'progress' 'current and target must be non-negative integers'; return 0 ;; - esac - [ -n "$_pg_current" ] && [ -n "$_pg_target" ] || { + if [ -z "$_pg_current" ] || [ -z "$_pg_target" ]; then np__drop 'progress' 'current and target must be non-negative integers' return 0 - } + fi + case "$_pg_current$_pg_target" in + *[!0-9]*) np__drop 'progress' 'current and target must be non-negative integers'; return 0 ;; + esac np__stage_facet "$_pg_h" "$NP_FACET_PROGRESS" \ "$(np__json_obj_raw current "$_pg_current" target "$_pg_target" \ unit "$(if [ -n "$_pg_unit" ]; then np__json_str "$_pg_unit"; fi)")" From ddbbb538a0207741e436336095d546c999f2c46c Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 10:30:20 -0300 Subject: [PATCH 05/52] chore(tracing): re-vendor nptrace.sh after the readability refactor in catalog-tracing-sh#6 --- nptrace.sh | 175 +++++++++++++++++++++++++++-------------------------- 1 file changed, 88 insertions(+), 87 deletions(-) diff --git a/nptrace.sh b/nptrace.sh index d70cd373..23319e7b 100755 --- a/nptrace.sh +++ b/nptrace.sh @@ -1281,90 +1281,115 @@ np__dataset_ref() { np__json_obj type dataset id "$1" } -# The shared body of produces/consumes. -# $1 handle, $2 edge type, $3 io facet namespace, $4 io side key (io_output / -# io_input), $5 dataset id, $6 pointer name, $7 pointer uri. +# np__emit_io_edge [pointer-name] [pointer-uri] # -# With a name+uri the io is declared ONCE as a pointer descriptor: it -# accumulates into the node's tracing.input/tracing.output facet AND becomes -# the edge's tracing.binding — the same single-source rule as the sibling -# SDKs. Bare (no pointer) records lineage only. +# Emit one lineage edge. The direction decides everything else: `out` is +# edge.produces + tracing.output, `in` is edge.consumes + tracing.input. +# +# With a pointer (name + uri) the io is declared ONCE: the descriptor +# accumulates into the node's io facet AND becomes the edge's tracing.binding +# — the same single-source rule as the sibling SDKs. Without one, the edge +# records lineage only. # # On a FOREIGN (adopted) node this is an observed fact, exactly like # np_trace_error: the edge is ours to say, and the staged io facet reaches the # wire through the foreign re-emit. -np__io_edge() { - _ie_h=$1 - _ie_type=$2 - _ie_facet=$3 - _ie_side=$4 - _ie_id=$5 - _ie_name=${6:-} - _ie_uri=${7:-} - - _ie_binding='' - if [ -n "$_ie_name" ] && [ -n "$_ie_uri" ]; then - _ie_binding=$(np__json_obj kind pointer name "$_ie_name" uri "$_ie_uri") - # Append to the side's descriptor list; the facet is re-staged whole each - # time (last write wins per namespace), so the array only ever grows. - _ie_list=$(np__node_get "$_ie_h" "$_ie_side") - if [ -n "$_ie_list" ]; then - _ie_list="$_ie_list,$_ie_binding" +np__emit_io_edge() { + _io_handle=$1 + _io_direction=$2 + _io_dataset_id=$3 + _io_pointer_name=${4:-} + _io_pointer_uri=${5:-} + + if [ "$_io_direction" = 'out' ]; then + _io_edge_type=$NP_TYPE_EDGE_PRODUCES + _io_facet_namespace=$NP_FACET_OUTPUT + _io_descriptor_store=io_output + else + _io_edge_type=$NP_TYPE_EDGE_CONSUMES + _io_facet_namespace=$NP_FACET_INPUT + _io_descriptor_store=io_input + fi + + _io_pointer='' + if [ -n "$_io_pointer_name" ] && [ -n "$_io_pointer_uri" ]; then + _io_pointer=$(np__json_obj kind pointer name "$_io_pointer_name" uri "$_io_pointer_uri") + # Append to the direction's descriptor list; the facet is re-staged whole + # each time (last write wins per namespace), so the array only ever grows. + _io_descriptors=$(np__node_get "$_io_handle" "$_io_descriptor_store") + if [ -n "$_io_descriptors" ]; then + _io_descriptors="$_io_descriptors,$_io_pointer" else - _ie_list="$_ie_binding" + _io_descriptors=$_io_pointer fi - np__node_set "$_ie_h" "$_ie_side" "$_ie_list" - np__stage_facet "$_ie_h" "$_ie_facet" "[$_ie_list]" + np__node_set "$_io_handle" "$_io_descriptor_store" "$_io_descriptors" + np__stage_facet "$_io_handle" "$_io_facet_namespace" "[$_io_descriptors]" fi # An edge must not point FROM a node the read model has never seen. - np_trace_start "$_ie_h" + np_trace_start "$_io_handle" - if [ -n "$_ie_binding" ]; then - _ie_data=$(np__json_obj_raw \ - from "$(np__ref_of "$_ie_h")" \ - to "$(np__dataset_ref "$_ie_id")" \ - facets "{$(np__json_str "$NP_FACET_BINDING"):$_ie_binding}") + if [ -n "$_io_pointer" ]; then + _io_edge_data=$(np__json_obj_raw \ + from "$(np__ref_of "$_io_handle")" \ + to "$(np__dataset_ref "$_io_dataset_id")" \ + facets "{$(np__json_str "$NP_FACET_BINDING"):$_io_pointer}") else - _ie_data=$(np__json_obj_raw \ - from "$(np__ref_of "$_ie_h")" \ - to "$(np__dataset_ref "$_ie_id")") + _io_edge_data=$(np__json_obj_raw \ + from "$(np__ref_of "$_io_handle")" \ + to "$(np__dataset_ref "$_io_dataset_id")") fi - np__spool "$_ie_type" "$(np__node_get "$_ie_h" nrn)" "$_ie_data" >/dev/null - np__flush_foreign "$_ie_h" + np__spool "$_io_edge_type" "$(np__node_get "$_io_handle" nrn)" "$_io_edge_data" >/dev/null + np__flush_foreign "$_io_handle" return 0 } -# np_trace_produces [handle] [--name --uri ] -# -# Declare this node WROTE the dataset. `--name`/`--uri` record the io as a -# pointer descriptor (the artifact's address) on both the node and the edge. -np_trace_produces() { +# The shared argv handling of np_trace_produces / np_trace_consumes: +# resolve the optional leading handle, take the dataset id, parse the +# pointer flags, and hand off to np__emit_io_edge. +# $1 direction (out|in), $2 verb name for drop records, then the caller's argv. +np__lineage_verb() { [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _pr_h=$(np__resolve_handle "${1:-}") + _lv_direction=$1 + _lv_verb=$2 + shift 2 + + _lv_handle=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_pr_h" || { np__drop 'produces' 'no node in scope'; return 0; } - _pr_id=${1:-} + np__is_handle "$_lv_handle" || { np__drop "$_lv_verb" 'no node in scope'; return 0; } + + _lv_dataset_id=${1:-} if [ "$#" -gt 0 ]; then shift fi - if [ -z "$_pr_id" ]; then - np__drop 'produces' 'dataset id is required' + if [ -z "$_lv_dataset_id" ]; then + np__drop "$_lv_verb" 'dataset id is required' return 0 fi - _pr_name='' - _pr_uri='' + + _lv_pointer_name='' + _lv_pointer_uri='' while [ "$#" -gt 0 ]; do case "$1" in - --name) _pr_name=${2:-}; shift 2 ;; - --uri) _pr_uri=${2:-}; shift 2 ;; + --name) _lv_pointer_name=${2:-}; shift 2 ;; + --uri) _lv_pointer_uri=${2:-}; shift 2 ;; *) shift ;; esac done - np__io_edge "$_pr_h" "$NP_TYPE_EDGE_PRODUCES" "$NP_FACET_OUTPUT" io_output \ - "$_pr_id" "$_pr_name" "$_pr_uri" + + np__emit_io_edge "$_lv_handle" "$_lv_direction" "$_lv_dataset_id" \ + "$_lv_pointer_name" "$_lv_pointer_uri" + return 0 +} + +# np_trace_produces [handle] [--name --uri ] +# +# Declare this node WROTE the dataset. `--name`/`--uri` record the io as a +# pointer descriptor (the artifact's address) on both the node and the edge. +np_trace_produces() { + np__lineage_verb out produces "$@" return 0 } @@ -1372,31 +1397,7 @@ np_trace_produces() { # # Declare this node READ the dataset; see np_trace_produces. np_trace_consumes() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _cn_h=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_cn_h" || { np__drop 'consumes' 'no node in scope'; return 0; } - _cn_id=${1:-} - if [ "$#" -gt 0 ]; then - shift - fi - if [ -z "$_cn_id" ]; then - np__drop 'consumes' 'dataset id is required' - return 0 - fi - _cn_name='' - _cn_uri='' - while [ "$#" -gt 0 ]; do - case "$1" in - --name) _cn_name=${2:-}; shift 2 ;; - --uri) _cn_uri=${2:-}; shift 2 ;; - *) shift ;; - esac - done - np__io_edge "$_cn_h" "$NP_TYPE_EDGE_CONSUMES" "$NP_FACET_INPUT" io_input \ - "$_cn_id" "$_cn_name" "$_cn_uri" + np__lineage_verb in consumes "$@" return 0 } @@ -1407,19 +1408,19 @@ np_trace_consumes() { # ('{"kind":"deploy-log",...}') or a bare array of them; the wire form is # always the array. np_trace_affordances() { - _af_h=$(np__resolve_handle "${1:-}") + _af_handle=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_af_h" || return 0 + np__is_handle "$_af_handle" || return 0 _af_body=${1:-} case "$_af_body" in \[*) ;; \{*) _af_body="[$_af_body]" ;; *) np__drop 'affordances' 'body must be a JSON object or array'; return 0 ;; esac - np__stage_facet "$_af_h" "$NP_FACET_AFFORDANCES" "$_af_body" - np__flush_foreign "$_af_h" + np__stage_facet "$_af_handle" "$NP_FACET_AFFORDANCES" "$_af_body" + np__flush_foreign "$_af_handle" return 0 } @@ -1429,11 +1430,11 @@ np_trace_affordances() { # instances 3 of 10, traffic 40 of 100. Non-negative integers; the optional # unit names what is counted ("percent", "instances"). np_trace_progress() { - _pg_h=$(np__resolve_handle "${1:-}") + _pg_handle=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_pg_h" || return 0 + np__is_handle "$_pg_handle" || return 0 _pg_current=${1:-} _pg_target=${2:-} _pg_unit=${3:-} @@ -1444,10 +1445,10 @@ np_trace_progress() { case "$_pg_current$_pg_target" in *[!0-9]*) np__drop 'progress' 'current and target must be non-negative integers'; return 0 ;; esac - np__stage_facet "$_pg_h" "$NP_FACET_PROGRESS" \ + np__stage_facet "$_pg_handle" "$NP_FACET_PROGRESS" \ "$(np__json_obj_raw current "$_pg_current" target "$_pg_target" \ unit "$(if [ -n "$_pg_unit" ]; then np__json_str "$_pg_unit"; fi)")" - np__flush_foreign "$_pg_h" + np__flush_foreign "$_pg_handle" return 0 } From 9f71a9cb2b14ecec64f61d8460d89e6d428e8938 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 11:07:34 -0300 Subject: [PATCH 06/52] =?UTF-8?q?feat(k8s):=20speak=20the=20platform's=20s?= =?UTF-8?q?tep=20vocabulary=20=E2=80=94=20SWM-shaped=20plans=20for=20custo?= =?UTF-8?q?m=20scopes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workflows declare each step's trace identity (np CLI trace: blocks) using the exact step keys, titles and plan groups the platform's own scopes emit — create-namespace, create-ingress, wait-for-ingress, create-dns for the scope provision; wait-for-instances (group waiting-instances, where the dashboard attaches the instance pips) and traffic-switch (group switching-traffic) for deployments. Step NAMES are untouched: customer overrides anchor on them. The actual create-* deploy steps are emitted per applied manifest by apply_templates as keyed sub-steps (deployment-*.yaml → create-deployment, secret-* → create-secret, scaling-* → create-hpa, service-* → create-service, pdb-* → create-pod-disruption-budget, ingress-* → configure-ingress/create-ingress) — the custom flow renders then bulk-applies, so this is where those resources really get created. Plumbing (load logging, assume role, renders, context builds) is trace: false; route53-only stations carry flavors: [route53] so an AKS or external_dns scope never declares a station it cannot reach; and scripts whose branch isn't taken declare np_step_skip so their step closes skipped, never a hollow completed. Requires nullplatform/cli#216 for the trace: blocks and np_step_skip; both degrade to exactly today's behavior on an older CLI (unknown YAML keys are ignored; np_step_skip is guarded). --- k8s/apply_templates | 25 +++++++++++++++++ k8s/deployment/verify_ingress_reconciliation | 3 ++ k8s/deployment/workflows/initial.yaml | 28 +++++++++++++++++++ k8s/deployment/workflows/switch_traffic.yaml | 29 ++++++++++++++++++++ k8s/scope/build_context | 5 ++++ k8s/scope/networking/wait_for_alb | 5 ++++ k8s/scope/wait_on_balancer | 6 ++++ k8s/scope/workflows/create.yaml | 27 ++++++++++++++++++ 8 files changed, 128 insertions(+) diff --git a/k8s/apply_templates b/k8s/apply_templates index 384f7890..5ddbcdff 100644 --- a/k8s/apply_templates +++ b/k8s/apply_templates @@ -31,6 +31,28 @@ while IFS= read -r TEMPLATE_FILE; do IGNORE_NOT_FOUND="--ignore-not-found=true" fi + # Each manifest apply is its own SUB-STEP, keyed by the platform's step + # vocabulary (create-deployment, create-service, ...) so a custom + # deploy's trace reads like a native one — the manifest filename prefix + # names what is being created. + TRACE_STEP_KEY="" + if [[ "$ACTION" == "apply" ]] && command -v np_scope_step_begin >/dev/null 2>&1; then + case "$FILENAME" in + deployment-*) TRACE_STEP_KEY="create-deployment" ;; + secret-*) TRACE_STEP_KEY="create-secret" ;; + scaling-*) TRACE_STEP_KEY="create-hpa" ;; + service-*) TRACE_STEP_KEY="create-service" ;; + pdb-*) TRACE_STEP_KEY="create-pod-disruption-budget" ;; + ingress-*) + if [[ -n "${DEPLOYMENT_ID:-}" ]]; then + TRACE_STEP_KEY="configure-ingress" + else + TRACE_STEP_KEY="create-ingress" + fi ;; + esac + [[ -n "$TRACE_STEP_KEY" ]] && np_scope_step_begin "$TRACE_STEP_KEY" + fi + # Captured (and re-echoed) so applied resources can be recorded as # lineage on the trace; the console output is unchanged. if KUBECTL_OUT=$(kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND); then @@ -38,10 +60,13 @@ while IFS= read -r TEMPLATE_FILE; do if [[ "$ACTION" == "apply" ]] && command -v np_scope_k8s_applied >/dev/null 2>&1; then np_scope_k8s_applied "${K8S_NAMESPACE:-}" "$KUBECTL_OUT" fi + [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 0 else [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" log error " ❌ Failed to apply" + [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 1 fi + TRACE_STEP_KEY="" fi DEST_DIR="${BASE_DIR}/$ACTION" diff --git a/k8s/deployment/verify_ingress_reconciliation b/k8s/deployment/verify_ingress_reconciliation index a75f47dc..a593dd31 100644 --- a/k8s/deployment/verify_ingress_reconciliation +++ b/k8s/deployment/verify_ingress_reconciliation @@ -18,6 +18,9 @@ DEPLOYMENT_STRATEGY=$(echo "$CONTEXT" | jq -r ".deployment.strategy") if [ "$ALB_RECONCILIATION_ENABLED" = "false" ] && [ "$DEPLOYMENT_STRATEGY" = "blue_green" ]; then log warn "⚠️ Skipping ALB verification (ALB access needed for blue-green traffic validation)" + if command -v np_step_skip >/dev/null 2>&1; then + np_step_skip "ALB verification disabled for blue-green" + fi return 0 fi diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index 92a0bb40..0b919fbd 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -2,10 +2,19 @@ include: - "$SERVICE_PATH/values.yaml" configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" +# Trace identities speak the platform's deploy step vocabulary (the actual +# create-* steps are emitted per applied manifest by apply_templates, under +# the apply step). Step NAMES never change — overrides anchor on them. +trace: + flavor: "$DNS_TYPE" + groups: + - {key: setting-up, title: Setting up} + - {key: waiting-instances, title: Waiting for instances to be healthy} steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function @@ -15,6 +24,7 @@ steps: - name: assume role type: script file: "$SERVICE_PATH/utils/assume_role_step" + trace: false output: - name: AWS_ACCESS_KEY_ID type: environment @@ -25,6 +35,7 @@ steps: - name: build context type: script file: "$SERVICE_PATH/deployment/build_context" + trace: false output: - name: CONTEXT type: environment @@ -41,9 +52,13 @@ steps: - name: validate alb target group capacity type: script file: "$SERVICE_PATH/deployment/validate_alb_target_group_capacity" + trace: + group: setting-up + flavors: [route53] - name: route traffic type: script file: "$SERVICE_PATH/deployment/networking/gateway/route_traffic" + trace: false configuration: TEMPLATE: "$INGRESS_TEMPLATE" output: @@ -53,6 +68,7 @@ steps: - name: create deployment type: script file: "$SERVICE_PATH/deployment/build_deployment" + trace: false output: - name: DEPLOYMENT_PATH type: file @@ -69,6 +85,9 @@ steps: - name: apply type: script file: "$SERVICE_PATH/apply_templates" + trace: + title: Apply manifests + group: setting-up configuration: ACTION: apply DRY_RUN: false @@ -79,17 +98,26 @@ steps: - name: notify_active_domains type: script file: "$SERVICE_PATH/deployment/notify_active_domains" + trace: false - name: verify_networking_reconciliation type: script file: "$SERVICE_PATH/deployment/verify_networking_reconciliation" + trace: + group: setting-up configuration: VERIFY_WEIGHTS: false - name: publish_alb_metrics type: script file: "$SERVICE_PATH/deployment/publish_alb_metrics" + trace: + flavors: [route53] - name: wait deployment active type: script file: "$SERVICE_PATH/deployment/wait_deployment_active" + trace: + key: wait-for-instances + title: Instance health check + group: waiting-instances configuration: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index ce9a9a67..014bb4d1 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -2,10 +2,19 @@ include: - "$SERVICE_PATH/values.yaml" configuration: INGRESS_TEMPLATE: "$BLUE_GREEN_INGRESS_PATH" +# Trace identities speak the platform's deploy step vocabulary. Step NAMES +# never change — overrides anchor on them. +trace: + flavor: "$DNS_TYPE" + groups: + - {key: setting-up, title: Setting up} + - {key: waiting-instances, title: Waiting for instances to be healthy} + - {key: switching-traffic, title: Switching traffic} steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function @@ -15,6 +24,7 @@ steps: - name: assume role type: script file: "$SERVICE_PATH/utils/assume_role_step" + trace: false output: - name: AWS_ACCESS_KEY_ID type: environment @@ -25,6 +35,7 @@ steps: - name: build context type: script file: "$SERVICE_PATH/deployment/build_context" + trace: false output: - name: CONTEXT type: environment @@ -41,15 +52,23 @@ steps: - name: create deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" + trace: + title: Scale green deployment + group: setting-up post: name: wait deployment active type: script file: "$SERVICE_PATH/deployment/wait_deployment_active" + trace: + key: wait-for-instances + title: Instance health check + group: waiting-instances configuration: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS - name: route traffic type: script file: "$SERVICE_PATH/deployment/networking/gateway/route_traffic" + trace: false configuration: type: ingress TEMPLATE: "$INGRESS_TEMPLATE" @@ -60,9 +79,15 @@ steps: - name: update blue deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" + trace: + group: setting-up - name: apply traffic type: script file: "$SERVICE_PATH/apply_templates" + trace: + key: traffic-switch + title: Switch traffic + group: switching-traffic configuration: ACTION: apply DRY_RUN: false @@ -73,8 +98,12 @@ steps: - name: verify_networking_reconciliation type: script file: "$SERVICE_PATH/deployment/verify_networking_reconciliation" + trace: + group: switching-traffic configuration: VERIFY_WEIGHTS: true - name: publish_alb_metrics type: script file: "$SERVICE_PATH/deployment/publish_alb_metrics" + trace: + flavors: [route53] diff --git a/k8s/scope/build_context b/k8s/scope/build_context index 4d5c0bf8..4d3cffef 100755 --- a/k8s/scope/build_context +++ b/k8s/scope/build_context @@ -142,6 +142,11 @@ if ! kubectl get namespace "$K8S_NAMESPACE" &> /dev/null; then fi else log info " ✅ Namespace '$K8S_NAMESPACE' exists" + # This step's trace identity is create-namespace: when the namespace + # already exists, that branch wasn't taken — close as `skipped`. + if command -v np_step_skip >/dev/null 2>&1; then + np_step_skip "namespace '$K8S_NAMESPACE' already exists" + fi fi USE_ACCOUNT_SLUG=$(get_config_value \ diff --git a/k8s/scope/networking/wait_for_alb b/k8s/scope/networking/wait_for_alb index aaf07730..3740cbba 100644 --- a/k8s/scope/networking/wait_for_alb +++ b/k8s/scope/networking/wait_for_alb @@ -24,6 +24,11 @@ # time out and fail the scope creation. if [ "${DNS_TYPE:-}" != "route53" ]; then log debug "📋 DNS type is '${DNS_TYPE:-unset}', skipping ALB active-state wait" + # Tell the platform this step's branch wasn't taken: it closes as + # `skipped`, not `completed`. (Guarded: defined by the np CLI's preamble.) + if command -v np_step_skip >/dev/null 2>&1; then + np_step_skip "no ALB on DNS type '${DNS_TYPE:-unset}'" + fi return 0 2>/dev/null || exit 0 fi diff --git a/k8s/scope/wait_on_balancer b/k8s/scope/wait_on_balancer index 72674d8e..2e45db78 100644 --- a/k8s/scope/wait_on_balancer +++ b/k8s/scope/wait_on_balancer @@ -72,9 +72,15 @@ case "$DNS_TYPE" in route53|azure) log debug "📋 DNS Type $DNS_TYPE - DNS should already be configured" log debug "📋 Skipping DNS wait check" + if command -v np_step_skip >/dev/null 2>&1; then + np_step_skip "DNS already configured for type '$DNS_TYPE'" + fi ;; *) log debug "📋 Unknown DNS type: $DNS_TYPE" log debug "📋 Skipping DNS wait check" + if command -v np_step_skip >/dev/null 2>&1; then + np_step_skip "no DNS wait for type '$DNS_TYPE'" + fi ;; esac diff --git a/k8s/scope/workflows/create.yaml b/k8s/scope/workflows/create.yaml index 69afe2c8..9d7cd19a 100644 --- a/k8s/scope/workflows/create.yaml +++ b/k8s/scope/workflows/create.yaml @@ -1,9 +1,16 @@ include: - "$SERVICE_PATH/values.yaml" +# The trace identity of each step (trace: blocks below) speaks the same step +# vocabulary the platform's own scopes emit, so a custom scope's provision +# reads identically in the timeline. Step NAMES never change — overrides +# anchor on them. +trace: + flavor: "$DNS_TYPE" steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function @@ -13,6 +20,7 @@ steps: - name: assume role type: script file: "$SERVICE_PATH/utils/assume_role_step" + trace: false output: - name: AWS_ACCESS_KEY_ID type: environment @@ -23,6 +31,10 @@ steps: - name: build context type: script file: "$SERVICE_PATH/scope/build_context" + trace: + key: create-namespace + title: Create namespace + optional: true output: - name: CONTEXT type: environment @@ -39,6 +51,9 @@ steps: - name: apply autocreated ingress type: script file: "$SERVICE_PATH/apply_templates" + trace: + key: create-ingress + title: Create ingress configuration: ACTION: apply DRY_RUN: false @@ -46,9 +61,15 @@ steps: name: wait for alb type: script file: "$SERVICE_PATH/scope/networking/wait_for_alb" + trace: + key: wait-for-ingress + title: Wait for ingress + flavors: [route53] - name: validate alb capacity type: script file: "$SERVICE_PATH/scope/validate_alb_capacity" + trace: + flavors: [route53] - name: iam type: workflow steps: @@ -82,12 +103,16 @@ steps: - name: create dns type: script file: "$SERVICE_PATH/scope/networking/dns/manage_dns" + trace: + key: create-dns + title: Create DNS configuration: ACTION: CREATE pre: name: build dns context type: script file: "$SERVICE_PATH/scope/networking/dns/build_dns_context" + trace: false output: - name: HOSTED_PUBLIC_ZONE_ID type: environment @@ -106,3 +131,5 @@ steps: - name: wait on balancer type: script file: "$SERVICE_PATH/scope/wait_on_balancer" + trace: + flavors: [external_dns] From 39d242915de7a3f71929878fcec37e0c91715a3b Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 11:47:10 -0300 Subject: [PATCH 07/52] fix(k8s): close the networking verification as skipped on flavors with nothing to verify --- k8s/deployment/verify_networking_reconciliation | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/k8s/deployment/verify_networking_reconciliation b/k8s/deployment/verify_networking_reconciliation index 506e57f5..c01dd16b 100644 --- a/k8s/deployment/verify_networking_reconciliation +++ b/k8s/deployment/verify_networking_reconciliation @@ -12,5 +12,10 @@ case "$DNS_TYPE" in ;; *) log warn "⚠️ Ingress reconciliation not available for DNS type: $DNS_TYPE, skipping" + # On azure/ARO there is nothing to verify: close the step as `skipped`, + # not a hollow `completed`. (Guarded: defined by the np CLI's preamble.) + if command -v np_step_skip >/dev/null 2>&1; then + np_step_skip "no networking verification for DNS type '$DNS_TYPE'" + fi ;; esac From 9c3610ec2e8138f5ad7f86504a394f6c7cd28a42 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 12:26:51 -0300 Subject: [PATCH 08/52] feat(k8s): declare the cluster flavor separately from the DNS type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit K8S_FLAVOR names the cluster variant (eks in the base values, aks and aro in the overlays) — an independent dimension from DNS_TYPE, so the workflows declare both as trace flavor tokens (flavors: ["$K8S_FLAVOR", "$DNS_TYPE"]) and each step gates on the dimension it genuinely depends on. --- azure-aro/values.yaml | 1 + azure/values.yaml | 1 + k8s/deployment/workflows/initial.yaml | 2 +- k8s/deployment/workflows/switch_traffic.yaml | 2 +- k8s/scope/workflows/create.yaml | 2 +- k8s/values.yaml | 1 + 6 files changed, 6 insertions(+), 3 deletions(-) diff --git a/azure-aro/values.yaml b/azure-aro/values.yaml index 7ff585b6..84d62b20 100644 --- a/azure-aro/values.yaml +++ b/azure-aro/values.yaml @@ -1,5 +1,6 @@ configuration: DNS_TYPE: azure + K8S_FLAVOR: aro USE_ACCOUNT_SLUG: false IMAGE_PULL_SECRETS: ENABLED: false diff --git a/azure/values.yaml b/azure/values.yaml index 0c3d54b9..52a58c4c 100644 --- a/azure/values.yaml +++ b/azure/values.yaml @@ -1,5 +1,6 @@ configuration: DNS_TYPE: azure + K8S_FLAVOR: aks USE_ACCOUNT_SLUG: false IMAGE_PULL_SECRETS: ENABLED: false diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index 0b919fbd..ea11f95e 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -6,7 +6,7 @@ configuration: # create-* steps are emitted per applied manifest by apply_templates, under # the apply step). Step NAMES never change — overrides anchor on them. trace: - flavor: "$DNS_TYPE" + flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] groups: - {key: setting-up, title: Setting up} - {key: waiting-instances, title: Waiting for instances to be healthy} diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index 014bb4d1..7b44f6fe 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -5,7 +5,7 @@ configuration: # Trace identities speak the platform's deploy step vocabulary. Step NAMES # never change — overrides anchor on them. trace: - flavor: "$DNS_TYPE" + flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] groups: - {key: setting-up, title: Setting up} - {key: waiting-instances, title: Waiting for instances to be healthy} diff --git a/k8s/scope/workflows/create.yaml b/k8s/scope/workflows/create.yaml index 9d7cd19a..24e27c84 100644 --- a/k8s/scope/workflows/create.yaml +++ b/k8s/scope/workflows/create.yaml @@ -5,7 +5,7 @@ include: # reads identically in the timeline. Step NAMES never change — overrides # anchor on them. trace: - flavor: "$DNS_TYPE" + flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] steps: - name: load logging type: script diff --git a/k8s/values.yaml b/k8s/values.yaml index edf18f08..4f43e079 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -10,6 +10,7 @@ configuration: # PRIVATE_DOMAIN: nullapps.io USE_ACCOUNT_SLUG: false DNS_TYPE: route53 # Available values route53 | azure | external_dns + K8S_FLAVOR: eks # The cluster variant (eks | aks | aro | gke) — independent of DNS_TYPE ALB_RECONCILIATION_ENABLED: false ALB_MAX_CAPACITY: 75 # 100 is the max target groups for ALB. Keeps 2 free for emergencies From e49592cd500272f7aa37f9cf4aeffdc5ba6e1739 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 12:38:46 -0300 Subject: [PATCH 09/52] =?UTF-8?q?fix(k8s):=20a=20station=20azure=20can=20n?= =?UTF-8?q?ever=20verify=20is=20not=20declared=20there=20=E2=80=94=20not?= =?UTF-8?q?=20even=20as=20skipped?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trace is for users: verify_networking_reconciliation has nothing to dispatch on azure/aro, so it is flavor-gated out of those plans entirely, like the ALB stations. skipped stays reserved for declared steps whose branch genuinely may or may not run on that flavor (a namespace that already exists). --- k8s/deployment/workflows/initial.yaml | 1 + k8s/deployment/workflows/switch_traffic.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index ea11f95e..40333da4 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -104,6 +104,7 @@ steps: file: "$SERVICE_PATH/deployment/verify_networking_reconciliation" trace: group: setting-up + flavors: [route53, external_dns] configuration: VERIFY_WEIGHTS: false - name: publish_alb_metrics diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index 7b44f6fe..b87a5259 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -100,6 +100,7 @@ steps: file: "$SERVICE_PATH/deployment/verify_networking_reconciliation" trace: group: switching-traffic + flavors: [route53, external_dns] configuration: VERIFY_WEIGHTS: true - name: publish_alb_metrics From 54bb8ca44248adf84a340df4efb6e7d7b2a3c52f Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 12:56:39 -0300 Subject: [PATCH 10/52] feat(k8s): declare job definitions so plans preview before any run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit k8s-scope-create, k8s-deployment-initial and k8s-deployment-switch-traffic carry their plans as previewable job definitions (version derived from the plan hash per flavor), and each run links instance_of — the same plan_source: job model the platform's own scopes use. --- k8s/deployment/workflows/initial.yaml | 1 + k8s/deployment/workflows/switch_traffic.yaml | 1 + k8s/scope/workflows/create.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index 40333da4..49961d36 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -7,6 +7,7 @@ configuration: # the apply step). Step NAMES never change — overrides anchor on them. trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + job: k8s-deployment-initial groups: - {key: setting-up, title: Setting up} - {key: waiting-instances, title: Waiting for instances to be healthy} diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index b87a5259..a2afec2c 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -6,6 +6,7 @@ configuration: # never change — overrides anchor on them. trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + job: k8s-deployment-switch-traffic groups: - {key: setting-up, title: Setting up} - {key: waiting-instances, title: Waiting for instances to be healthy} diff --git a/k8s/scope/workflows/create.yaml b/k8s/scope/workflows/create.yaml index 24e27c84..422e6034 100644 --- a/k8s/scope/workflows/create.yaml +++ b/k8s/scope/workflows/create.yaml @@ -6,6 +6,7 @@ include: # anchor on them. trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + job: k8s-scope-create steps: - name: load logging type: script From 755d05172445e8102b0c720f4a65ccea153c4afe Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 13:10:38 -0300 Subject: [PATCH 11/52] =?UTF-8?q?feat(k8s):=20full=20SWM=20narrative=20?= =?UTF-8?q?=E2=80=94=20counted=20io,=20health=20meter,=20severity=20explai?= =?UTF-8?q?ns,=20traffic=20set?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wait_deployment_active now reports everything the platform's own wait-for-instances does, from the data the script genuinely observes: the 'instances' counted io (healthy/launched/desired, plus per-pod restart history from the pods' containerStatuses), the instances-health affordance meter with unhealthy counts and probe reasons, and a per-heartbeat explain at the operator's severity — warn while probes fail or restarts accumulate ('All 3 instances healthy — after 4 restarts' keeps a healed crash-loop visible), plain while booting, error with structured details when it gives up. The blue/green switch reports the same set the platform's tracedTrafficSwitch emits — labels, the from→to narrative, the traffic-switch affordance with the resulting split, the request as input, what landed as output, and the convergence toward 100% — from apply_templates, gated explicitly by TRACE_TRAFFIC_SWITCH on the switch_traffic step. Vendored nptrace.sh rebuilt from catalog-tracing-sh#7 (inline io + error --details). --- k8s/apply_templates | 20 ++++ k8s/deployment/wait_deployment_active | 88 ++++++++++++++++++ k8s/deployment/workflows/switch_traffic.yaml | 1 + k8s/logging | 56 +++++++++++ k8s/scope/networking/wait_for_alb | 2 + k8s/utils/tests/trace_logging.bats | 54 +++++++++++ nptrace.sh | 97 +++++++++++++++++--- scheduled_task/logging | 56 +++++++++++ 8 files changed, 363 insertions(+), 11 deletions(-) diff --git a/k8s/apply_templates b/k8s/apply_templates index 5ddbcdff..734b1f8b 100644 --- a/k8s/apply_templates +++ b/k8s/apply_templates @@ -85,4 +85,24 @@ if [[ "$DRY_RUN" == "true" ]]; then exit 1 fi +# When this apply IS the blue/green traffic switch (switch_traffic.yaml sets +# TRACE_TRAFFIC_SWITCH on the step), report the switch the way the platform's +# own scopes do: the request as input, what landed as output, the resulting +# traffic split as the affordance a host renders, and the phase's convergence +# toward 100%. +if [[ "${TRACE_TRAFFIC_SWITCH:-false}" == "true" ]] && [[ -n "${CONTEXT:-}" ]] \ + && command -v np_scope_explain >/dev/null 2>&1; then + TRAFFIC_TO=$(echo "$CONTEXT" | jq -r '.deployment.strategy_data.desired_switched_traffic // 100') + TRAFFIC_FROM=$(echo "$CONTEXT" | jq -r '.deployment.strategy_data.switched_traffic // 0') + if [[ "$TRAFFIC_TO" =~ ^[0-9]+$ ]] && [[ "$TRAFFIC_FROM" =~ ^[0-9]+$ ]]; then + np_scope_labels "deployment.id=${DEPLOYMENT_ID:-}" "scope.id=${SCOPE_ID:-}" "action=traffic-switch" + np_scope_explain --title "Switch traffic for deployment ${DEPLOYMENT_ID:-}" \ + --what "Switching blue/green traffic for deployment ${DEPLOYMENT_ID:-} from ${TRAFFIC_FROM}% to ${TRAFFIC_TO}%" + np_scope_affordance "{\"kind\":\"traffic-switch\",\"deployment_id\":\"${DEPLOYMENT_ID:-}\",\"current_traffic\":$TRAFFIC_TO,\"new_traffic\":$TRAFFIC_TO,\"old_traffic\":$((100 - TRAFFIC_TO)),\"target_traffic\":100}" + np_scope_input traffic "{\"from\":$TRAFFIC_FROM,\"desired\":$TRAFFIC_TO}" + np_scope_output traffic "{\"switched\":$TRAFFIC_TO}" + np_scope_progress "$TRAFFIC_TO" 100 percent + fi +fi + source "$SERVICE_PATH/backup/backup_templates" --action="$ACTION" --files "${APPLIED_FILES[@]}" diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index c7de39b0..b7ed2a31 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -55,6 +55,64 @@ iteration=0 LATEST_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") SKIP_DEPLOYMENT_STATUS_CHECK="${SKIP_DEPLOYMENT_STATUS_CHECK:=false}" LAST_REPORTED_COUNTS="" +# The wait's memory for the trace narrative: how many pods the LAST event +# sweep saw failing health checks (and why), and the pods' restart history — +# a crash-loop that heals mid-wait must not read as an uneventful wait. +UNHEALTHY_POD_COUNT=0 +UNHEALTHY_POD_REASONS="" +WAIT_TITLE="Instance health check" + +# Report the wait's live narrative onto the trace — the counted io, the +# instances-health meter a host renders as pips, and the plain-language +# explain with the severity an operator should read it at. Re-emitted per +# heartbeat: facets fold last-writer-wins, so the latest state (and a +# recovered warning) always wins. Best-effort, like every trace call. +report_wait_narrative() { + command -v np_scope_explain >/dev/null 2>&1 || return 0 + local ready_now="$1" desired_now="$2" launched_now="$3" all_healthy="${4:-false}" + + local restarted restart_total + restarted=$(kubectl get pods -n "$K8S_NAMESPACE" -l "deployment_id=${DEPLOYMENT_ID}" -o json 2>/dev/null \ + | jq -c '[.items[] | {name: .metadata.name, restarts: ([.status.containerStatuses[]?.restartCount] | add // 0)} | select(.restarts > 0)]' 2>/dev/null) || restarted="[]" + [ -n "$restarted" ] || restarted="[]" + restart_total=$(echo "$restarted" | jq 'map(.restarts) | add // 0' 2>/dev/null) || restart_total=0 + + local instances meter + instances=$(jq -nc --argjson h "$ready_now" --argjson l "$launched_now" --argjson d "$desired_now" \ + --argjson u "$UNHEALTHY_POD_COUNT" --argjson r "$restarted" \ + '{healthy: $h, launched: $l, desired: $d} + (if $u > 0 then {unhealthy: $u} else {} end) + (if ($r | length) > 0 then {restarted: $r} else {} end)') + np_scope_output instances "$instances" + + meter=$(jq -nc --argjson h "$ready_now" --argjson l "$launched_now" --argjson d "$desired_now" \ + --argjson u "$UNHEALTHY_POD_COUNT" --arg reasons "$UNHEALTHY_POD_REASONS" --argjson t "$restart_total" \ + '{kind: "instances-health", healthy: $h, launched: $l, desired: $d} + + (if $u > 0 then {unhealthy: $u, reasons: ($reasons | split(", ") | map(select(. != "")))} else {} end) + + (if $t > 0 then {restarts: $t} else {} end)') + np_scope_affordance "$meter" + + local restarts_label="restarts" + [ "$restart_total" -eq 1 ] && restarts_label="restart" + if [ "$UNHEALTHY_POD_COUNT" -gt 0 ]; then + np_scope_explain --title "$WAIT_TITLE" --severity warn \ + --what "Waiting for $ready_now/$desired_now instances to be healthy — $UNHEALTHY_POD_COUNT failing health checks${UNHEALTHY_POD_REASONS:+ ($UNHEALTHY_POD_REASONS)}" \ + --impact "The deployment fails if the instances don't become healthy before the health-check timeout." + elif [ "$restart_total" -gt 0 ]; then + if [ "$all_healthy" = "true" ]; then + np_scope_explain --title "$WAIT_TITLE" --severity warn \ + --what "All $desired_now instances healthy — after $restart_total $restarts_label" \ + --impact "Repeated restarts during startup usually mean crashes or out-of-memory kills; check the instance logs and memory limits." + else + np_scope_explain --title "$WAIT_TITLE" --severity warn \ + --what "Waiting for $ready_now/$desired_now instances to be healthy — $restart_total $restarts_label so far" \ + --impact "Repeated restarts during startup usually mean crashes or out-of-memory kills; check the instance logs and memory limits." + fi + elif [ "$all_healthy" = "true" ]; then + np_scope_explain --title "$WAIT_TITLE" --what "All $desired_now instances healthy" + else + np_scope_explain --title "$WAIT_TITLE" --what "Waiting for $ready_now/$desired_now instances to be healthy" + fi + return 0 +} # Report the instance counters onto the deployment's strategy_data # (amount_instances_to_wait / launched_instances / healthy_instances) so the @@ -113,6 +171,13 @@ while true; do source "$SERVICE_PATH/deployment/print_failed_deployment_hints" if command -v np_scope_step_timeout >/dev/null 2>&1; then + # The terminal narrative replaces the polling one: what it gave up + # with, at ERROR severity, with the structured counts as evidence. + np_scope_explain --title "$WAIT_TITLE" --severity error \ + --what "Gave up with ${ready:-0}/${desired:-0} instances healthy${UNHEALTHY_POD_REASONS:+ ($UNHEALTHY_POD_REASONS)}" + np_scope_error "deployment '$K8S_DEPLOYMENT_NAME' not active after ${TIMEOUT}s" \ + "$(jq -nc --argjson h "${ready:-0}" --argjson l "${launched:-0}" --argjson d "${desired:-0}" \ + '{instances: {healthy: $h, launched: $l, desired: $d}}')" np_scope_step_timeout "deployment '$K8S_DEPLOYMENT_NAME' not active after ${TIMEOUT}s" fi exit 1 @@ -157,6 +222,12 @@ while true; do log info "✅ All pods in deployment '$K8S_DEPLOYMENT_NAME' are available and ready!" if command -v np_scope_step_end >/dev/null 2>&1; then np_scope_progress "$ready" "$desired" instances + # Every pod is ready: any earlier probe warning has recovered, and + # the terminal narrative must say so (a crash-loop on the way here + # stays visible as "healthy — after N restarts"). + UNHEALTHY_POD_COUNT=0 + UNHEALTHY_POD_REASONS="" + report_wait_narrative "$ready" "$desired" "$launched" true # Lineage, by the same canonical ids the platform's own scopes use, # so a custom scope's graph joins by value: @@ -195,6 +266,7 @@ while true; do "wait.desired=$desired" "wait.launched=$launched" \ "wait.ready=$ready" "wait.available=$current" "wait.updated=$updated" np_scope_progress "$ready" "$desired" instances + report_wait_narrative "$ready" "$desired" "$launched" fi fi @@ -262,8 +334,24 @@ while true; do ') if [ -n "$UNHEALTHY_GROUPS" ]; then + UNHEALTHY_POD_COUNT=0 while IFS=$'\t' read -r ts pod_name messages_concat; do [ -z "$pod_name" ] && continue + ((UNHEALTHY_POD_COUNT++)) + # Remember WHY for the trace narrative — the probe kind when the + # message parses, a generic reason when it doesn't. + first_msg=$(printf '%s' "$messages_concat" | tr '\001' '\n' | head -1) + parsed=$(parse_probe_message "$first_msg" 2>/dev/null) || parsed="" + probe_kind="${parsed%%|*}" + if [ -n "$probe_kind" ]; then + unhealthy_reason="$probe_kind probe failing" + else + unhealthy_reason="failing health checks" + fi + case ", $UNHEALTHY_POD_REASONS," in + *", $unhealthy_reason,"*) ;; + *) UNHEALTHY_POD_REASONS="${UNHEALTHY_POD_REASONS:+$UNHEALTHY_POD_REASONS, }$unhealthy_reason" ;; + esac log_unhealthy_group "$ts" "$pod_name" "$messages_concat" \ || log_unhealthy_raw "$ts" "$pod_name" "$messages_concat" done <<< "$UNHEALTHY_GROUPS" diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index a2afec2c..bfdf8023 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -92,6 +92,7 @@ steps: configuration: ACTION: apply DRY_RUN: false + TRACE_TRAFFIC_SWITCH: true post: name: post_apply_checks type: workflow diff --git a/k8s/logging b/k8s/logging index dc6fd3e0..1b6f7c95 100644 --- a/k8s/logging +++ b/k8s/logging @@ -253,6 +253,62 @@ np_scope_progress() { return 0 } +# np_scope_output / np_scope_input +# +# Record what the current step (or open sub-step) produced or consumed as an +# inline value carried in the event — the counted evidence a host renders +# next to the narrative ('instances' with healthy/desired, 'traffic' with the +# switched level). +np_scope_output() { + command -v np_trace_output >/dev/null 2>&1 || return 0 + local _ot_node + _ot_node=$(_np_scopes_node) || return 0 + np_trace_output "$_ot_node" "$1" "$2" + return 0 +} + +np_scope_input() { + command -v np_trace_input >/dev/null 2>&1 || return 0 + local _in_node + _in_node=$(_np_scopes_node) || return 0 + np_trace_input "$_in_node" "$1" "$2" + return 0 +} + +# np_scope_explain --title T [--what W] [--severity ok|warn|error] ... +# +# The plain-language narrative on the current step (or open sub-step) — what +# the phase is doing, in words, with the severity an operator should read it +# at. Re-emitted per poll on waits so the latest state wins. +np_scope_explain() { + command -v np_trace_explain >/dev/null 2>&1 || return 0 + local _ex_node + _ex_node=$(_np_scopes_node) || return 0 + np_trace_explain "$_ex_node" "$@" + return 0 +} + +# np_scope_error [] +# +# An observed failure with its structured evidence. +np_scope_error() { + command -v np_trace_error >/dev/null 2>&1 || return 0 + local _sr_node + _sr_node=$(_np_scopes_node) || return 0 + np_trace_error "$_sr_node" --message "$1" ${2:+--details "$2"} + _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" + return 0 +} + +# np_scope_labels +np_scope_labels() { + command -v np_trace_labels >/dev/null 2>&1 || return 0 + local _lb_node + _lb_node=$(_np_scopes_node) || return 0 + np_trace_labels "$_lb_node" "$@" + return 0 +} + # np_scope_k8s_applied # # Turn `kubectl apply` output lines (`deployment.apps/name created`) into diff --git a/k8s/scope/networking/wait_for_alb b/k8s/scope/networking/wait_for_alb index 3740cbba..74207df8 100644 --- a/k8s/scope/networking/wait_for_alb +++ b/k8s/scope/networking/wait_for_alb @@ -110,6 +110,8 @@ if [ "$state" != "active" ]; then log error " • Check controller logs: kubectl -n kube-system logs deploy/aws-load-balancer-controller" log error " • Verify ALB quota: aws service-quotas get-service-quota --service-code elasticloadbalancing --quota-code L-53DA6B97" if command -v np_scope_step_timeout >/dev/null 2>&1; then + np_scope_explain --title "Wait for ingress" --severity error \ + --what "Gave up waiting for ALB '$ALB_NAME' to become active (last state: ${state:-pending})" np_scope_step_timeout "ALB '$ALB_NAME' not active after ${TIMEOUT_SECONDS}s (last state: ${state:-pending})" fi exit 1 diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index b10a91c4..9d539a60 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -302,3 +302,57 @@ secret/sec-1 created" [ "$status" -eq 0 ] [[ "$output" == *"rc=0"* ]] } + +# --- narrative: inline io, explain, structured errors ------------------------ + +@test "inline output and explain land on the open sub-step" { + run_logged ' + np_scope_step_begin wait-deployment-active + np_scope_output instances "{\"healthy\":2,\"desired\":3}" + np_scope_explain --title "Instance health check" --severity warn --what "Waiting for 2/3 instances to be healthy" + np_scope_step_end 0 + ' + [ "$status" -eq 0 ] + echo "$output" | grep '"tracing.output"' | grep -q 'wait-deployment-active@0.0' + echo "$output" | grep -q '"value":{"healthy":2,"desired":3}' + echo "$output" | grep -q '"severity":"warn"' + echo "$output" | grep -q 'Waiting for 2/3 instances to be healthy' +} + +@test "the traffic switch set lands whole on the current step" { + run_logged ' + np_scope_labels "deployment.id=777" "action=traffic-switch" + np_scope_explain --title "Switch traffic for deployment 777" --what "Switching blue/green traffic for deployment 777 from 0% to 100%" + np_scope_affordance "{\"kind\":\"traffic-switch\",\"deployment_id\":\"777\",\"current_traffic\":100,\"new_traffic\":100,\"old_traffic\":0,\"target_traffic\":100}" + np_scope_input traffic "{\"from\":0,\"desired\":100}" + np_scope_output traffic "{\"switched\":100}" + np_scope_progress 100 100 percent + ' + [ "$status" -eq 0 ] + echo "$output" | grep -q '"kind":"traffic-switch"' + echo "$output" | grep -q '"tracing.input":\[{"kind":"inline","name":"traffic","value":{"from":0,"desired":100}}\]' + echo "$output" | grep -q '"tracing.output":\[{"kind":"inline","name":"traffic","value":{"switched":100}}\]' + echo "$output" | grep -q '"tracing.progress":{"current":100,"target":100,"unit":"percent"}' + echo "$output" | grep -q '"action":"traffic-switch"' +} + +@test "np_scope_error carries structured details and stands down the exit trap" { + run "$BASH" -c " + ( source '$LOGGING'; np_scope_error 'gave up' '{\"instances\":{\"healthy\":1,\"desired\":3}}'; exit 3 ) || true + source '$LOGGING'; trap - EXIT ERR + for f in \"\$NP_TRACE_DIR\"/spool/*.json \"\$NP_TRACE_DIR\"/failed/*.json; do + [ -f \"\$f\" ] && cat \"\$f\" && echo + done + true + " + [ "$status" -eq 0 ] + echo "$output" | grep -q '"details":{"instances":{"healthy":1,"desired":3}}' + [ "$(echo "$output" | grep -c 'tracing.error')" -eq 1 ] +} + +@test "narrative helpers are defined no-ops when untraced" { + unset NP_TRACE + run "$BASH" -c "source '$LOGGING'; np_scope_output x '{}'; np_scope_input x '{}'; np_scope_explain --title t; np_scope_error m; np_scope_labels a=b; echo rc=\$?" + [ "$status" -eq 0 ] + [[ "$output" == *"rc=0"* ]] +} diff --git a/nptrace.sh b/nptrace.sh index 23319e7b..45e24943 100755 --- a/nptrace.sh +++ b/nptrace.sh @@ -1222,11 +1222,15 @@ np_trace_error() { _er_msg='' _er_code='' _er_stack='' + _er_details='' while [ "$#" -gt 0 ]; do case "$1" in --message) _er_msg=${2:-}; shift 2 ;; --code) _er_code=${2:-}; shift 2 ;; --stack-trace) _er_stack=${2:-}; shift 2 ;; + # A JSON object with the diagnosis's structured evidence (counts, the + # failing probe, ...) — the sibling SDKs' error `details`. + --details) _er_details=${2:-}; shift 2 ;; *) if [ -z "$_er_msg" ]; then _er_msg=$1 @@ -1236,8 +1240,16 @@ np_trace_error() { esac done [ -n "$_er_msg" ] || return 0 + case "$_er_details" in + '' | \{*) ;; + *) _er_details='' ;; + esac np__stage_facet "$_er_h" "$NP_FACET_ERROR" \ - "$(np__json_obj message "$_er_msg" code "$_er_code" stack_trace "$_er_stack")" + "$(np__json_obj_raw \ + message "$(np__json_str "$_er_msg")" \ + code "$(if [ -n "$_er_code" ]; then np__json_str "$_er_code"; fi)" \ + stack_trace "$(if [ -n "$_er_stack" ]; then np__json_str "$_er_stack"; fi)" \ + details "$_er_details")" np__flush_foreign "$_er_h" return 0 } @@ -1281,6 +1293,78 @@ np__dataset_ref() { np__json_obj type dataset id "$1" } +# Append one io descriptor to a direction's list; the facet is re-staged +# whole each time (last write wins per namespace), so the array only ever +# grows. $1 handle, $2 facet namespace, $3 descriptor store key, $4 the +# already-formed descriptor JSON. +np__append_io_descriptor() { + _ai_descriptors=$(np__node_get "$1" "$3") + if [ -n "$_ai_descriptors" ]; then + _ai_descriptors="$_ai_descriptors,$4" + else + _ai_descriptors=$4 + fi + np__node_set "$1" "$3" "$_ai_descriptors" + np__stage_facet "$1" "$2" "[$_ai_descriptors]" + return 0 +} + +# The shared body of np_trace_output / np_trace_input: an INLINE io +# descriptor — a small value carried in the event itself, the sibling SDKs' +# step.output(name, value). $1 direction (out|in), $2 verb name for drop +# records, then the caller's argv: [handle] . +np__inline_io() { + _ii_direction=$1 + _ii_verb=$2 + shift 2 + _ii_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_ii_handle" || { np__drop "$_ii_verb" 'no node in scope'; return 0; } + _ii_name=${1:-} + _ii_value=${2:-} + if [ -z "$_ii_name" ] || [ -z "$_ii_value" ]; then + np__drop "$_ii_verb" 'name and a JSON value are required' + return 0 + fi + case "$_ii_value" in + \{* | \[* | \"* | [0-9-]* | true | false | null) ;; + *) np__drop "$_ii_verb" 'value must be JSON'; return 0 ;; + esac + if [ "$_ii_direction" = 'out' ]; then + _ii_facet_namespace=$NP_FACET_OUTPUT + _ii_store=io_output + else + _ii_facet_namespace=$NP_FACET_INPUT + _ii_store=io_input + fi + _ii_descriptor=$(np__json_obj_raw kind '"inline"' name "$(np__json_str "$_ii_name")" value "$_ii_value") + np__append_io_descriptor "$_ii_handle" "$_ii_facet_namespace" "$_ii_store" "$_ii_descriptor" + np__flush_foreign "$_ii_handle" + return 0 +} + +# np_trace_output [handle] +# +# Record what this node PRODUCED as an inline value carried in the event — +# `np_trace_output instances '{"healthy":2,"desired":3}'`. For an artifact +# with an address, prefer np_trace_produces (pointer + lineage edge). +np_trace_output() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + np__inline_io out output "$@" + return 0 +} + +# np_trace_input [handle] +# +# Record what this node CONSUMED as an inline value; see np_trace_output. +np_trace_input() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + np__inline_io in input "$@" + return 0 +} + # np__emit_io_edge [pointer-name] [pointer-uri] # # Emit one lineage edge. The direction decides everything else: `out` is @@ -1314,16 +1398,7 @@ np__emit_io_edge() { _io_pointer='' if [ -n "$_io_pointer_name" ] && [ -n "$_io_pointer_uri" ]; then _io_pointer=$(np__json_obj kind pointer name "$_io_pointer_name" uri "$_io_pointer_uri") - # Append to the direction's descriptor list; the facet is re-staged whole - # each time (last write wins per namespace), so the array only ever grows. - _io_descriptors=$(np__node_get "$_io_handle" "$_io_descriptor_store") - if [ -n "$_io_descriptors" ]; then - _io_descriptors="$_io_descriptors,$_io_pointer" - else - _io_descriptors=$_io_pointer - fi - np__node_set "$_io_handle" "$_io_descriptor_store" "$_io_descriptors" - np__stage_facet "$_io_handle" "$_io_facet_namespace" "[$_io_descriptors]" + np__append_io_descriptor "$_io_handle" "$_io_facet_namespace" "$_io_descriptor_store" "$_io_pointer" fi # An edge must not point FROM a node the read model has never seen. diff --git a/scheduled_task/logging b/scheduled_task/logging index dc6fd3e0..1b6f7c95 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -253,6 +253,62 @@ np_scope_progress() { return 0 } +# np_scope_output / np_scope_input +# +# Record what the current step (or open sub-step) produced or consumed as an +# inline value carried in the event — the counted evidence a host renders +# next to the narrative ('instances' with healthy/desired, 'traffic' with the +# switched level). +np_scope_output() { + command -v np_trace_output >/dev/null 2>&1 || return 0 + local _ot_node + _ot_node=$(_np_scopes_node) || return 0 + np_trace_output "$_ot_node" "$1" "$2" + return 0 +} + +np_scope_input() { + command -v np_trace_input >/dev/null 2>&1 || return 0 + local _in_node + _in_node=$(_np_scopes_node) || return 0 + np_trace_input "$_in_node" "$1" "$2" + return 0 +} + +# np_scope_explain --title T [--what W] [--severity ok|warn|error] ... +# +# The plain-language narrative on the current step (or open sub-step) — what +# the phase is doing, in words, with the severity an operator should read it +# at. Re-emitted per poll on waits so the latest state wins. +np_scope_explain() { + command -v np_trace_explain >/dev/null 2>&1 || return 0 + local _ex_node + _ex_node=$(_np_scopes_node) || return 0 + np_trace_explain "$_ex_node" "$@" + return 0 +} + +# np_scope_error [] +# +# An observed failure with its structured evidence. +np_scope_error() { + command -v np_trace_error >/dev/null 2>&1 || return 0 + local _sr_node + _sr_node=$(_np_scopes_node) || return 0 + np_trace_error "$_sr_node" --message "$1" ${2:+--details "$2"} + _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" + return 0 +} + +# np_scope_labels +np_scope_labels() { + command -v np_trace_labels >/dev/null 2>&1 || return 0 + local _lb_node + _lb_node=$(_np_scopes_node) || return 0 + np_trace_labels "$_lb_node" "$@" + return 0 +} + # np_scope_k8s_applied # # Turn `kubectl apply` output lines (`deployment.apps/name created`) into From a125e9280cb283dc5b3de1848bdf77c38b3e0719 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 14:31:08 -0300 Subject: [PATCH 12/52] =?UTF-8?q?feat(k8s):=20complete=20the=20annotation?= =?UTF-8?q?=20coverage=20=E2=80=94=20every=20workflow,=20every=20flavor,?= =?UTF-8?q?=20the=20run's=20own=20reason?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every workflow now declares its trace identity, not just the three lifecycle ones: blue_green overrides the job it would otherwise inherit from initial (k8s-deployment-blue-green, with its own groups), update overrides create's (k8s-scope-update), finalize/rollback/delete and the scheduled-task trigger get their own jobs, finalize's rollout wait is keyed wait-for-instances, and the plumbing steps (load logging, assume role, deployment build context) are trace: false across all twenty-odd workflow files. A fatal exit now mirrors the REAL reason onto the RUN as well as the step — the last log error's message, not the exit mechanism — so list views name the cause without drilling into steps. A run-level NP_TRACE (no step segment) mirrors nowhere extra. --- k8s/deployment/workflows/blue_green.yaml | 6 +++ k8s/deployment/workflows/delete.yaml | 6 +++ k8s/deployment/workflows/diagnose.yaml | 1 + k8s/deployment/workflows/finalize.yaml | 13 +++++++ k8s/deployment/workflows/kill_instance.yaml | 1 + k8s/deployment/workflows/rollback.yaml | 9 +++++ k8s/logging | 34 +++++++++++++++-- k8s/scope/workflows/delete.yaml | 5 +++ k8s/scope/workflows/diagnose.yaml | 1 + k8s/scope/workflows/pause-autoscaling.yaml | 1 + k8s/scope/workflows/restart-pods.yaml | 1 + k8s/scope/workflows/resume-autoscaling.yaml | 1 + .../workflows/set-desired-instance-count.yaml | 1 + k8s/scope/workflows/update.yaml | 3 ++ k8s/utils/tests/trace_logging.bats | 37 +++++++++++++++++-- scheduled_task/logging | 34 +++++++++++++++-- .../scope/workflows/trigger-job.yaml | 3 ++ 17 files changed, 146 insertions(+), 11 deletions(-) diff --git a/k8s/deployment/workflows/blue_green.yaml b/k8s/deployment/workflows/blue_green.yaml index 7fac13fb..e1bb8c86 100644 --- a/k8s/deployment/workflows/blue_green.yaml +++ b/k8s/deployment/workflows/blue_green.yaml @@ -2,6 +2,12 @@ include: - "$SERVICE_PATH/deployment/workflows/initial.yaml" configuration: INGRESS_TEMPLATE: "$BLUE_GREEN_INGRESS_PATH" +trace: + flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + job: k8s-deployment-blue-green + groups: + - {key: setting-up, title: Setting up} + - {key: waiting-instances, title: Waiting for instances to be healthy} steps: - name: update blue deployment type: script diff --git a/k8s/deployment/workflows/delete.yaml b/k8s/deployment/workflows/delete.yaml index 538679a5..0c99a14e 100644 --- a/k8s/deployment/workflows/delete.yaml +++ b/k8s/deployment/workflows/delete.yaml @@ -1,9 +1,13 @@ include: - "$SERVICE_PATH/values.yaml" +trace: + flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + job: k8s-deployment-delete steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function @@ -13,6 +17,7 @@ steps: - name: assume role type: script file: "$SERVICE_PATH/utils/assume_role_step" + trace: false output: - name: AWS_ACCESS_KEY_ID type: environment @@ -23,6 +28,7 @@ steps: - name: build context type: script file: "$SERVICE_PATH/deployment/build_context" + trace: false output: - name: CONTEXT type: environment diff --git a/k8s/deployment/workflows/diagnose.yaml b/k8s/deployment/workflows/diagnose.yaml index 45d837c3..faf96a44 100644 --- a/k8s/deployment/workflows/diagnose.yaml +++ b/k8s/deployment/workflows/diagnose.yaml @@ -5,6 +5,7 @@ steps: - name: load_functions type: script file: "$SERVICE_PATH/diagnose/utils/diagnose_utils" + trace: false output: - name: update_check_result type: function diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index e0246180..ff1e06de 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -2,10 +2,17 @@ include: - "$SERVICE_PATH/values.yaml" configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" +trace: + flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + job: k8s-deployment-finalize + groups: + - {key: setting-up, title: Setting up} + - {key: waiting-instances, title: Waiting for instances to be healthy} steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function @@ -15,6 +22,7 @@ steps: - name: assume role type: script file: "$SERVICE_PATH/utils/assume_role_step" + trace: false output: - name: AWS_ACCESS_KEY_ID type: environment @@ -25,6 +33,7 @@ steps: - name: build context type: script file: "$SERVICE_PATH/deployment/build_context" + trace: false output: - name: CONTEXT type: environment @@ -45,6 +54,10 @@ steps: name: wait deployment active type: script file: "$SERVICE_PATH/deployment/wait_deployment_active" + trace: + key: wait-for-instances + title: Instance health check + group: waiting-instances configuration: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS SKIP_DEPLOYMENT_STATUS_CHECK: true diff --git a/k8s/deployment/workflows/kill_instance.yaml b/k8s/deployment/workflows/kill_instance.yaml index 74f8427c..5a8d2676 100644 --- a/k8s/deployment/workflows/kill_instance.yaml +++ b/k8s/deployment/workflows/kill_instance.yaml @@ -4,6 +4,7 @@ steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index 29a919b2..4db8f0d3 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -2,10 +2,17 @@ include: - "$SERVICE_PATH/values.yaml" configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" +trace: + flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + job: k8s-deployment-rollback + groups: + - {key: setting-up, title: Setting up} + - {key: waiting-instances, title: Waiting for instances to be healthy} steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function @@ -15,6 +22,7 @@ steps: - name: assume role type: script file: "$SERVICE_PATH/utils/assume_role_step" + trace: false output: - name: AWS_ACCESS_KEY_ID type: environment @@ -25,6 +33,7 @@ steps: - name: build context type: script file: "$SERVICE_PATH/deployment/build_context" + trace: false output: - name: CONTEXT type: environment diff --git a/k8s/logging b/k8s/logging index 1b6f7c95..6a14dca1 100644 --- a/k8s/logging +++ b/k8s/logging @@ -96,9 +96,11 @@ _np_scopes_trace_error() { local _lt_node _lt_node=$(_np_scopes_node) || return 0 np_trace_error "$_lt_node" --message "$1" ${2:+--code "$2"} - # Remember which step already carries a real message, so the exit trap does - # not shadow it with a generic one. + # Remember which step already carries a real message (so the exit trap does + # not shadow it with a generic one) AND the message itself — on a fatal + # exit, the run-level mirror repeats the real diagnosis, not the mechanism. _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" + _NP_SCOPES_LAST_REASON="$1" return 0 } @@ -342,11 +344,35 @@ _np_scopes_on_err() { _NP_SCOPES_LAST_ERR="$BASH_COMMAND (exit ${1:-1})" } +# On a fatal exit the RUN's terminal deserves the real reason too: the step +# carries it already; this mirrors it one level up, so list views (which lead +# with the run) name the cause without drilling into steps. The run id is the +# step path minus its last segment — an observed fact on an adopted node, +# like every other enrichment here. +_np_scopes_error_on_run() { + command -v np_trace_adopt >/dev/null 2>&1 || return 0 + [ -n "${NP_TRACE:-}" ] || return 0 + local _ru_rest="${NP_TRACE#*|}" + local _ru_version="${NP_TRACE%%|*}" _ru_trace="${_ru_rest%%|*}" _ru_run="${_ru_rest#*|}" + case "$_ru_run" in + *~*) _ru_run="${_ru_run%~*}" ;; + *) return 0 ;; + esac + local _ru_node + _ru_node=$(np_trace_adopt "${_ru_version}|${_ru_trace}|${_ru_run}" 2>/dev/null) || return 0 + [ -n "$_ru_node" ] || return 0 + np_trace_error "$_ru_node" --message "$1" + return 0 +} + _np_scopes_on_exit() { local _ex_rc="${1:-0}" + local _ex_reason="${_NP_SCOPES_LAST_REASON:-${_NP_SCOPES_LAST_ERR:-workflow shell exited with status $_ex_rc}}" if [ "$_ex_rc" -ne 0 ] && [ "${_NP_SCOPES_ERRED_ON:-}" != "${NP_TRACE:-}" ]; then - _np_scopes_trace_error \ - "${_NP_SCOPES_LAST_ERR:-workflow shell exited with status $_ex_rc}" || true + _np_scopes_trace_error "$_ex_reason" || true + fi + if [ "$_ex_rc" -ne 0 ]; then + _np_scopes_error_on_run "$_ex_reason" || true fi # A sub-step still open when the shell dies inherits the shell's outcome — # closed AFTER the error report above so the failure detail lands on it. diff --git a/k8s/scope/workflows/delete.yaml b/k8s/scope/workflows/delete.yaml index e411bedb..e22b313a 100644 --- a/k8s/scope/workflows/delete.yaml +++ b/k8s/scope/workflows/delete.yaml @@ -1,9 +1,13 @@ include: - "$SERVICE_PATH/values.yaml" +trace: + flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + job: k8s-scope-delete steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function @@ -13,6 +17,7 @@ steps: - name: assume role type: script file: "$SERVICE_PATH/utils/assume_role_step" + trace: false output: - name: AWS_ACCESS_KEY_ID type: environment diff --git a/k8s/scope/workflows/diagnose.yaml b/k8s/scope/workflows/diagnose.yaml index 45d837c3..faf96a44 100644 --- a/k8s/scope/workflows/diagnose.yaml +++ b/k8s/scope/workflows/diagnose.yaml @@ -5,6 +5,7 @@ steps: - name: load_functions type: script file: "$SERVICE_PATH/diagnose/utils/diagnose_utils" + trace: false output: - name: update_check_result type: function diff --git a/k8s/scope/workflows/pause-autoscaling.yaml b/k8s/scope/workflows/pause-autoscaling.yaml index 362ef27c..fce1a9da 100644 --- a/k8s/scope/workflows/pause-autoscaling.yaml +++ b/k8s/scope/workflows/pause-autoscaling.yaml @@ -4,6 +4,7 @@ steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function diff --git a/k8s/scope/workflows/restart-pods.yaml b/k8s/scope/workflows/restart-pods.yaml index 7771041a..3b024367 100644 --- a/k8s/scope/workflows/restart-pods.yaml +++ b/k8s/scope/workflows/restart-pods.yaml @@ -4,6 +4,7 @@ steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function diff --git a/k8s/scope/workflows/resume-autoscaling.yaml b/k8s/scope/workflows/resume-autoscaling.yaml index 8b155b68..c6cb36f0 100644 --- a/k8s/scope/workflows/resume-autoscaling.yaml +++ b/k8s/scope/workflows/resume-autoscaling.yaml @@ -4,6 +4,7 @@ steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function diff --git a/k8s/scope/workflows/set-desired-instance-count.yaml b/k8s/scope/workflows/set-desired-instance-count.yaml index 03e3ba0f..12759b4b 100644 --- a/k8s/scope/workflows/set-desired-instance-count.yaml +++ b/k8s/scope/workflows/set-desired-instance-count.yaml @@ -4,6 +4,7 @@ steps: - name: load logging type: script file: "$SERVICE_PATH/logging" + trace: false output: - name: log type: function diff --git a/k8s/scope/workflows/update.yaml b/k8s/scope/workflows/update.yaml index b1e48442..2feb3e4d 100644 --- a/k8s/scope/workflows/update.yaml +++ b/k8s/scope/workflows/update.yaml @@ -1,5 +1,8 @@ include: - "$SERVICE_PATH/scope/workflows/create.yaml" +trace: + flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + job: k8s-scope-update steps: - name: networking type: workflow diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index 9d539a60..74e00324 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -98,8 +98,9 @@ run_logged() { " [ "$status" -eq 0 ] echo "$output" | grep -q 'the real reason' - # exactly one error facet: the generic exit report stood down - [ "$(echo "$output" | grep -c 'tracing.error')" -eq 1 ] + # exactly one error facet ON THE STEP: the generic exit report stood down + # (the run-level mirror is a separate node and is asserted elsewhere) + [ "$(echo "$output" | grep 'apply-manifests@0.0"' | grep -c 'tracing.error')" -eq 1 ] } @test "wait heartbeat marks the step waiting with progress labels" { @@ -347,7 +348,7 @@ secret/sec-1 created" " [ "$status" -eq 0 ] echo "$output" | grep -q '"details":{"instances":{"healthy":1,"desired":3}}' - [ "$(echo "$output" | grep -c 'tracing.error')" -eq 1 ] + [ "$(echo "$output" | grep 'apply-manifests@0.0"' | grep -c 'tracing.error')" -eq 1 ] } @test "narrative helpers are defined no-ops when untraced" { @@ -356,3 +357,33 @@ secret/sec-1 created" [ "$status" -eq 0 ] [[ "$output" == *"rc=0"* ]] } + +@test "a fatal exit mirrors the real reason onto the RUN, one level up" { + run "$BASH" -c " + ( source '$LOGGING'; log error 'the real reason'; exit 3 ) || true + source '$LOGGING'; trap - EXIT ERR + for f in \"\$NP_TRACE_DIR\"/spool/*.json \"\$NP_TRACE_DIR\"/failed/*.json; do + [ -f \"\$f\" ] && cat \"\$f\" && echo + done + true + " + [ "$status" -eq 0 ] + # the step carries it... + echo "$output" | grep '"the real reason"' | grep -q 'scope-provision-42~apply-manifests@0.0"' + # ...and so does the run (the step path minus its last segment) + echo "$output" | grep '"the real reason"' | grep -q '"run_id":"scope-provision-42"' +} + +@test "a run-level NP_TRACE (no step segment) mirrors nowhere extra" { + run "$BASH" -c " + export NP_TRACE='1|trace-9|scope-provision-42' + ( source '$LOGGING'; log error 'root failure'; exit 3 ) || true + source '$LOGGING'; trap - EXIT ERR + for f in \"\$NP_TRACE_DIR\"/spool/*.json \"\$NP_TRACE_DIR\"/failed/*.json; do + [ -f \"\$f\" ] && cat \"\$f\" && echo + done + true + " + [ "$status" -eq 0 ] + [ "$(echo "$output" | grep -c '"root failure"')" -eq 1 ] +} diff --git a/scheduled_task/logging b/scheduled_task/logging index 1b6f7c95..6a14dca1 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -96,9 +96,11 @@ _np_scopes_trace_error() { local _lt_node _lt_node=$(_np_scopes_node) || return 0 np_trace_error "$_lt_node" --message "$1" ${2:+--code "$2"} - # Remember which step already carries a real message, so the exit trap does - # not shadow it with a generic one. + # Remember which step already carries a real message (so the exit trap does + # not shadow it with a generic one) AND the message itself — on a fatal + # exit, the run-level mirror repeats the real diagnosis, not the mechanism. _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" + _NP_SCOPES_LAST_REASON="$1" return 0 } @@ -342,11 +344,35 @@ _np_scopes_on_err() { _NP_SCOPES_LAST_ERR="$BASH_COMMAND (exit ${1:-1})" } +# On a fatal exit the RUN's terminal deserves the real reason too: the step +# carries it already; this mirrors it one level up, so list views (which lead +# with the run) name the cause without drilling into steps. The run id is the +# step path minus its last segment — an observed fact on an adopted node, +# like every other enrichment here. +_np_scopes_error_on_run() { + command -v np_trace_adopt >/dev/null 2>&1 || return 0 + [ -n "${NP_TRACE:-}" ] || return 0 + local _ru_rest="${NP_TRACE#*|}" + local _ru_version="${NP_TRACE%%|*}" _ru_trace="${_ru_rest%%|*}" _ru_run="${_ru_rest#*|}" + case "$_ru_run" in + *~*) _ru_run="${_ru_run%~*}" ;; + *) return 0 ;; + esac + local _ru_node + _ru_node=$(np_trace_adopt "${_ru_version}|${_ru_trace}|${_ru_run}" 2>/dev/null) || return 0 + [ -n "$_ru_node" ] || return 0 + np_trace_error "$_ru_node" --message "$1" + return 0 +} + _np_scopes_on_exit() { local _ex_rc="${1:-0}" + local _ex_reason="${_NP_SCOPES_LAST_REASON:-${_NP_SCOPES_LAST_ERR:-workflow shell exited with status $_ex_rc}}" if [ "$_ex_rc" -ne 0 ] && [ "${_NP_SCOPES_ERRED_ON:-}" != "${NP_TRACE:-}" ]; then - _np_scopes_trace_error \ - "${_NP_SCOPES_LAST_ERR:-workflow shell exited with status $_ex_rc}" || true + _np_scopes_trace_error "$_ex_reason" || true + fi + if [ "$_ex_rc" -ne 0 ]; then + _np_scopes_error_on_run "$_ex_reason" || true fi # A sub-step still open when the shell dies inherits the shell's outcome — # closed AFTER the error report above so the failure detail lands on it. diff --git a/scheduled_task/scope/workflows/trigger-job.yaml b/scheduled_task/scope/workflows/trigger-job.yaml index 02df28f0..b07ade7e 100644 --- a/scheduled_task/scope/workflows/trigger-job.yaml +++ b/scheduled_task/scope/workflows/trigger-job.yaml @@ -3,10 +3,13 @@ include: provider_categories: - container-orchestration - cloud-providers +trace: + job: scheduled-task-trigger steps: - name: load logging type: script file: "$OVERRIDES_PATH/logging" + trace: false output: - name: log type: function From 314f41a9ed1989fc0f149e8c852aa5b2ec0268ce Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 17:26:21 -0300 Subject: [PATCH 13/52] chore(tracing): re-vendor nptrace.sh with the complete surface (catalog-tracing-sh#8) --- nptrace.sh | 1511 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 1046 insertions(+), 465 deletions(-) diff --git a/nptrace.sh b/nptrace.sh index 45e24943..b26096db 100755 --- a/nptrace.sh +++ b/nptrace.sh @@ -30,12 +30,12 @@ NP_TRACE_VERSION="0.1.0" # 2001-09-09 and stay 13 until 2286, while seconds are 10. Anything shorter than # 13 is not milliseconds, whatever it looks like. np__epoch_ms() { - _cm_ms=$(date -u +%s%3N 2>/dev/null) || _cm_ms='' - case "$_cm_ms" in - '' | *[!0-9]*) _cm_ms='' ;; + _epoch_ms_ms=$(date -u +%s%3N 2>/dev/null) || _epoch_ms_ms='' + case "$_epoch_ms_ms" in + '' | *[!0-9]*) _epoch_ms_ms='' ;; esac - if [ -n "$_cm_ms" ] && [ "${#_cm_ms}" -ge 13 ]; then - printf '%s' "$_cm_ms" + if [ -n "$_epoch_ms_ms" ] && [ "${#_epoch_ms_ms}" -ge 13 ]; then + printf '%s' "$_epoch_ms_ms" return 0 fi # Second precision. Event ids stay unique via their random bits. @@ -44,9 +44,9 @@ np__epoch_ms() { # Exactly $1 lowercase hex characters from the kernel CSPRNG. np__rand_hex() { - _rh_want=$1 - _rh_bytes=$(( (_rh_want + 1) / 2 )) - od -An -tx1 -N"$_rh_bytes" /dev/urandom | tr -d ' \n' | cut -c1-"$_rh_want" + _rand_hex_want=$1 + _rand_hex_bytes=$(( (_rand_hex_want + 1) / 2 )) + od -An -tx1 -N"$_rand_hex_bytes" /dev/urandom | tr -d ' \n' | cut -c1-"$_rand_hex_want" } # RFC 3339 UTC, second precision — the envelope `time` field. @@ -114,33 +114,54 @@ np__json_str() { # JSON strings. A pair whose key or value is empty is OMITTED — an absent # optional is absent, never the string "". np__json_obj() { - _jo_out='' + _json_obj_out='' while [ "$#" -ge 2 ]; do if [ -n "$1" ] && [ -n "$2" ]; then - if [ -n "$_jo_out" ]; then - _jo_out="$_jo_out," + if [ -n "$_json_obj_out" ]; then + _json_obj_out="$_json_obj_out," fi - _jo_out="$_jo_out$(np__json_str "$1"):$(np__json_str "$2")" + _json_obj_out="$_json_obj_out$(np__json_str "$1"):$(np__json_str "$2")" fi shift 2 done - printf '{%s}' "$_jo_out" + printf '{%s}' "$_json_obj_out" } # As np__json_obj, but each value is already-formed JSON inserted verbatim. # Use for nested objects, arrays, numbers, and booleans. np__json_obj_raw() { - _jor_out='' + _json_obj_raw_out='' while [ "$#" -ge 2 ]; do if [ -n "$1" ] && [ -n "$2" ]; then - if [ -n "$_jor_out" ]; then - _jor_out="$_jor_out," + if [ -n "$_json_obj_raw_out" ]; then + _json_obj_raw_out="$_json_obj_raw_out," fi - _jor_out="$_jor_out$(np__json_str "$1"):$2" + _json_obj_raw_out="$_json_obj_raw_out$(np__json_str "$1"):$2" fi shift 2 done - printf '{%s}' "$_jor_out" + printf '{%s}' "$_json_obj_raw_out" +} + +# A JSON array of strings from a comma-separated list ("a, b" → ["a","b"]). +# Surrounding whitespace per item is trimmed; empty items are omitted. +np__json_str_array_csv() { + _json_str_array_csv_out='' + _json_str_array_csv_rest=$1 + while [ -n "$_json_str_array_csv_rest" ]; do + case "$_json_str_array_csv_rest" in + *,*) _json_str_array_csv_item=${_json_str_array_csv_rest%%,*}; _json_str_array_csv_rest=${_json_str_array_csv_rest#*,} ;; + *) _json_str_array_csv_item=$_json_str_array_csv_rest; _json_str_array_csv_rest='' ;; + esac + _json_str_array_csv_item=$(printf '%s' "$_json_str_array_csv_item" | sed 's/^ *//; s/ *$//') + if [ -n "$_json_str_array_csv_item" ]; then + if [ -n "$_json_str_array_csv_out" ]; then + _json_str_array_csv_out="$_json_str_array_csv_out," + fi + _json_str_array_csv_out="$_json_str_array_csv_out$(np__json_str "$_json_str_array_csv_item")" + fi + done + printf '[%s]' "$_json_str_array_csv_out" } # ---- src/uuid.sh ---- @@ -252,26 +273,26 @@ np__parse_node_id() { *"$NP_ID_DELIMITER"*) ;; *) return 1 ;; esac - _pn_parent=${1%"$NP_ID_DELIMITER"*} - _pn_tail=${1##*"$NP_ID_DELIMITER"} - case "$_pn_tail" in + _parse_node_id_parent=${1%"$NP_ID_DELIMITER"*} + _parse_node_id_tail=${1##*"$NP_ID_DELIMITER"} + case "$_parse_node_id_tail" in *@*.*) ;; *) return 1 ;; esac - _pn_key=${_pn_tail%%@*} - _pn_coord=${_pn_tail#*@} - _pn_attempt=${_pn_coord%%.*} - _pn_iteration=${_pn_coord#*.} - if [ -z "$_pn_parent" ] || [ -z "$_pn_key" ]; then + _parse_node_id_key=${_parse_node_id_tail%%@*} + _parse_node_id_coord=${_parse_node_id_tail#*@} + _parse_node_id_attempt=${_parse_node_id_coord%%.*} + _parse_node_id_iteration=${_parse_node_id_coord#*.} + if [ -z "$_parse_node_id_parent" ] || [ -z "$_parse_node_id_key" ]; then return 1 fi - case "$_pn_attempt" in + case "$_parse_node_id_attempt" in '' | *[!0-9]*) return 1 ;; esac - case "$_pn_iteration" in + case "$_parse_node_id_iteration" in '' | *[!0-9]*) return 1 ;; esac - printf '%s %s %s %s' "$_pn_parent" "$_pn_key" "$_pn_attempt" "$_pn_iteration" + printf '%s %s %s %s' "$_parse_node_id_parent" "$_parse_node_id_key" "$_parse_node_id_attempt" "$_parse_node_id_iteration" } # Join parts into a stable id, dropping empty parts. Use instead of @@ -389,15 +410,15 @@ np__state_init() { # Allocate the next handle. Handles are opaque by contract: consumers never # parse them. np__handle_new() { - _hn_seq=$(cat "$NP_TRACE_DIR/seq" 2>/dev/null || printf '0') - case "$_hn_seq" in - '' | *[!0-9]*) _hn_seq=0 ;; + _handle_new_seq=$(cat "$NP_TRACE_DIR/seq" 2>/dev/null || printf '0') + case "$_handle_new_seq" in + '' | *[!0-9]*) _handle_new_seq=0 ;; esac - _hn_seq=$((_hn_seq + 1)) - printf '%s' "$_hn_seq" > "$NP_TRACE_DIR/seq" - _hn_handle="n$_hn_seq" - : > "$NP_TRACE_DIR/nodes/$_hn_handle" - printf '%s' "$_hn_handle" + _handle_new_seq=$((_handle_new_seq + 1)) + printf '%s' "$_handle_new_seq" > "$NP_TRACE_DIR/seq" + _handle_new_handle="n$_handle_new_seq" + : > "$NP_TRACE_DIR/nodes/$_handle_new_handle" + printf '%s' "$_handle_new_handle" } # THE rule the whole public surface rests on: an argument is a handle iff it @@ -413,23 +434,23 @@ np__is_handle() { } np__node_set() { - _ns_file="$NP_TRACE_DIR/nodes/$1" - [ -f "$_ns_file" ] || return 0 + _node_set_file="$NP_TRACE_DIR/nodes/$1" + [ -f "$_node_set_file" ] || return 0 # Drop any prior value for this key, then append the new one. The trailing # '=' in the match means a key that is a prefix of another never collides. - if grep -q "^$2=" "$_ns_file" 2>/dev/null; then - grep -v "^$2=" "$_ns_file" > "$_ns_file.tmp" 2>/dev/null || : > "$_ns_file.tmp" - mv "$_ns_file.tmp" "$_ns_file" + if grep -q "^$2=" "$_node_set_file" 2>/dev/null; then + grep -v "^$2=" "$_node_set_file" > "$_node_set_file.tmp" 2>/dev/null || : > "$_node_set_file.tmp" + mv "$_node_set_file.tmp" "$_node_set_file" fi - printf '%s=%s\n' "$2" "$3" >> "$_ns_file" + printf '%s=%s\n' "$2" "$3" >> "$_node_set_file" return 0 } np__node_get() { - _ng_file="$NP_TRACE_DIR/nodes/$1" - [ -f "$_ng_file" ] || return 0 + _node_get_file="$NP_TRACE_DIR/nodes/$1" + [ -f "$_node_get_file" ] || return 0 # Strip only the leading "key=", so a value containing '=' survives intact. - sed -n "s/^$2=//p" "$_ng_file" 2>/dev/null | head -n 1 + sed -n "s/^$2=//p" "$_node_get_file" 2>/dev/null | head -n 1 return 0 } @@ -485,31 +506,31 @@ np__resolve_handle() { # np__spool -> prints the event id np__spool() { - _sp_id=$(np__uuidv7) - _sp_env=$(np__json_obj_raw \ - id "$(np__json_str "$_sp_id")" \ + _spool_id=$(np__uuidv7) + _spool_env=$(np__json_obj_raw \ + id "$(np__json_str "$_spool_id")" \ time "$(np__json_str "$(np__iso8601)")" \ type "$(np__json_str "$1")" \ nrn "$(if [ -n "$2" ]; then np__json_str "$2"; fi)" \ producer "$(np__json_str "${NP_TRACE_PRODUCER:-}")" \ data "$3") - _sp_tmp="$NP_TRACE_DIR/spool/$_sp_id.json.tmp" - _sp_final="$NP_TRACE_DIR/spool/$_sp_id.json" - printf '%s' "$_sp_env" > "$_sp_tmp" 2>/dev/null || return 0 + _spool_tmp="$NP_TRACE_DIR/spool/$_spool_id.json.tmp" + _spool_final="$NP_TRACE_DIR/spool/$_spool_id.json" + printf '%s' "$_spool_env" > "$_spool_tmp" 2>/dev/null || return 0 # Create-then-rename: a concurrent flush never sees a half-written envelope. - mv "$_sp_tmp" "$_sp_final" 2>/dev/null || return 0 - printf '%s' "$_sp_id" + mv "$_spool_tmp" "$_spool_final" 2>/dev/null || return 0 + printf '%s' "$_spool_id" return 0 } np__spool_count() { - _sc_n=0 - for _sc_f in "$NP_TRACE_DIR/spool"/*.json; do - [ -f "$_sc_f" ] || continue - _sc_n=$((_sc_n + 1)) + _spool_count_n=0 + for _spool_count_f in "$NP_TRACE_DIR/spool"/*.json; do + [ -f "$_spool_count_f" ] || continue + _spool_count_n=$((_spool_count_n + 1)) done - printf '%s' "$_sc_n" + printf '%s' "$_spool_count_n" return 0 } @@ -576,34 +597,34 @@ np__token_exchange() { return 0 fi - _tk_cache="$NP_TRACE_DIR/token" - if [ -f "$_tk_cache" ]; then - _tk_exp=$(sed -n '1p' "$_tk_cache" 2>/dev/null) - _tk_val=$(sed -n '2p' "$_tk_cache" 2>/dev/null) - case "$_tk_exp" in - '' | *[!0-9]*) _tk_exp=0 ;; + _token_exchange_cache="$NP_TRACE_DIR/token" + if [ -f "$_token_exchange_cache" ]; then + _token_exchange_exp=$(sed -n '1p' "$_token_exchange_cache" 2>/dev/null) + _token_exchange_val=$(sed -n '2p' "$_token_exchange_cache" 2>/dev/null) + case "$_token_exchange_exp" in + '' | *[!0-9]*) _token_exchange_exp=0 ;; esac - if [ -n "$_tk_val" ] && [ "$_tk_exp" -gt "$(date +%s)" ]; then - printf '%s' "$_tk_val" + if [ -n "$_token_exchange_val" ] && [ "$_token_exchange_exp" -gt "$(date +%s)" ]; then + printf '%s' "$_token_exchange_val" return 0 fi fi - _tk_body=$(curl -sS -X POST \ + _token_exchange_body=$(curl -sS -X POST \ --connect-timeout "$NP_TRACE_CONNECT_TIMEOUT" --max-time "$NP_TRACE_MAX_TIME" \ -H 'Content-Type: application/json' \ -d "$(np__json_obj apiKey "$NP_TRACE_API_KEY")" \ - "${NP_TRACE_AUTH_URL:-$NP_TRACE_DEFAULT_AUTH_URL}/token" 2>/dev/null) || _tk_body='' + "${NP_TRACE_AUTH_URL:-$NP_TRACE_DEFAULT_AUTH_URL}/token" 2>/dev/null) || _token_exchange_body='' - _tk_new=$(printf '%s' "$_tk_body" | + _token_exchange_new=$(printf '%s' "$_token_exchange_body" | sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p') - if [ -z "$_tk_new" ]; then + if [ -z "$_token_exchange_new" ]; then np__drop 'auth' 'token exchange failed' printf '' return 0 fi - ( umask 077; printf '%s\n%s\n' "$(( $(date +%s) + 3540 ))" "$_tk_new" > "$_tk_cache" ) - printf '%s' "$_tk_new" + ( umask 077; printf '%s\n%s\n' "$(( $(date +%s) + 3540 ))" "$_token_exchange_new" > "$_token_exchange_cache" ) + printf '%s' "$_token_exchange_new" return 0 } @@ -612,27 +633,27 @@ np__token_exchange() { # straight into the build log. np__auth_config() { np__secret_begin - _ac_file="$NP_TRACE_DIR/curlcfg.$$" - ( umask 077; printf 'header = "Authorization: Bearer %s"\n' "$(np__token)" > "$_ac_file" ) + _auth_config_file="$NP_TRACE_DIR/curlcfg.$$" + ( umask 077; printf 'header = "Authorization: Bearer %s"\n' "$(np__token)" > "$_auth_config_file" ) np__secret_end - printf '%s' "$_ac_file" + printf '%s' "$_auth_config_file" return 0 } # POST one spool file. Prints the HTTP status code, or 000 on a network failure. np__post_event() { - _pe_cfg=$(np__auth_config) - _pe_code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ - --config "$_pe_cfg" \ + _post_event_cfg=$(np__auth_config) + _post_event_code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + --config "$_post_event_cfg" \ --connect-timeout "$NP_TRACE_CONNECT_TIMEOUT" --max-time "$NP_TRACE_MAX_TIME" \ -H 'Content-Type: application/json' \ --data-binary "@$1" \ - "${NP_TRACE_BASE_URL:-$NP_TRACE_DEFAULT_BASE_URL}/events" 2>/dev/null) || _pe_code='000' - rm -f "$_pe_cfg" 2>/dev/null || : - case "$_pe_code" in - '' | *[!0-9]*) _pe_code='000' ;; + "${NP_TRACE_BASE_URL:-$NP_TRACE_DEFAULT_BASE_URL}/events" 2>/dev/null) || _post_event_code='000' + rm -f "$_post_event_cfg" 2>/dev/null || : + case "$_post_event_code" in + '' | *[!0-9]*) _post_event_code='000' ;; esac - printf '%s' "$_pe_code" + printf '%s' "$_post_event_code" return 0 } @@ -644,11 +665,11 @@ NP_TRACE_FLUSH_TIMEOUT="${NP_TRACE_FLUSH_TIMEOUT:-10}" NP_TRACE_MAX_RETRIES="${NP_TRACE_MAX_RETRIES:-3}" np__attempts_of() { - _ao_n=$(cat "$1.attempts" 2>/dev/null || printf '0') - case "$_ao_n" in - '' | *[!0-9]*) _ao_n=0 ;; + _attempts_of_n=$(cat "$1.attempts" 2>/dev/null || printf '0') + case "$_attempts_of_n" in + '' | *[!0-9]*) _attempts_of_n=0 ;; esac - printf '%s' "$_ao_n" + printf '%s' "$_attempts_of_n" } np__fail_event() { @@ -662,37 +683,37 @@ np_trace_flush() { [ -n "${NP_TRACE_DIR:-}" ] || return 0 [ -d "$NP_TRACE_DIR/spool" ] || return 0 [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _fl_deadline=$(( $(date +%s) + NP_TRACE_FLUSH_TIMEOUT )) + _flush_deadline=$(( $(date +%s) + NP_TRACE_FLUSH_TIMEOUT )) - for _fl_file in "$NP_TRACE_DIR/spool"/*.json; do - [ -f "$_fl_file" ] || continue - if [ "$(date +%s)" -ge "$_fl_deadline" ]; then + for _flush_file in "$NP_TRACE_DIR/spool"/*.json; do + [ -f "$_flush_file" ] || continue + if [ "$(date +%s)" -ge "$_flush_deadline" ]; then # Budget spent. Remaining events stay on disk for the next flush or a # later np_trace_recover; the process exits on time regardless. This is # the guarantee that a dead API cannot hang a build. return 0 fi - _fl_code=$(np__post_event "$_fl_file") - case "$_fl_code" in + _flush_code=$(np__post_event "$_flush_file") + case "$_flush_code" in 201 | 200) # 200 is an idempotent re-POST of an already-accepted event. - rm -f "$_fl_file" "$_fl_file.attempts" 2>/dev/null || : + rm -f "$_flush_file" "$_flush_file.attempts" 2>/dev/null || : ;; 400) # A contract violation. Never retried — retrying cannot change it. - np__fail_event "$_fl_file" "rejected 400" + np__fail_event "$_flush_file" "rejected 400" ;; 401 | 403) rm -f "$NP_TRACE_DIR/token" 2>/dev/null || : - np__fail_event "$_fl_file" "unauthorized $_fl_code" + np__fail_event "$_flush_file" "unauthorized $_flush_code" ;; *) - _fl_n=$(( $(np__attempts_of "$_fl_file") + 1 )) - if [ "$_fl_n" -gt "$NP_TRACE_MAX_RETRIES" ]; then - np__fail_event "$_fl_file" "gave up after $_fl_n attempts (last status $_fl_code)" + _flush_n=$(( $(np__attempts_of "$_flush_file") + 1 )) + if [ "$_flush_n" -gt "$NP_TRACE_MAX_RETRIES" ]; then + np__fail_event "$_flush_file" "gave up after $_flush_n attempts (last status $_flush_code)" else - printf '%s' "$_fl_n" > "$_fl_file.attempts" 2>/dev/null || : + printf '%s' "$_flush_n" > "$_flush_file.attempts" 2>/dev/null || : fi ;; esac @@ -747,12 +768,12 @@ np__install_trap() { # so `NP_TRACE=$(np_trace_inject)` is always safe. np_trace_inject() { [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _ij_h=$(np__resolve_handle "${1:-}") - np__is_handle "$_ij_h" || return 0 + _inject_h=$(np__resolve_handle "${1:-}") + np__is_handle "$_inject_h" || return 0 printf '%s%s%s%s%s' \ "$NP_CARRIER_VERSION" "$NP_CARRIER_DELIMITER" \ - "$(np__node_get "$_ij_h" trace_id)" "$NP_CARRIER_DELIMITER" \ - "$(np__node_get "$_ij_h" run_id)" + "$(np__node_get "$_inject_h" trace_id)" "$NP_CARRIER_DELIMITER" \ + "$(np__node_get "$_inject_h" run_id)" return 0 } @@ -766,31 +787,31 @@ np_trace_inject() { # When only a trace id is present it is used for both, matching the Go SDK, so # the result is always a usable pair. np_trace_extract() { - _ex_raw=${1-${NP_TRACE:-}} - [ -n "$_ex_raw" ] || return 1 + _extract_raw=${1-${NP_TRACE:-}} + [ -n "$_extract_raw" ] || return 1 - case "$_ex_raw" in + case "$_extract_raw" in "$NP_CARRIER_VERSION$NP_CARRIER_DELIMITER"*) ;; *) return 1 ;; esac - _ex_rest=${_ex_raw#*"$NP_CARRIER_DELIMITER"} + _extract_rest=${_extract_raw#*"$NP_CARRIER_DELIMITER"} # trace_id is up to the next delimiter; run_id is the whole remainder, which # may itself contain '~' and '@' but never a delimiter. - case "$_ex_rest" in + case "$_extract_rest" in *"$NP_CARRIER_DELIMITER"*) - _ex_trace=${_ex_rest%%"$NP_CARRIER_DELIMITER"*} - _ex_run=${_ex_rest#*"$NP_CARRIER_DELIMITER"} + _extract_trace=${_extract_rest%%"$NP_CARRIER_DELIMITER"*} + _extract_run=${_extract_rest#*"$NP_CARRIER_DELIMITER"} ;; *) - _ex_trace=$_ex_rest - _ex_run=$_ex_rest + _extract_trace=$_extract_rest + _extract_run=$_extract_rest ;; esac - [ -n "$_ex_trace" ] || return 1 - [ -n "$_ex_run" ] || _ex_run=$_ex_trace + [ -n "$_extract_trace" ] || return 1 + [ -n "$_extract_run" ] || _extract_run=$_extract_trace - printf '%s %s' "$_ex_trace" "$_ex_run" + printf '%s %s' "$_extract_trace" "$_extract_run" return 0 } @@ -812,37 +833,37 @@ np_trace_extract() { # run instead. np_trace_adopt() { [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 1 - _ad_ctx=$(np_trace_extract "${1-${NP_TRACE:-}}") || return 1 - _ad_trace=${_ad_ctx%% *} - _ad_run=${_ad_ctx#* } + _adopt_ctx=$(np_trace_extract "${1-${NP_TRACE:-}}") || return 1 + _adopt_trace=${_adopt_ctx%% *} + _adopt_run=${_adopt_ctx#* } - if ! _ad_why=$(np__trace_id_violation "$_ad_trace"); then - np__drop 'adopt' "trace_id $_ad_why" + if ! _adopt_why=$(np__trace_id_violation "$_adopt_trace"); then + np__drop 'adopt' "trace_id $_adopt_why" return 1 fi # An upstream run_id is commonly a DERIVED path (parent~key@attempt.iteration) # rather than a named id — the np CLI hands us the step it is running. Accept # either: parse it as a node path first, and only fall back to the named-id # rules when it has no delimiter. - if ! np__parse_node_id "$_ad_run" >/dev/null 2>&1; then - if ! _ad_why=$(np__named_id_violation "$_ad_run"); then - np__drop 'adopt' "run_id $_ad_why" + if ! np__parse_node_id "$_adopt_run" >/dev/null 2>&1; then + if ! _adopt_why=$(np__named_id_violation "$_adopt_run"); then + np__drop 'adopt' "run_id $_adopt_why" return 1 fi fi - _ad_h=$(np__handle_new) - np__node_set "$_ad_h" kind run - np__node_set "$_ad_h" trace_id "$_ad_trace" - np__node_set "$_ad_h" run_id "$_ad_run" - np__node_set "$_ad_h" nrn "${NP_TRACE_NRN:-}" - np__node_set "$_ad_h" foreign 1 + _adopt_h=$(np__handle_new) + np__node_set "$_adopt_h" kind run + np__node_set "$_adopt_h" trace_id "$_adopt_trace" + np__node_set "$_adopt_h" run_id "$_adopt_run" + np__node_set "$_adopt_h" nrn "${NP_TRACE_NRN:-}" + np__node_set "$_adopt_h" foreign 1 # started=1 suppresses the lazy `started` emit; closed=0 keeps it usable as a # parent for the whole script. - np__node_set "$_ad_h" started 1 - np__node_set "$_ad_h" closed 0 - np__ambient_set "$_ad_h" - printf '%s' "$_ad_h" + np__node_set "$_adopt_h" started 1 + np__node_set "$_adopt_h" closed 0 + np__ambient_set "$_adopt_h" + printf '%s' "$_adopt_h" return 0 } @@ -888,37 +909,37 @@ np_trace_init() { # Emit the node event for a handle at the given status, carrying whatever # context is currently staged. np__emit_node() { - _en_h=$1 - _en_status=$2 + _emit_node_h=$1 + _emit_node_status=$2 [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _en_labels=$(np__node_get "$_en_h" labels) - _en_facets=$(np__node_get "$_en_h" facets) - _en_key=$(np__node_get "$_en_h" key) - _en_schema=$(np__node_get "$_en_h" schema_url) - - if [ -n "$_en_key" ]; then - _en_data=$(np__json_obj_raw \ - trace_id "$(np__json_str "$(np__node_get "$_en_h" trace_id)")" \ - run_id "$(np__json_str "$(np__node_get "$_en_h" run_id)")" \ - key "$(np__json_str "$_en_key")" \ - attempt "$(np__node_get "$_en_h" attempt)" \ - iteration "$(np__node_get "$_en_h" iteration)" \ - status "$(np__json_str "$_en_status")" \ - labels "$_en_labels" \ - facets "$_en_facets" \ - schema_url "$(if [ -n "$_en_schema" ]; then np__json_str "$_en_schema"; fi)") + _emit_node_labels=$(np__node_get "$_emit_node_h" labels) + _emit_node_facets=$(np__node_get "$_emit_node_h" facets) + _emit_node_key=$(np__node_get "$_emit_node_h" key) + _emit_node_schema=$(np__node_get "$_emit_node_h" schema_url) + + if [ -n "$_emit_node_key" ]; then + _emit_node_data=$(np__json_obj_raw \ + trace_id "$(np__json_str "$(np__node_get "$_emit_node_h" trace_id)")" \ + run_id "$(np__json_str "$(np__node_get "$_emit_node_h" run_id)")" \ + key "$(np__json_str "$_emit_node_key")" \ + attempt "$(np__node_get "$_emit_node_h" attempt)" \ + iteration "$(np__node_get "$_emit_node_h" iteration)" \ + status "$(np__json_str "$_emit_node_status")" \ + labels "$_emit_node_labels" \ + facets "$_emit_node_facets" \ + schema_url "$(if [ -n "$_emit_node_schema" ]; then np__json_str "$_emit_node_schema"; fi)") else - _en_data=$(np__json_obj_raw \ - trace_id "$(np__json_str "$(np__node_get "$_en_h" trace_id)")" \ - run_id "$(np__json_str "$(np__node_get "$_en_h" run_id)")" \ - status "$(np__json_str "$_en_status")" \ - labels "$_en_labels" \ - facets "$_en_facets" \ - schema_url "$(if [ -n "$_en_schema" ]; then np__json_str "$_en_schema"; fi)") + _emit_node_data=$(np__json_obj_raw \ + trace_id "$(np__json_str "$(np__node_get "$_emit_node_h" trace_id)")" \ + run_id "$(np__json_str "$(np__node_get "$_emit_node_h" run_id)")" \ + status "$(np__json_str "$_emit_node_status")" \ + labels "$_emit_node_labels" \ + facets "$_emit_node_facets" \ + schema_url "$(if [ -n "$_emit_node_schema" ]; then np__json_str "$_emit_node_schema"; fi)") fi - np__spool "$NP_TYPE_NODE_RUN" "$(np__node_get "$_en_h" nrn)" "$_en_data" >/dev/null + np__spool "$NP_TYPE_NODE_RUN" "$(np__node_get "$_emit_node_h" nrn)" "$_emit_node_data" >/dev/null return 0 } @@ -932,8 +953,8 @@ np__ref_of() { np__emit_parent_edge() { [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _pe_data=$(np__json_obj_raw from "$(np__ref_of "$1")" to "$(np__ref_of "$2")") - np__spool "$NP_TYPE_EDGE_PARENT" "$(np__node_get "$1" nrn)" "$_pe_data" >/dev/null + _emit_parent_edge_data=$(np__json_obj_raw from "$(np__ref_of "$1")" to "$(np__ref_of "$2")") + np__spool "$NP_TYPE_EDGE_PARENT" "$(np__node_get "$1" nrn)" "$_emit_parent_edge_data" >/dev/null return 0 } @@ -944,13 +965,13 @@ np__emit_parent_edge() { # staged before that lands on `started`; context staged after lands on the # terminal. Same observable semantics as the JS and Go SDKs, without a timer. np_trace_start() { - _st_h=$(np__resolve_handle "${1:-}") - np__is_handle "$_st_h" || return 0 - if [ "$(np__node_get "$_st_h" started)" = '1' ]; then + _start_h=$(np__resolve_handle "${1:-}") + np__is_handle "$_start_h" || return 0 + if [ "$(np__node_get "$_start_h" started)" = '1' ]; then return 0 fi - np__node_set "$_st_h" started 1 - np__emit_node "$_st_h" "$NP_STATUS_STARTED" + np__node_set "$_start_h" started 1 + np__emit_node "$_start_h" "$NP_STATUS_STARTED" return 0 } @@ -960,135 +981,135 @@ np_trace_start() { np_trace_run() { [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _rn_trace='' - _rn_run='' - _rn_nrn="${NP_TRACE_NRN:-}" + _run_trace='' + _run_run='' + _run_nrn="${NP_TRACE_NRN:-}" while [ "$#" -gt 0 ]; do case "$1" in - --trace-id) _rn_trace=${2:-}; shift 2 ;; - --run-id) _rn_run=${2:-}; shift 2 ;; - --nrn) _rn_nrn=${2:-}; shift 2 ;; + --trace-id) _run_trace=${2:-}; shift 2 ;; + --run-id) _run_run=${2:-}; shift 2 ;; + --nrn) _run_nrn=${2:-}; shift 2 ;; *) shift ;; esac done # A lone root run's trace_id defaults to its run_id, and vice versa. - [ -n "$_rn_trace" ] || _rn_trace=$_rn_run - [ -n "$_rn_run" ] || _rn_run=$_rn_trace + [ -n "$_run_trace" ] || _run_trace=$_run_run + [ -n "$_run_run" ] || _run_run=$_run_trace - if ! _rn_why=$(np__trace_id_violation "$_rn_trace"); then - np__drop 'run' "trace_id $_rn_why" + if ! _run_why=$(np__trace_id_violation "$_run_trace"); then + np__drop 'run' "trace_id $_run_why" return 0 fi - if ! _rn_why=$(np__named_id_violation "$_rn_run"); then - np__drop 'run' "run_id $_rn_why" + if ! _run_why=$(np__named_id_violation "$_run_run"); then + np__drop 'run' "run_id $_run_why" return 0 fi - _rn_h=$(np__handle_new) - np__node_set "$_rn_h" kind run - np__node_set "$_rn_h" trace_id "$_rn_trace" - np__node_set "$_rn_h" run_id "$_rn_run" - np__node_set "$_rn_h" nrn "$_rn_nrn" - np__node_set "$_rn_h" auto_started_at "$(np__iso8601)" - np__node_set "$_rn_h" started 0 - np__node_set "$_rn_h" closed 0 - np__ambient_set "$_rn_h" - printf '%s' "$_rn_h" + _run_h=$(np__handle_new) + np__node_set "$_run_h" kind run + np__node_set "$_run_h" trace_id "$_run_trace" + np__node_set "$_run_h" run_id "$_run_run" + np__node_set "$_run_h" nrn "$_run_nrn" + np__node_set "$_run_h" auto_started_at "$(np__iso8601)" + np__node_set "$_run_h" started 0 + np__node_set "$_run_h" closed 0 + np__ambient_set "$_run_h" + printf '%s' "$_run_h" return 0 } # np_trace_step [handle] [--attempt N] [--iteration N] np_trace_step() { [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _sp_parent=$(np__resolve_handle "${1:-}") + _step_parent=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - _sp_key=${1:-} + _step_key=${1:-} if [ "$#" -gt 0 ]; then shift fi - _sp_attempt=0 - _sp_iteration=0 + _step_attempt=0 + _step_iteration=0 while [ "$#" -gt 0 ]; do case "$1" in - --attempt) _sp_attempt=${2:-0}; shift 2 ;; - --iteration) _sp_iteration=${2:-0}; shift 2 ;; + --attempt) _step_attempt=${2:-0}; shift 2 ;; + --iteration) _step_iteration=${2:-0}; shift 2 ;; *) shift ;; esac done - if ! np__is_handle "$_sp_parent"; then + if ! np__is_handle "$_step_parent"; then np__drop 'step' 'no parent node in scope' return 0 fi - if ! _sp_why=$(np__key_violation "$_sp_key"); then - np__drop 'step' "key $_sp_why" + if ! _step_why=$(np__key_violation "$_step_key"); then + np__drop 'step' "key $_step_why" return 0 fi - case "$_sp_attempt$_sp_iteration" in + case "$_step_attempt$_step_iteration" in '' | *[!0-9]*) np__drop 'step' 'attempt and iteration must be integers'; return 0 ;; esac # Opening a child forces the parent's started: a parent edge must not point # at a node the read model has never seen. - np_trace_start "$_sp_parent" - - _sp_id=$(np__derive_child_id "$(np__node_get "$_sp_parent" run_id)" \ - "$_sp_key" "$_sp_attempt" "$_sp_iteration") - - _sp_h=$(np__handle_new) - np__node_set "$_sp_h" kind step - np__node_set "$_sp_h" trace_id "$(np__node_get "$_sp_parent" trace_id)" - np__node_set "$_sp_h" run_id "$_sp_id" - np__node_set "$_sp_h" nrn "$(np__node_get "$_sp_parent" nrn)" - np__node_set "$_sp_h" key "$_sp_key" - np__node_set "$_sp_h" attempt "$_sp_attempt" - np__node_set "$_sp_h" iteration "$_sp_iteration" - np__node_set "$_sp_h" parent "$_sp_parent" - np__node_set "$_sp_h" auto_started_at "$(np__iso8601)" - np__node_set "$_sp_h" started 0 - np__node_set "$_sp_h" closed 0 - - np_trace_start "$_sp_h" - np__emit_parent_edge "$_sp_parent" "$_sp_h" - np__ambient_set "$_sp_h" - printf '%s' "$_sp_h" + np_trace_start "$_step_parent" + + _step_id=$(np__derive_child_id "$(np__node_get "$_step_parent" run_id)" \ + "$_step_key" "$_step_attempt" "$_step_iteration") + + _step_h=$(np__handle_new) + np__node_set "$_step_h" kind step + np__node_set "$_step_h" trace_id "$(np__node_get "$_step_parent" trace_id)" + np__node_set "$_step_h" run_id "$_step_id" + np__node_set "$_step_h" nrn "$(np__node_get "$_step_parent" nrn)" + np__node_set "$_step_h" key "$_step_key" + np__node_set "$_step_h" attempt "$_step_attempt" + np__node_set "$_step_h" iteration "$_step_iteration" + np__node_set "$_step_h" parent "$_step_parent" + np__node_set "$_step_h" auto_started_at "$(np__iso8601)" + np__node_set "$_step_h" started 0 + np__node_set "$_step_h" closed 0 + + np_trace_start "$_step_h" + np__emit_parent_edge "$_step_parent" "$_step_h" + np__ambient_set "$_step_h" + printf '%s' "$_step_h" return 0 } # A named child run — a new scope under the same trace. np_trace_child() { [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _ch_parent=$(np__resolve_handle "${1:-}") + _child_parent=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - _ch_run='' + _child_run='' while [ "$#" -gt 0 ]; do case "$1" in - --run-id) _ch_run=${2:-}; shift 2 ;; + --run-id) _child_run=${2:-}; shift 2 ;; *) shift ;; esac done - if ! np__is_handle "$_ch_parent"; then + if ! np__is_handle "$_child_parent"; then np__drop 'child' 'no parent node in scope' return 0 fi - if ! _ch_why=$(np__named_id_violation "$_ch_run"); then - np__drop 'child' "run_id $_ch_why" + if ! _child_why=$(np__named_id_violation "$_child_run"); then + np__drop 'child' "run_id $_child_why" return 0 fi - np_trace_start "$_ch_parent" - _ch_h=$(np_trace_run --trace-id "$(np__node_get "$_ch_parent" trace_id)" \ - --run-id "$_ch_run" \ - --nrn "$(np__node_get "$_ch_parent" nrn)") - np__is_handle "$_ch_h" || return 0 - np__node_set "$_ch_h" parent "$_ch_parent" - np_trace_start "$_ch_h" - np__emit_parent_edge "$_ch_parent" "$_ch_h" - np__ambient_set "$_ch_h" - printf '%s' "$_ch_h" + np_trace_start "$_child_parent" + _child_h=$(np_trace_run --trace-id "$(np__node_get "$_child_parent" trace_id)" \ + --run-id "$_child_run" \ + --nrn "$(np__node_get "$_child_parent" nrn)") + np__is_handle "$_child_h" || return 0 + np__node_set "$_child_h" parent "$_child_parent" + np_trace_start "$_child_h" + np__emit_parent_edge "$_child_parent" "$_child_h" + np__ambient_set "$_child_h" + printf '%s' "$_child_h" return 0 } @@ -1098,23 +1119,23 @@ np_trace_child() { # Merge a pre-formed `"key":value` fragment into the node's staged labels. np__stage_label() { - _sl_cur=$(np__node_get "$1" labels) - if [ -z "$_sl_cur" ] || [ "$_sl_cur" = '{}' ]; then + _stage_label_cur=$(np__node_get "$1" labels) + if [ -z "$_stage_label_cur" ] || [ "$_stage_label_cur" = '{}' ]; then np__node_set "$1" labels "{$2}" else - np__node_set "$1" labels "${_sl_cur%\}},$2}" + np__node_set "$1" labels "${_stage_label_cur%\}},$2}" fi return 0 } np__stage_facet() { - _sf_cur=$(np__node_get "$1" facets) - _sf_entry="$(np__json_str "$2"):$3" - if [ -z "$_sf_cur" ] || [ "$_sf_cur" = '{}' ]; then - np__node_set "$1" facets "{$_sf_entry}" + _stage_facet_cur=$(np__node_get "$1" facets) + _stage_facet_entry="$(np__json_str "$2"):$3" + if [ -z "$_stage_facet_cur" ] || [ "$_stage_facet_cur" = '{}' ]; then + np__node_set "$1" facets "{$_stage_facet_entry}" else # Last write wins per namespace: drop any prior entry for this facet. - np__node_set "$1" facets "${_sf_cur%\}},$_sf_entry}" + np__node_set "$1" facets "${_stage_facet_cur%\}},$_stage_facet_entry}" fi return 0 } @@ -1133,137 +1154,137 @@ np__flush_foreign() { # np_trace_labels [handle] key=value ... np_trace_labels() { - _lb_h=$(np__resolve_handle "${1:-}") + _labels_h=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_lb_h" || return 0 - for _lb_pair in "$@"; do - case "$_lb_pair" in + np__is_handle "$_labels_h" || return 0 + for _labels_pair in "$@"; do + case "$_labels_pair" in *=*) ;; *) continue ;; esac - _lb_k=${_lb_pair%%=*} - _lb_v=${_lb_pair#*=} + _labels_k=${_labels_pair%%=*} + _labels_v=${_labels_pair#*=} # An absent optional is omitted, never recorded as the string "null". - if [ -n "$_lb_k" ] && [ -n "$_lb_v" ]; then - np__stage_label "$_lb_h" "$(np__json_str "$_lb_k"):$(np__json_str "$_lb_v")" + if [ -n "$_labels_k" ] && [ -n "$_labels_v" ]; then + np__stage_label "$_labels_h" "$(np__json_str "$_labels_k"):$(np__json_str "$_labels_v")" fi done - np__flush_foreign "$_lb_h" + np__flush_foreign "$_labels_h" return 0 } # np_trace_facet [handle] — your own namespace. np_trace_facet() { - _fc_h=$(np__resolve_handle "${1:-}") + _facet_h=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_fc_h" || return 0 + np__is_handle "$_facet_h" || return 0 if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then return 0 fi - np__stage_facet "$_fc_h" "$1" "$2" - np__flush_foreign "$_fc_h" + np__stage_facet "$_facet_h" "$1" "$2" + np__flush_foreign "$_facet_h" return 0 } np_trace_schema() { - _sc_h=$(np__resolve_handle "${1:-}") + _schema_h=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_sc_h" || return 0 - np__node_set "$_sc_h" schema_url "${1:-}" + np__is_handle "$_schema_h" || return 0 + np__node_set "$_schema_h" schema_url "${1:-}" return 0 } np_trace_explain() { - _ex_h=$(np__resolve_handle "${1:-}") + _explain_h=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_ex_h" || return 0 - _ex_title='' - _ex_what='' - _ex_why='' - _ex_impact='' - _ex_next='' - _ex_sev='' + np__is_handle "$_explain_h" || return 0 + _explain_title='' + _explain_what='' + _explain_why='' + _explain_impact='' + _explain_next='' + _explain_sev='' while [ "$#" -gt 0 ]; do case "$1" in - --title) _ex_title=${2:-}; shift 2 ;; - --what) _ex_what=${2:-}; shift 2 ;; - --why) _ex_why=${2:-}; shift 2 ;; - --impact) _ex_impact=${2:-}; shift 2 ;; - --next) _ex_next=${2:-}; shift 2 ;; - --severity) _ex_sev=${2:-}; shift 2 ;; + --title) _explain_title=${2:-}; shift 2 ;; + --what) _explain_what=${2:-}; shift 2 ;; + --why) _explain_why=${2:-}; shift 2 ;; + --impact) _explain_impact=${2:-}; shift 2 ;; + --next) _explain_next=${2:-}; shift 2 ;; + --severity) _explain_sev=${2:-}; shift 2 ;; *) shift ;; esac done - if [ -z "$_ex_title" ]; then + if [ -z "$_explain_title" ]; then np__drop 'explain' 'title is required' return 0 fi - np__stage_facet "$_ex_h" "$NP_FACET_EXPLAIN" \ - "$(np__json_obj title "$_ex_title" severity "$_ex_sev" what "$_ex_what" \ - why "$_ex_why" impact "$_ex_impact" next "$_ex_next")" - np__flush_foreign "$_ex_h" + np__stage_facet "$_explain_h" "$NP_FACET_EXPLAIN" \ + "$(np__json_obj title "$_explain_title" severity "$_explain_sev" what "$_explain_what" \ + why "$_explain_why" impact "$_explain_impact" next "$_explain_next")" + np__flush_foreign "$_explain_h" return 0 } np_trace_error() { - _er_h=$(np__resolve_handle "${1:-}") + _error_h=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_er_h" || return 0 - _er_msg='' - _er_code='' - _er_stack='' - _er_details='' + np__is_handle "$_error_h" || return 0 + _error_msg='' + _error_code='' + _error_stack='' + _error_details='' while [ "$#" -gt 0 ]; do case "$1" in - --message) _er_msg=${2:-}; shift 2 ;; - --code) _er_code=${2:-}; shift 2 ;; - --stack-trace) _er_stack=${2:-}; shift 2 ;; + --message) _error_msg=${2:-}; shift 2 ;; + --code) _error_code=${2:-}; shift 2 ;; + --stack-trace) _error_stack=${2:-}; shift 2 ;; # A JSON object with the diagnosis's structured evidence (counts, the # failing probe, ...) — the sibling SDKs' error `details`. - --details) _er_details=${2:-}; shift 2 ;; + --details) _error_details=${2:-}; shift 2 ;; *) - if [ -z "$_er_msg" ]; then - _er_msg=$1 + if [ -z "$_error_msg" ]; then + _error_msg=$1 fi shift ;; esac done - [ -n "$_er_msg" ] || return 0 - case "$_er_details" in + [ -n "$_error_msg" ] || return 0 + case "$_error_details" in '' | \{*) ;; - *) _er_details='' ;; + *) _error_details='' ;; esac - np__stage_facet "$_er_h" "$NP_FACET_ERROR" \ + np__stage_facet "$_error_h" "$NP_FACET_ERROR" \ "$(np__json_obj_raw \ - message "$(np__json_str "$_er_msg")" \ - code "$(if [ -n "$_er_code" ]; then np__json_str "$_er_code"; fi)" \ - stack_trace "$(if [ -n "$_er_stack" ]; then np__json_str "$_er_stack"; fi)" \ - details "$_er_details")" - np__flush_foreign "$_er_h" + message "$(np__json_str "$_error_msg")" \ + code "$(if [ -n "$_error_code" ]; then np__json_str "$_error_code"; fi)" \ + stack_trace "$(if [ -n "$_error_stack" ]; then np__json_str "$_error_stack"; fi)" \ + details "$_error_details")" + np__flush_foreign "$_error_h" return 0 } np_trace_timing() { - _tm_h=$(np__resolve_handle "${1:-}") + _timing_h=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_tm_h" || return 0 + np__is_handle "$_timing_h" || return 0 while [ "$#" -gt 0 ]; do case "$1" in - --started-at) np__node_set "$_tm_h" started_at "${2:-}"; shift 2 ;; - --ended-at) np__node_set "$_tm_h" ended_at "${2:-}"; shift 2 ;; + --started-at) np__node_set "$_timing_h" started_at "${2:-}"; shift 2 ;; + --ended-at) np__node_set "$_timing_h" ended_at "${2:-}"; shift 2 ;; *) shift ;; esac done @@ -1272,12 +1293,12 @@ np_trace_timing() { # Stamp the auto timing facet, letting any manual override win per field. np__stage_timing() { - _sg_started=$(np__node_get "$1" started_at) - _sg_ended=$(np__node_get "$1" ended_at) - [ -n "$_sg_started" ] || _sg_started=$(np__node_get "$1" auto_started_at) - [ -n "$_sg_ended" ] || _sg_ended=$2 + _stage_timing_started=$(np__node_get "$1" started_at) + _stage_timing_ended=$(np__node_get "$1" ended_at) + [ -n "$_stage_timing_started" ] || _stage_timing_started=$(np__node_get "$1" auto_started_at) + [ -n "$_stage_timing_ended" ] || _stage_timing_ended=$2 np__stage_facet "$1" "$NP_FACET_TIMING" \ - "$(np__json_obj started_at "$_sg_started" ended_at "$_sg_ended")" + "$(np__json_obj started_at "$_stage_timing_started" ended_at "$_stage_timing_ended")" return 0 } @@ -1298,124 +1319,175 @@ np__dataset_ref() { # grows. $1 handle, $2 facet namespace, $3 descriptor store key, $4 the # already-formed descriptor JSON. np__append_io_descriptor() { - _ai_descriptors=$(np__node_get "$1" "$3") - if [ -n "$_ai_descriptors" ]; then - _ai_descriptors="$_ai_descriptors,$4" + _append_io_descriptor_descriptors=$(np__node_get "$1" "$3") + if [ -n "$_append_io_descriptor_descriptors" ]; then + _append_io_descriptor_descriptors="$_append_io_descriptor_descriptors,$4" else - _ai_descriptors=$4 + _append_io_descriptor_descriptors=$4 fi - np__node_set "$1" "$3" "$_ai_descriptors" - np__stage_facet "$1" "$2" "[$_ai_descriptors]" + np__node_set "$1" "$3" "$_append_io_descriptor_descriptors" + np__stage_facet "$1" "$2" "[$_append_io_descriptor_descriptors]" return 0 } -# The shared body of np_trace_output / np_trace_input: an INLINE io -# descriptor — a small value carried in the event itself, the sibling SDKs' -# step.output(name, value). $1 direction (out|in), $2 verb name for drop -# records, then the caller's argv: [handle] . -np__inline_io() { - _ii_direction=$1 - _ii_verb=$2 +# Build one io descriptor from its parsed parts, choosing the kind by which +# parts are present: a uri is a POINTER (large data referenced, not inlined), +# a source+external-id is a REF (an entity in an external catalog), a JSON +# value is INLINE (carried in the event itself). Prints the descriptor, or +# nothing (with a drop) when the parts don't form one. +# $1 verb (for drop records), $2 name, $3 inline JSON, $4 uri, $5 ref source, +# $6 ref external id, $7 ref version. +np__build_io_descriptor() { + _build_io_descriptor_verb=$1 + _build_io_descriptor_name=$2 + _build_io_descriptor_inline=$3 + _build_io_descriptor_uri=$4 + _build_io_descriptor_ref_source=$5 + _build_io_descriptor_ref_id=$6 + _build_io_descriptor_ref_version=$7 + if [ -z "$_build_io_descriptor_name" ]; then + np__drop "$_build_io_descriptor_verb" 'a descriptor name is required' + return 1 + fi + if [ -n "$_build_io_descriptor_uri" ]; then + np__json_obj kind pointer name "$_build_io_descriptor_name" uri "$_build_io_descriptor_uri" + return 0 + fi + if [ -n "$_build_io_descriptor_ref_source" ] && [ -n "$_build_io_descriptor_ref_id" ]; then + np__json_obj kind ref name "$_build_io_descriptor_name" source "$_build_io_descriptor_ref_source" \ + external_id "$_build_io_descriptor_ref_id" version "$_build_io_descriptor_ref_version" + return 0 + fi + if [ -n "$_build_io_descriptor_inline" ]; then + case "$_build_io_descriptor_inline" in + \{* | \[* | \"* | [0-9-]* | true | false | null) + np__json_obj_raw kind '"inline"' name "$(np__json_str "$_build_io_descriptor_name")" value "$_build_io_descriptor_inline" + return 0 + ;; + esac + np__drop "$_build_io_descriptor_verb" 'value must be JSON' + return 1 + fi + np__drop "$_build_io_descriptor_verb" 'a JSON value, --uri, or --source + --external-id is required' + return 1 +} + +# The shared body of np_trace_output / np_trace_input. +# $1 direction (out|in), $2 verb, then the caller's argv: +# [handle] [] [--uri U] [--source S --external-id E [--version V]] +np__declare_io() { + _declare_io_direction=$1 + _declare_io_verb=$2 shift 2 - _ii_handle=$(np__resolve_handle "${1:-}") + _declare_io_handle=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_ii_handle" || { np__drop "$_ii_verb" 'no node in scope'; return 0; } - _ii_name=${1:-} - _ii_value=${2:-} - if [ -z "$_ii_name" ] || [ -z "$_ii_value" ]; then - np__drop "$_ii_verb" 'name and a JSON value are required' - return 0 + np__is_handle "$_declare_io_handle" || { np__drop "$_declare_io_verb" 'no node in scope'; return 0; } + _declare_io_name=${1:-} + if [ "$#" -gt 0 ]; then + shift fi - case "$_ii_value" in - \{* | \[* | \"* | [0-9-]* | true | false | null) ;; - *) np__drop "$_ii_verb" 'value must be JSON'; return 0 ;; - esac - if [ "$_ii_direction" = 'out' ]; then - _ii_facet_namespace=$NP_FACET_OUTPUT - _ii_store=io_output + _declare_io_inline='' + _declare_io_uri='' + _declare_io_ref_source='' + _declare_io_ref_id='' + _declare_io_ref_version='' + while [ "$#" -gt 0 ]; do + case "$1" in + --uri) _declare_io_uri=${2:-}; shift 2 ;; + --source) _declare_io_ref_source=${2:-}; shift 2 ;; + --external-id) _declare_io_ref_id=${2:-}; shift 2 ;; + --version) _declare_io_ref_version=${2:-}; shift 2 ;; + *) + if [ -z "$_declare_io_inline" ]; then + _declare_io_inline=$1 + fi + shift + ;; + esac + done + _declare_io_descriptor=$(np__build_io_descriptor "$_declare_io_verb" "$_declare_io_name" "$_declare_io_inline" \ + "$_declare_io_uri" "$_declare_io_ref_source" "$_declare_io_ref_id" "$_declare_io_ref_version") || return 0 + if [ "$_declare_io_direction" = 'out' ]; then + np__append_io_descriptor "$_declare_io_handle" "$NP_FACET_OUTPUT" io_output "$_declare_io_descriptor" else - _ii_facet_namespace=$NP_FACET_INPUT - _ii_store=io_input + np__append_io_descriptor "$_declare_io_handle" "$NP_FACET_INPUT" io_input "$_declare_io_descriptor" fi - _ii_descriptor=$(np__json_obj_raw kind '"inline"' name "$(np__json_str "$_ii_name")" value "$_ii_value") - np__append_io_descriptor "$_ii_handle" "$_ii_facet_namespace" "$_ii_store" "$_ii_descriptor" - np__flush_foreign "$_ii_handle" + np__flush_foreign "$_declare_io_handle" return 0 } -# np_trace_output [handle] +# np_trace_output [handle] [] [--uri U] [--source S --external-id E [--version V]] # -# Record what this node PRODUCED as an inline value carried in the event — -# `np_trace_output instances '{"healthy":2,"desired":3}'`. For an artifact -# with an address, prefer np_trace_produces (pointer + lineage edge). +# Record what this node PRODUCED: an inline value carried in the event +# (`np_trace_output instances '{"healthy":2}'`), a pointer to large data +# (`--uri`), or a ref to an external catalog entity (`--source`/`--external-id`). +# For an artifact that should ALSO join the lineage graph, prefer +# np_trace_produces (descriptor + edge in one call). np_trace_output() { [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - np__inline_io out output "$@" + np__declare_io out output "$@" return 0 } -# np_trace_input [handle] +# np_trace_input [handle] [] [--uri U] [--source S --external-id E [--version V]] # -# Record what this node CONSUMED as an inline value; see np_trace_output. +# Record what this node CONSUMED; see np_trace_output. np_trace_input() { [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - np__inline_io in input "$@" + np__declare_io in input "$@" return 0 } -# np__emit_io_edge [pointer-name] [pointer-uri] +# np__emit_io_edge [descriptor-json] # # Emit one lineage edge. The direction decides everything else: `out` is # edge.produces + tracing.output, `in` is edge.consumes + tracing.input. # -# With a pointer (name + uri) the io is declared ONCE: the descriptor -# accumulates into the node's io facet AND becomes the edge's tracing.binding -# — the same single-source rule as the sibling SDKs. Without one, the edge -# records lineage only. +# With a descriptor the io is declared ONCE: it accumulates into the node's +# io facet AND becomes the edge's tracing.binding — the same single-source +# rule as the sibling SDKs. Without one, the edge records lineage only. # # On a FOREIGN (adopted) node this is an observed fact, exactly like # np_trace_error: the edge is ours to say, and the staged io facet reaches the # wire through the foreign re-emit. np__emit_io_edge() { - _io_handle=$1 - _io_direction=$2 - _io_dataset_id=$3 - _io_pointer_name=${4:-} - _io_pointer_uri=${5:-} - - if [ "$_io_direction" = 'out' ]; then - _io_edge_type=$NP_TYPE_EDGE_PRODUCES - _io_facet_namespace=$NP_FACET_OUTPUT - _io_descriptor_store=io_output + _emit_io_edge_handle=$1 + _emit_io_edge_direction=$2 + _emit_io_edge_dataset_id=$3 + + if [ "$_emit_io_edge_direction" = 'out' ]; then + _emit_io_edge_edge_type=$NP_TYPE_EDGE_PRODUCES + _emit_io_edge_facet_namespace=$NP_FACET_OUTPUT + _emit_io_edge_descriptor_store=io_output else - _io_edge_type=$NP_TYPE_EDGE_CONSUMES - _io_facet_namespace=$NP_FACET_INPUT - _io_descriptor_store=io_input + _emit_io_edge_edge_type=$NP_TYPE_EDGE_CONSUMES + _emit_io_edge_facet_namespace=$NP_FACET_INPUT + _emit_io_edge_descriptor_store=io_input fi - _io_pointer='' - if [ -n "$_io_pointer_name" ] && [ -n "$_io_pointer_uri" ]; then - _io_pointer=$(np__json_obj kind pointer name "$_io_pointer_name" uri "$_io_pointer_uri") - np__append_io_descriptor "$_io_handle" "$_io_facet_namespace" "$_io_descriptor_store" "$_io_pointer" + _emit_io_edge_binding=$4 + + if [ -n "$_emit_io_edge_binding" ]; then + np__append_io_descriptor "$_emit_io_edge_handle" "$_emit_io_edge_facet_namespace" "$_emit_io_edge_descriptor_store" "$_emit_io_edge_binding" fi # An edge must not point FROM a node the read model has never seen. - np_trace_start "$_io_handle" + np_trace_start "$_emit_io_edge_handle" - if [ -n "$_io_pointer" ]; then - _io_edge_data=$(np__json_obj_raw \ - from "$(np__ref_of "$_io_handle")" \ - to "$(np__dataset_ref "$_io_dataset_id")" \ - facets "{$(np__json_str "$NP_FACET_BINDING"):$_io_pointer}") + if [ -n "$_emit_io_edge_binding" ]; then + _emit_io_edge_edge_data=$(np__json_obj_raw \ + from "$(np__ref_of "$_emit_io_edge_handle")" \ + to "$(np__dataset_ref "$_emit_io_edge_dataset_id")" \ + facets "{$(np__json_str "$NP_FACET_BINDING"):$_emit_io_edge_binding}") else - _io_edge_data=$(np__json_obj_raw \ - from "$(np__ref_of "$_io_handle")" \ - to "$(np__dataset_ref "$_io_dataset_id")") + _emit_io_edge_edge_data=$(np__json_obj_raw \ + from "$(np__ref_of "$_emit_io_edge_handle")" \ + to "$(np__dataset_ref "$_emit_io_edge_dataset_id")") fi - np__spool "$_io_edge_type" "$(np__node_get "$_io_handle" nrn)" "$_io_edge_data" >/dev/null - np__flush_foreign "$_io_handle" + np__spool "$_emit_io_edge_edge_type" "$(np__node_get "$_emit_io_edge_handle" nrn)" "$_emit_io_edge_edge_data" >/dev/null + np__flush_foreign "$_emit_io_edge_handle" return 0 } @@ -1423,56 +1495,565 @@ np__emit_io_edge() { # resolve the optional leading handle, take the dataset id, parse the # pointer flags, and hand off to np__emit_io_edge. # $1 direction (out|in), $2 verb name for drop records, then the caller's argv. -np__lineage_verb() { +np__declare_lineage() { [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _lv_direction=$1 - _lv_verb=$2 + _declare_lineage_direction=$1 + _declare_lineage_verb=$2 shift 2 - _lv_handle=$(np__resolve_handle "${1:-}") + _declare_lineage_handle=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_lv_handle" || { np__drop "$_lv_verb" 'no node in scope'; return 0; } + np__is_handle "$_declare_lineage_handle" || { np__drop "$_declare_lineage_verb" 'no node in scope'; return 0; } - _lv_dataset_id=${1:-} + _declare_lineage_dataset_id=${1:-} if [ "$#" -gt 0 ]; then shift fi - if [ -z "$_lv_dataset_id" ]; then - np__drop "$_lv_verb" 'dataset id is required' + if [ -z "$_declare_lineage_dataset_id" ]; then + np__drop "$_declare_lineage_verb" 'dataset id is required' return 0 fi - _lv_pointer_name='' - _lv_pointer_uri='' + _declare_lineage_name='' + _declare_lineage_inline='' + _declare_lineage_uri='' + _declare_lineage_ref_source='' + _declare_lineage_ref_id='' + _declare_lineage_ref_version='' while [ "$#" -gt 0 ]; do case "$1" in - --name) _lv_pointer_name=${2:-}; shift 2 ;; - --uri) _lv_pointer_uri=${2:-}; shift 2 ;; + --name) _declare_lineage_name=${2:-}; shift 2 ;; + --uri) _declare_lineage_uri=${2:-}; shift 2 ;; + --value) _declare_lineage_inline=${2:-}; shift 2 ;; + --source) _declare_lineage_ref_source=${2:-}; shift 2 ;; + --external-id) _declare_lineage_ref_id=${2:-}; shift 2 ;; + --version) _declare_lineage_ref_version=${2:-}; shift 2 ;; *) shift ;; esac done - np__emit_io_edge "$_lv_handle" "$_lv_direction" "$_lv_dataset_id" \ - "$_lv_pointer_name" "$_lv_pointer_uri" + _declare_lineage_binding='' + if [ -n "$_declare_lineage_name" ]; then + _declare_lineage_binding=$(np__build_io_descriptor "$_declare_lineage_verb" "$_declare_lineage_name" "$_declare_lineage_inline" \ + "$_declare_lineage_uri" "$_declare_lineage_ref_source" "$_declare_lineage_ref_id" "$_declare_lineage_ref_version") || return 0 + fi + + np__emit_io_edge "$_declare_lineage_handle" "$_declare_lineage_direction" "$_declare_lineage_dataset_id" "$_declare_lineage_binding" return 0 } -# np_trace_produces [handle] [--name --uri ] +# np_trace_produces [handle] [--name (--uri U | --value JSON | --source S --external-id E [--version V])] # -# Declare this node WROTE the dataset. `--name`/`--uri` record the io as a -# pointer descriptor (the artifact's address) on both the node and the edge. +# Declare this node WROTE the dataset. With `--name` the io is declared once +# — a pointer (`--uri`, the artifact's address), an inline value (`--value`), +# or a catalog ref (`--source`/`--external-id`) — on both the node and the +# edge's binding. Bare form records lineage only. np_trace_produces() { - np__lineage_verb out produces "$@" + np__declare_lineage out produces "$@" return 0 } -# np_trace_consumes [handle] [--name --uri ] +# np_trace_consumes [handle] [--name (--uri U | --value JSON | --source S --external-id E [--version V])] # # Declare this node READ the dataset; see np_trace_produces. np_trace_consumes() { - np__lineage_verb in consumes "$@" + np__declare_lineage in consumes "$@" + return 0 +} + +# --------------------------------------------------------------------------- +# Run-to-run edges — how operations relate across the graph +# --------------------------------------------------------------------------- + +# Resolve an edge target: a handle from this process, or a PACKED CARRIER +# ("1||") — the natural address in shell, where the other +# end of an edge usually arrived via an env var. Prints the target's ref. +np__edge_target_ref() { + if np__is_handle "$1"; then + np__ref_of "$1" + return 0 + fi + _edge_target_ref_context=$(np_trace_extract "$1") || return 1 + _edge_target_ref_trace=${_edge_target_ref_context%% *} + _edge_target_ref_run=${_edge_target_ref_context#* } + np__json_obj type run trace_id "$_edge_target_ref_trace" run_id "$_edge_target_ref_run" + return 0 +} + +# Emit one relationship edge from a node this process holds. +# $1 handle, $2 edge type, $3 target ref JSON, $4 verb for drop records. +np__emit_ref_edge() { + _emit_ref_edge_from=$(np__ref_of "$1") + if [ "$_emit_ref_edge_from" = "$3" ]; then + np__drop "$4" 'self-edge forbidden' + return 0 + fi + # An edge must not point FROM a node the read model has never seen. + np_trace_start "$1" + _emit_ref_edge_data=$(np__json_obj_raw from "$_emit_ref_edge_from" to "$3") + np__spool "$2" "$(np__node_get "$1" nrn)" "$_emit_ref_edge_data" >/dev/null + np__flush_foreign "$1" + return 0 +} + +# The shared argv handling of the run-to-run edge verbs. +# $1 edge type, $2 verb, then the caller's argv: [handle] . +np__declare_relation() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _declare_relation_type=$1 + _declare_relation_verb=$2 + shift 2 + _declare_relation_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_declare_relation_handle" || { np__drop "$_declare_relation_verb" 'no node in scope'; return 0; } + if [ -z "${1:-}" ]; then + np__drop "$_declare_relation_verb" 'a target (handle or packed carrier) is required' + return 0 + fi + _declare_relation_target=$(np__edge_target_ref "$1") || { + np__drop "$_declare_relation_verb" 'target is not a handle or a valid carrier' + return 0 + } + np__emit_ref_edge "$_declare_relation_handle" "$_declare_relation_type" "$_declare_relation_target" "$_declare_relation_verb" + return 0 +} + +# np_trace_triggered_by [handle] +# +# The operation that CAUSED this one — a cross-trace fact (the target is +# usually another trace's run, addressed by its packed carrier). +np_trace_triggered_by() { + np__declare_relation "$NP_TYPE_EDGE_TRIGGERED_BY" triggered_by "$@" + return 0 +} + +# np_trace_retry_of [handle] — this run retries that one. +np_trace_retry_of() { + np__declare_relation "$NP_TYPE_EDGE_RETRY_OF" retry_of "$@" + return 0 +} + +# np_trace_continues [handle] — this run resumes that one's work. +np_trace_continues() { + np__declare_relation "$NP_TYPE_EDGE_CONTINUES" continues "$@" + return 0 +} + +# np_trace_correlates [handle] — related, with no causal claim. +np_trace_correlates() { + np__declare_relation "$NP_TYPE_EDGE_CORRELATES" correlates "$@" + return 0 +} + +# np_trace_compensates [handle] — this run undoes that one's effect. +np_trace_compensates() { + np__declare_relation "$NP_TYPE_EDGE_COMPENSATES" compensates "$@" + return 0 +} + +# np_trace_link [handle] +# +# Escape hatch over the named verbs — emit any known edge type. Prefer the +# named functions when one fits. +np_trace_link() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _link_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_link_handle" || { np__drop 'link' 'no node in scope'; return 0; } + _link_type=${1:-} + case "$_link_type" in + "$NP_TYPE_EDGE_TRIGGERED_BY" | "$NP_TYPE_EDGE_RETRY_OF" | "$NP_TYPE_EDGE_CONTINUES" \ + | "$NP_TYPE_EDGE_CORRELATES" | "$NP_TYPE_EDGE_COMPENSATES" | "$NP_TYPE_EDGE_PARENT") ;; + *) np__drop 'link' "unknown edge type '${_link_type}'"; return 0 ;; + esac + if [ -z "${2:-}" ]; then + np__drop 'link' 'a target (handle or packed carrier) is required' + return 0 + fi + _link_target=$(np__edge_target_ref "$2") || { + np__drop 'link' 'target is not a handle or a valid carrier' + return 0 + } + np__emit_ref_edge "$_link_handle" "$_link_type" "$_link_target" link + return 0 +} + +# np_trace_instance_of [handle] [--nrn N] +# +# This run instantiates a reusable JOB definition — the read model resolves +# the run's plan from the definition. Emit the definition itself with +# np_trace_job. +np_trace_instance_of() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _instance_of_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_instance_of_handle" || { np__drop 'instance_of' 'no node in scope'; return 0; } + _instance_of_namespace=${1:-} + _instance_of_name=${2:-} + _instance_of_version=${3:-} + if [ "$#" -ge 3 ]; then + shift 3 + fi + _instance_of_nrn='' + while [ "$#" -gt 0 ]; do + case "$1" in + --nrn) _instance_of_nrn=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_instance_of_namespace" ] || [ -z "$_instance_of_name" ] || [ -z "$_instance_of_version" ]; then + np__drop 'instance_of' 'namespace, name and version are required' + return 0 + fi + _instance_of_target=$(np__json_obj type job namespace "$_instance_of_namespace" \ + name "$_instance_of_name" version "$_instance_of_version" nrn "$_instance_of_nrn") + np__emit_ref_edge "$_instance_of_handle" "$NP_TYPE_EDGE_INSTANCE_OF" "$_instance_of_target" instance_of + return 0 +} + +# --------------------------------------------------------------------------- +# Definition nodes — identities, not executions +# --------------------------------------------------------------------------- + +# np_trace_dataset [--nrn N] +# +# Emit a dataset node — an identity a lineage edge can point at. The id is +# the CANONICAL address (see np_trace_produces); edges to an unemitted +# dataset still resolve, so this is only needed to carry the node itself. +np_trace_dataset() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _dataset_id=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + _dataset_nrn='' + while [ "$#" -gt 0 ]; do + case "$1" in + --nrn) _dataset_nrn=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_dataset_id" ]; then + np__drop 'dataset' 'an id is required' + return 0 + fi + np__spool "$NP_TYPE_NODE_DATASET" "$_dataset_nrn" "$(np__json_obj id "$_dataset_id")" >/dev/null + return 0 +} + +# np_trace_job [--nrn N] [--plan JSON] +# +# Emit a job definition node — the reusable spec runs link instance_of, with +# its expected step plan (previewable before any run exists). +np_trace_job() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _job_namespace=${1:-} + _job_name=${2:-} + _job_version=${3:-} + if [ "$#" -ge 3 ]; then + shift 3 + fi + _job_nrn='' + _job_plan='' + while [ "$#" -gt 0 ]; do + case "$1" in + --nrn) _job_nrn=${2:-}; shift 2 ;; + --plan) _job_plan=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_job_namespace" ] || [ -z "$_job_name" ] || [ -z "$_job_version" ]; then + np__drop 'job' 'namespace, name and version are required' + return 0 + fi + case "$_job_plan" in + '' | \[*) ;; + *) np__drop 'job' 'the plan must be a JSON array of steps'; return 0 ;; + esac + if [ -n "$_job_plan" ]; then + _job_data=$(np__json_obj_raw \ + namespace "$(np__json_str "$_job_namespace")" \ + name "$(np__json_str "$_job_name")" \ + version "$(np__json_str "$_job_version")" \ + facets "{$(np__json_str "$NP_FACET_PLAN"):$_job_plan}") + else + _job_data=$(np__json_obj namespace "$_job_namespace" name "$_job_name" version "$_job_version") + fi + np__spool "$NP_TYPE_NODE_JOB" "$_job_nrn" "$_job_data" >/dev/null + return 0 +} + +# --------------------------------------------------------------------------- +# The remaining core-facet setters +# --------------------------------------------------------------------------- + +# np_trace_actor [handle] [--source S] +# +# WHO acted. The sibling SDKs also accept a bearer JWT and decode it; that +# sugar needs base64, which this SDK's runtime toolset excludes — pass the +# identity explicitly (the np CLI stamps the actor on workflow runs already). +np_trace_actor() { + _actor_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_actor_handle" || return 0 + _actor_kind=${1:-} + _actor_id=${2:-} + if [ "$#" -ge 2 ]; then + shift 2 + fi + _actor_source='' + while [ "$#" -gt 0 ]; do + case "$1" in + --source) _actor_source=${2:-}; shift 2 ;; + *) shift ;; + esac + done + case "$_actor_kind" in + user | service) ;; + *) np__drop 'actor' "kind must be user or service, got '${_actor_kind}'"; return 0 ;; + esac + if [ -z "$_actor_id" ]; then + np__drop 'actor' 'an id is required' + return 0 + fi + np__stage_facet "$_actor_handle" "$NP_FACET_ACTOR" \ + "$(np__json_obj kind "$_actor_kind" id "$_actor_id" source "$_actor_source")" + np__flush_foreign "$_actor_handle" + return 0 +} + +# np_trace_decision [handle] [--available a,b,c] [--expression E] +# +# The branch(es) this node chose, with the option set and the human-readable +# expression when known. +np_trace_decision() { + _decision_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_decision_handle" || return 0 + _decision_chosen=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + _decision_available='' + _decision_expression='' + while [ "$#" -gt 0 ]; do + case "$1" in + --available) _decision_available=${2:-}; shift 2 ;; + --expression) _decision_expression=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_decision_chosen" ]; then + np__drop 'decision' 'at least one chosen branch is required' + return 0 + fi + np__stage_facet "$_decision_handle" "$NP_FACET_DECISION" \ + "$(np__json_obj_raw \ + chosen "$(np__json_str_array_csv "$_decision_chosen")" \ + available "$(if [ -n "$_decision_available" ]; then np__json_str_array_csv "$_decision_available"; fi)" \ + expression "$(if [ -n "$_decision_expression" ]; then np__json_str "$_decision_expression"; fi)")" + np__flush_foreign "$_decision_handle" + return 0 +} + +# np_trace_retry [handle] [--next-attempt N] [--delay-ms MS] +np_trace_retry() { + _retry_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_retry_handle" || return 0 + _retry_attempt=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + _retry_next='' + _retry_delay='' + while [ "$#" -gt 0 ]; do + case "$1" in + --next-attempt) _retry_next=${2:-}; shift 2 ;; + --delay-ms) _retry_delay=${2:-}; shift 2 ;; + *) shift ;; + esac + done + case "$_retry_attempt$_retry_next$_retry_delay" in + '' | *[!0-9]*) np__drop 'retry' 'attempt, next-attempt and delay-ms must be non-negative integers'; return 0 ;; + esac + np__stage_facet "$_retry_handle" "$NP_FACET_RETRY" \ + "$(np__json_obj_raw attempt "$_retry_attempt" next_attempt "$_retry_next" delay_ms "$_retry_delay")" + np__flush_foreign "$_retry_handle" + return 0 +} + +# np_trace_signal [handle] [--timeout-ms MS] +np_trace_signal() { + _signal_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_signal_handle" || return 0 + _signal_name=${1:-} + _signal_direction=${2:-} + if [ "$#" -ge 2 ]; then + shift 2 + fi + _signal_timeout='' + while [ "$#" -gt 0 ]; do + case "$1" in + --timeout-ms) _signal_timeout=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_signal_name" ]; then + np__drop 'signal' 'a name is required' + return 0 + fi + case "$_signal_direction" in + wait | received) ;; + *) np__drop 'signal' "direction must be wait or received, got '${_signal_direction}'"; return 0 ;; + esac + case "$_signal_timeout" in + '' | *[!0-9]*) + if [ -n "$_signal_timeout" ]; then + np__drop 'signal' 'timeout-ms must be a non-negative integer' + return 0 + fi + ;; + esac + np__stage_facet "$_signal_handle" "$NP_FACET_SIGNAL" \ + "$(np__json_obj_raw \ + name "$(np__json_str "$_signal_name")" \ + direction "$(np__json_str "$_signal_direction")" \ + timeout_ms "$_signal_timeout")" + np__flush_foreign "$_signal_handle" + return 0 +} + +# np_trace_external_links [handle] [--label L] +# +# One off-platform link (a CI run, a dashboard). Accumulates: call once per +# link, the facet is the array of everything declared so far. +np_trace_external_links() { + _external_links_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_external_links_handle" || return 0 + _external_links_rel=${1:-} + _external_links_uri=${2:-} + if [ "$#" -ge 2 ]; then + shift 2 + fi + _external_links_label='' + while [ "$#" -gt 0 ]; do + case "$1" in + --label) _external_links_label=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_external_links_rel" ] || [ -z "$_external_links_uri" ]; then + np__drop 'external_links' 'rel and uri are required' + return 0 + fi + _external_links_link=$(np__json_obj rel "$_external_links_rel" uri "$_external_links_uri" label "$_external_links_label") + _external_links_links=$(np__node_get "$_external_links_handle" external_links) + if [ -n "$_external_links_links" ]; then + _external_links_links="$_external_links_links,$_external_links_link" + else + _external_links_links=$_external_links_link + fi + np__node_set "$_external_links_handle" external_links "$_external_links_links" + np__stage_facet "$_external_links_handle" "$NP_FACET_EXTERNAL_LINKS" "[$_external_links_links]" + np__flush_foreign "$_external_links_handle" + return 0 +} + +# np_trace_engine_status [handle] [--raw JSON] +# +# The underlying engine's own view of this node (a k8s rollout's status, a +# queue's verdict), verbatim. +np_trace_engine_status() { + _engine_status_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_engine_status_handle" || return 0 + _engine_status_engine=${1:-} + _engine_status_state=${2:-} + if [ "$#" -ge 2 ]; then + shift 2 + fi + _engine_status_raw='' + while [ "$#" -gt 0 ]; do + case "$1" in + --raw) _engine_status_raw=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_engine_status_engine" ] || [ -z "$_engine_status_state" ]; then + np__drop 'engine_status' 'engine and state are required' + return 0 + fi + case "$_engine_status_raw" in + '' | \{*) ;; + *) np__drop 'engine_status' 'raw must be a JSON object'; return 0 ;; + esac + np__stage_facet "$_engine_status_handle" "$NP_FACET_ENGINE_STATUS" \ + "$(np__json_obj_raw \ + engine "$(np__json_str "$_engine_status_engine")" \ + state "$(np__json_str "$_engine_status_state")" \ + raw "$_engine_status_raw")" + np__flush_foreign "$_engine_status_handle" + return 0 +} + +# np_trace_dropped [handle] +# +# A record of data intentionally dropped — pair with np_trace_skip. +np_trace_dropped() { + _dropped_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_dropped_handle" || return 0 + if [ -z "${1:-}" ]; then + np__drop 'dropped' 'a reason is required' + return 0 + fi + np__stage_facet "$_dropped_handle" "$NP_FACET_DROPPED" "$(np__json_obj reason "$1")" + np__flush_foreign "$_dropped_handle" + return 0 +} + +# np_trace_plan [handle] +# +# Declare the node's EXPECTED step plan ([{"key":...,"title":...}, ...]) so +# the read model reports expected-vs-observed progress. On a reusable +# definition, prefer np_trace_job --plan. +np_trace_plan() { + _plan_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_plan_handle" || return 0 + case "${1:-}" in + \[*) ;; + *) np__drop 'plan' 'the plan must be a JSON array of steps'; return 0 ;; + esac + np__stage_facet "$_plan_handle" "$NP_FACET_PLAN" "$1" + np__flush_foreign "$_plan_handle" return 0 } @@ -1483,19 +2064,19 @@ np_trace_consumes() { # ('{"kind":"deploy-log",...}') or a bare array of them; the wire form is # always the array. np_trace_affordances() { - _af_handle=$(np__resolve_handle "${1:-}") + _affordances_handle=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_af_handle" || return 0 - _af_body=${1:-} - case "$_af_body" in + np__is_handle "$_affordances_handle" || return 0 + _affordances_body=${1:-} + case "$_affordances_body" in \[*) ;; - \{*) _af_body="[$_af_body]" ;; + \{*) _affordances_body="[$_affordances_body]" ;; *) np__drop 'affordances' 'body must be a JSON object or array'; return 0 ;; esac - np__stage_facet "$_af_handle" "$NP_FACET_AFFORDANCES" "$_af_body" - np__flush_foreign "$_af_handle" + np__stage_facet "$_affordances_handle" "$NP_FACET_AFFORDANCES" "$_affordances_body" + np__flush_foreign "$_affordances_handle" return 0 } @@ -1505,25 +2086,25 @@ np_trace_affordances() { # instances 3 of 10, traffic 40 of 100. Non-negative integers; the optional # unit names what is counted ("percent", "instances"). np_trace_progress() { - _pg_handle=$(np__resolve_handle "${1:-}") + _progress_handle=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_pg_handle" || return 0 - _pg_current=${1:-} - _pg_target=${2:-} - _pg_unit=${3:-} - if [ -z "$_pg_current" ] || [ -z "$_pg_target" ]; then + np__is_handle "$_progress_handle" || return 0 + _progress_current=${1:-} + _progress_target=${2:-} + _progress_unit=${3:-} + if [ -z "$_progress_current" ] || [ -z "$_progress_target" ]; then np__drop 'progress' 'current and target must be non-negative integers' return 0 fi - case "$_pg_current$_pg_target" in + case "$_progress_current$_progress_target" in *[!0-9]*) np__drop 'progress' 'current and target must be non-negative integers'; return 0 ;; esac - np__stage_facet "$_pg_handle" "$NP_FACET_PROGRESS" \ - "$(np__json_obj_raw current "$_pg_current" target "$_pg_target" \ - unit "$(if [ -n "$_pg_unit" ]; then np__json_str "$_pg_unit"; fi)")" - np__flush_foreign "$_pg_handle" + np__stage_facet "$_progress_handle" "$NP_FACET_PROGRESS" \ + "$(np__json_obj_raw current "$_progress_current" target "$_progress_target" \ + unit "$(if [ -n "$_progress_unit" ]; then np__json_str "$_progress_unit"; fi)")" + np__flush_foreign "$_progress_handle" return 0 } @@ -1550,10 +2131,10 @@ np__terminalize() { np__emit_node "$1" "$2" np__ambient_clear "$1" # Restore the parent as ambient so a sibling opened next lands correctly. - _tz_parent=$(np__node_get "$1" parent) - if [ -n "$_tz_parent" ] && np__is_handle "$_tz_parent"; then - if [ "$(np__node_get "$_tz_parent" closed)" != '1' ]; then - np__ambient_set "$_tz_parent" + _terminalize_parent=$(np__node_get "$1" parent) + if [ -n "$_terminalize_parent" ] && np__is_handle "$_terminalize_parent"; then + if [ "$(np__node_get "$_terminalize_parent" closed)" != '1' ]; then + np__ambient_set "$_terminalize_parent" fi fi return 0 @@ -1571,7 +2152,7 @@ np_trace_end() { } np_trace_fail() { - _fa_h=$(np__resolve_handle "${1:-}") + _fail_h=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi @@ -1579,18 +2160,18 @@ np_trace_fail() { # (error facet emitted via the foreign flush, close refused) would smear an # unowned outcome onto the node. Recording an observed fact on a foreign # node is np_trace_error, deliberately. - if np__is_foreign "$_fa_h"; then + if np__is_foreign "$_fail_h"; then np__drop 'terminal' 'refusing to close an adopted node' return 0 fi if [ -n "${1:-}" ]; then - np_trace_error "$_fa_h" --message "$1" + np_trace_error "$_fail_h" --message "$1" fi # fail cascades to still-open child steps; complete deliberately does not — # auto-completing an open child would assert a success the SDK cannot vouch # for, and back-date its duration. - np__cascade_fail "$_fa_h" "${1:-}" - np__terminalize "$_fa_h" "$NP_STATUS_FAILED" + np__cascade_fail "$_fail_h" "${1:-}" + np__terminalize "$_fail_h" "$NP_STATUS_FAILED" return 0 } @@ -1599,14 +2180,14 @@ np_trace_fail() { # clobbers its caller's loop variables — which silently skipped intermediate # nodes in the cascade. np__is_descendant_of() { - _dz_cur=$(np__node_get "$1" parent) - _dz_guard=0 - while [ -n "$_dz_cur" ] && [ "$_dz_guard" -lt 64 ]; do - if [ "$_dz_cur" = "$2" ]; then + _is_descendant_of_cur=$(np__node_get "$1" parent) + _is_descendant_of_guard=0 + while [ -n "$_is_descendant_of_cur" ] && [ "$_is_descendant_of_guard" -lt 64 ]; do + if [ "$_is_descendant_of_cur" = "$2" ]; then return 0 fi - _dz_cur=$(np__node_get "$_dz_cur" parent) - _dz_guard=$((_dz_guard + 1)) + _is_descendant_of_cur=$(np__node_get "$_is_descendant_of_cur" parent) + _is_descendant_of_guard=$((_is_descendant_of_guard + 1)) done return 1 } @@ -1614,55 +2195,55 @@ np__is_descendant_of() { # Fail every still-open descendant. One flat pass over the registry, deepest # first, so a node is closed before anything reads it as a parent. np__cascade_fail() { - _cf_depth=64 - while [ "$_cf_depth" -ge 0 ]; do - for _cf_file in "$NP_TRACE_DIR/nodes"/*; do - [ -f "$_cf_file" ] || continue - _cf_h=${_cf_file##*/} - [ "$_cf_h" = "$1" ] && continue - [ "$(np__node_get "$_cf_h" closed)" = '1' ] && continue - np__is_descendant_of "$_cf_h" "$1" || continue - [ "$(np__depth_of "$_cf_h")" -eq "$_cf_depth" ] || continue + _cascade_fail_depth=64 + while [ "$_cascade_fail_depth" -ge 0 ]; do + for _cascade_fail_file in "$NP_TRACE_DIR/nodes"/*; do + [ -f "$_cascade_fail_file" ] || continue + _cascade_fail_h=${_cascade_fail_file##*/} + [ "$_cascade_fail_h" = "$1" ] && continue + [ "$(np__node_get "$_cascade_fail_h" closed)" = '1' ] && continue + np__is_descendant_of "$_cascade_fail_h" "$1" || continue + [ "$(np__depth_of "$_cascade_fail_h")" -eq "$_cascade_fail_depth" ] || continue if [ -n "$2" ]; then - np_trace_error "$_cf_h" --message "$2" + np_trace_error "$_cascade_fail_h" --message "$2" fi - np__terminalize "$_cf_h" "$NP_STATUS_FAILED" + np__terminalize "$_cascade_fail_h" "$NP_STATUS_FAILED" done - _cf_depth=$((_cf_depth - 1)) + _cascade_fail_depth=$((_cascade_fail_depth - 1)) done return 0 } # How many parent links sit above this node. np__depth_of() { - _do_cur=$(np__node_get "$1" parent) - _do_n=0 - while [ -n "$_do_cur" ] && [ "$_do_n" -lt 64 ]; do - _do_n=$((_do_n + 1)) - _do_cur=$(np__node_get "$_do_cur" parent) + _depth_of_cur=$(np__node_get "$1" parent) + _depth_of_n=0 + while [ -n "$_depth_of_cur" ] && [ "$_depth_of_n" -lt 64 ]; do + _depth_of_n=$((_depth_of_n + 1)) + _depth_of_cur=$(np__node_get "$_depth_of_cur" parent) done - printf '%s' "$_do_n" + printf '%s' "$_depth_of_n" } np_trace_skip() { - _sk_h=$(np__resolve_handle "${1:-}") + _skip_h=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__is_handle "$_sk_h" || return 0 + np__is_handle "$_skip_h" || return 0 if [ -n "${1:-}" ]; then - np__stage_facet "$_sk_h" "$NP_FACET_DROPPED" "$(np__json_obj reason "$1")" + np__stage_facet "$_skip_h" "$NP_FACET_DROPPED" "$(np__json_obj reason "$1")" fi - np__terminalize "$_sk_h" "$NP_STATUS_SKIPPED" + np__terminalize "$_skip_h" "$NP_STATUS_SKIPPED" return 0 } np_trace_cancel() { - _cn_h=$(np__resolve_handle "${1:-}") + _cancel_h=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then shift fi - np__terminalize "$_cn_h" "$NP_STATUS_CANCELLED" + np__terminalize "$_cancel_h" "$NP_STATUS_CANCELLED" return 0 } @@ -1673,10 +2254,10 @@ np_trace_timeout() { # Non-terminal: the node stays open. np_trace_waiting() { - _wt_h=$(np__resolve_handle "${1:-}") - np__is_handle "$_wt_h" || return 0 - np_trace_start "$_wt_h" - np__emit_node "$_wt_h" "$NP_STATUS_WAITING" + _waiting_h=$(np__resolve_handle "${1:-}") + np__is_handle "$_waiting_h" || return 0 + np_trace_start "$_waiting_h" + np__emit_node "$_waiting_h" "$NP_STATUS_WAITING" return 0 } From 529cbccba730abc910b3ad19be308b0b31c1fbe2 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 17:45:14 -0300 Subject: [PATCH 14/52] fix(k8s): failed applies carry kubectl's actual reason; restarts carry theirs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified by executing the failure stories end to end: a Forbidden apply previously traced as a bare 'Failed to apply' — kubectl's stderr (the principal, the resource, the namespace) never reached the trace. The capture now includes stderr, and the failure names the manifest and the server's reason. Restart narratives gain their cause the same way: '4 restarts so far (OOMKilled)' instead of just the count, with restart_reasons on the instances-health meter. --- k8s/apply_templates | 11 ++++++----- k8s/deployment/wait_deployment_active | 19 ++++++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/k8s/apply_templates b/k8s/apply_templates index 734b1f8b..4191070a 100644 --- a/k8s/apply_templates +++ b/k8s/apply_templates @@ -53,17 +53,18 @@ while IFS= read -r TEMPLATE_FILE; do [[ -n "$TRACE_STEP_KEY" ]] && np_scope_step_begin "$TRACE_STEP_KEY" fi - # Captured (and re-echoed) so applied resources can be recorded as - # lineage on the trace; the console output is unchanged. - if KUBECTL_OUT=$(kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND); then + # Captured with stderr (and re-echoed) so applied resources can be + # recorded as lineage — and so a FAILURE carries kubectl's actual reason + # onto the trace, not a bare "failed to apply". + if KUBECTL_OUT=$(kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND 2>&1); then [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" if [[ "$ACTION" == "apply" ]] && command -v np_scope_k8s_applied >/dev/null 2>&1; then np_scope_k8s_applied "${K8S_NAMESPACE:-}" "$KUBECTL_OUT" fi [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 0 else - [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" - log error " ❌ Failed to apply" + [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" >&2 + log error " ❌ Failed to apply $FILENAME${KUBECTL_OUT:+: $KUBECTL_OUT}" [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 1 fi TRACE_STEP_KEY="" diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index b7ed2a31..11a50254 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -71,11 +71,18 @@ report_wait_narrative() { command -v np_scope_explain >/dev/null 2>&1 || return 0 local ready_now="$1" desired_now="$2" launched_now="$3" all_healthy="${4:-false}" - local restarted restart_total + local restarted restart_total restart_reasons restart_clause restarted=$(kubectl get pods -n "$K8S_NAMESPACE" -l "deployment_id=${DEPLOYMENT_ID}" -o json 2>/dev/null \ - | jq -c '[.items[] | {name: .metadata.name, restarts: ([.status.containerStatuses[]?.restartCount] | add // 0)} | select(.restarts > 0)]' 2>/dev/null) || restarted="[]" + | jq -c '[.items[] | {name: .metadata.name, + restarts: ([.status.containerStatuses[]?.restartCount] | add // 0), + reason: ([.status.containerStatuses[]?.lastState.terminated.reason // empty] | first // empty)} + | select(.restarts > 0) | with_entries(select(.value != "" and .value != null))]' 2>/dev/null) || restarted="[]" [ -n "$restarted" ] || restarted="[]" restart_total=$(echo "$restarted" | jq 'map(.restarts) | add // 0' 2>/dev/null) || restart_total=0 + # WHY they restarted (OOMKilled, Error, ...) — the clause that turns + # "4 restarts" into a diagnosis. + restart_reasons=$(echo "$restarted" | jq -r '[.[].reason // empty] | unique | join(", ")' 2>/dev/null) || restart_reasons="" + restart_clause="${restart_reasons:+ ($restart_reasons)}" local instances meter instances=$(jq -nc --argjson h "$ready_now" --argjson l "$launched_now" --argjson d "$desired_now" \ @@ -85,9 +92,11 @@ report_wait_narrative() { meter=$(jq -nc --argjson h "$ready_now" --argjson l "$launched_now" --argjson d "$desired_now" \ --argjson u "$UNHEALTHY_POD_COUNT" --arg reasons "$UNHEALTHY_POD_REASONS" --argjson t "$restart_total" \ + --arg rreasons "$restart_reasons" \ '{kind: "instances-health", healthy: $h, launched: $l, desired: $d} + (if $u > 0 then {unhealthy: $u, reasons: ($reasons | split(", ") | map(select(. != "")))} else {} end) - + (if $t > 0 then {restarts: $t} else {} end)') + + (if $t > 0 then {restarts: $t} else {} end) + + (if $rreasons != "" then {restart_reasons: ($rreasons | split(", "))} else {} end)') np_scope_affordance "$meter" local restarts_label="restarts" @@ -99,11 +108,11 @@ report_wait_narrative() { elif [ "$restart_total" -gt 0 ]; then if [ "$all_healthy" = "true" ]; then np_scope_explain --title "$WAIT_TITLE" --severity warn \ - --what "All $desired_now instances healthy — after $restart_total $restarts_label" \ + --what "All $desired_now instances healthy — after $restart_total $restarts_label$restart_clause" \ --impact "Repeated restarts during startup usually mean crashes or out-of-memory kills; check the instance logs and memory limits." else np_scope_explain --title "$WAIT_TITLE" --severity warn \ - --what "Waiting for $ready_now/$desired_now instances to be healthy — $restart_total $restarts_label so far" \ + --what "Waiting for $ready_now/$desired_now instances to be healthy — $restart_total $restarts_label so far$restart_clause" \ --impact "Repeated restarts during startup usually mean crashes or out-of-memory kills; check the instance logs and memory limits." fi elif [ "$all_healthy" = "true" ]; then From cc3882c969f0b163dc0ffff33dd17f08086b0823 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 12 Aug 2026 18:30:22 -0300 Subject: [PATCH 15/52] =?UTF-8?q?feat(k8s):=20show=20the=20REAL=20error,?= =?UTF-8?q?=20proactively=20=E2=80=94=20kubernetes'=20words=20and=20the=20?= =?UTF-8?q?app's=20own?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified by executing the failure stories: an ImagePullBackOff now traces the registry's verbatim message ('Back-off pulling image …: manifest unknown') live within one poll and cause-first at the terminal; a crash-loop attaches the previous container's last log lines and leads with the app's own final line ('FATAL: bind: address already in use'). The classification is live POD STATE (waiting reasons and messages), so problems that predate the wait or throttle their events are still named — the event sweep only fills in what state cannot say (which probe, which path). Boot churn is never presented as a problem, and the narrative re-evaluates every poll, emitting only when the situation changes. The 'possible causes / how to fix' bursts no longer shadow the cause: facets fold last-writer-wins, so recording each hint line as the error left the LAST HINT as the step's message. The burst's first message is now the error everywhere (step, run mirror, terminal — one cause-first message, mechanism in parentheses) and the hints ride as structured details.hints. --- k8s/deployment/wait_deployment_active | 109 +++++++++++++++++++++----- k8s/logging | 21 +++++ k8s/utils/tests/trace_logging.bats | 25 ++++++ scheduled_task/logging | 21 +++++ 4 files changed, 155 insertions(+), 21 deletions(-) diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index 11a50254..c6d9d2eb 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -71,27 +71,80 @@ report_wait_narrative() { command -v np_scope_explain >/dev/null 2>&1 || return 0 local ready_now="$1" desired_now="$2" launched_now="$3" all_healthy="${4:-false}" - local restarted restart_total restart_reasons restart_clause - restarted=$(kubectl get pods -n "$K8S_NAMESPACE" -l "deployment_id=${DEPLOYMENT_ID}" -o json 2>/dev/null \ + # ONE pod-state read feeds everything: the restart history (with its + # reasons), and the LIVE problem classification with kubernetes' own + # verbatim message — an ImagePullBackOff shows the registry's actual + # error, not a "check your image" hint. Boot churn (ContainerCreating, + # PodInitializing) is never presented as a problem. + local pods_json restarted restart_total restart_reasons restart_clause + local problems problem_count problem_reasons real_detail + pods_json=$(kubectl get pods -n "$K8S_NAMESPACE" -l "deployment_id=${DEPLOYMENT_ID}" -o json 2>/dev/null) || pods_json="" + restarted=$(echo "$pods_json" \ | jq -c '[.items[] | {name: .metadata.name, restarts: ([.status.containerStatuses[]?.restartCount] | add // 0), reason: ([.status.containerStatuses[]?.lastState.terminated.reason // empty] | first // empty)} | select(.restarts > 0) | with_entries(select(.value != "" and .value != null))]' 2>/dev/null) || restarted="[]" [ -n "$restarted" ] || restarted="[]" restart_total=$(echo "$restarted" | jq 'map(.restarts) | add // 0' 2>/dev/null) || restart_total=0 - # WHY they restarted (OOMKilled, Error, ...) — the clause that turns - # "4 restarts" into a diagnosis. restart_reasons=$(echo "$restarted" | jq -r '[.[].reason // empty] | unique | join(", ")' 2>/dev/null) || restart_reasons="" restart_clause="${restart_reasons:+ ($restart_reasons)}" + problems=$(echo "$pods_json" \ + | jq -c '[.items[] | .metadata.name as $pod | .status.containerStatuses[]? + | select(.state.waiting.reason != null + and .state.waiting.reason != "ContainerCreating" + and .state.waiting.reason != "PodInitializing") + | {pod: $pod, reason: .state.waiting.reason, + message: ((.state.waiting.message // "") | .[0:300])} + | with_entries(select(.value != ""))]' 2>/dev/null) || problems="[]" + [ -n "$problems" ] || problems="[]" + problem_count=$(echo "$problems" | jq 'length' 2>/dev/null) || problem_count=0 + problem_reasons=$(echo "$problems" | jq -r '[.[].reason] | unique | join(", ")' 2>/dev/null) || problem_reasons="" + # Kubernetes' own words for the FIRST problem — the real error, verbatim. + real_detail=$(echo "$problems" | jq -r '[.[].message // empty] | first // ""' 2>/dev/null \ + | tr '\n' ' ' | sed 's/[[:space:]]*$//') || real_detail="" + + # The event sweep fills in what pod state alone cannot name (which probe, + # on which path) when the state itself shows nothing. + if [ "${problem_count:-0}" -eq 0 ] && [ "$UNHEALTHY_POD_COUNT" -gt 0 ]; then + problem_count=$UNHEALTHY_POD_COUNT + problem_reasons=$UNHEALTHY_POD_REASONS + fi + WAIT_EFFECTIVE_REASONS="${problem_reasons:-$restart_reasons}" + WAIT_REAL_DETAIL="$real_detail" + + # A crash-loop's REAL error is what the app printed before it died: fetch + # the previous container's last lines once per situation change, bounded. + local crash_log="" + if [ "$restart_total" -gt 0 ]; then + local crash_pod + crash_pod=$(echo "$restarted" | jq -r '.[0].name // empty' 2>/dev/null) || crash_pod="" + if [ -n "$crash_pod" ]; then + crash_log=$(kubectl logs "$crash_pod" -n "$K8S_NAMESPACE" --previous --tail=15 2>/dev/null | tail -c 1500) || crash_log="" + fi + if [ -n "$crash_log" ] && [ -z "$real_detail" ]; then + WAIT_REAL_DETAIL=$(printf '%s' "$crash_log" | tail -1 | cut -c1-200) + fi + fi + + # Realtime without noise: emit only when the SITUATION changes (counts, a + # new problem, a recovery) — the loop calls this every poll. + local snapshot="$ready_now/$desired_now/$launched_now/$problem_count/$problem_reasons/$restart_total/$all_healthy" + if [ "$snapshot" = "${WAIT_NARRATIVE_SNAPSHOT:-}" ]; then + return 0 + fi + WAIT_NARRATIVE_SNAPSHOT="$snapshot" local instances meter instances=$(jq -nc --argjson h "$ready_now" --argjson l "$launched_now" --argjson d "$desired_now" \ - --argjson u "$UNHEALTHY_POD_COUNT" --argjson r "$restarted" \ - '{healthy: $h, launched: $l, desired: $d} + (if $u > 0 then {unhealthy: $u} else {} end) + (if ($r | length) > 0 then {restarted: $r} else {} end)') + --argjson p "$problems" --argjson r "$restarted" --arg crash "$crash_log" \ + '{healthy: $h, launched: $l, desired: $d} + + (if ($p | length) > 0 then {problems: $p} else {} end) + + (if ($r | length) > 0 then {restarted: $r} else {} end) + + (if $crash != "" then {last_crash_log: $crash} else {} end)') np_scope_output instances "$instances" meter=$(jq -nc --argjson h "$ready_now" --argjson l "$launched_now" --argjson d "$desired_now" \ - --argjson u "$UNHEALTHY_POD_COUNT" --arg reasons "$UNHEALTHY_POD_REASONS" --argjson t "$restart_total" \ + --argjson u "$problem_count" --arg reasons "$problem_reasons" --argjson t "$restart_total" \ --arg rreasons "$restart_reasons" \ '{kind: "instances-health", healthy: $h, launched: $l, desired: $d} + (if $u > 0 then {unhealthy: $u, reasons: ($reasons | split(", ") | map(select(. != "")))} else {} end) @@ -99,21 +152,22 @@ report_wait_narrative() { + (if $rreasons != "" then {restart_reasons: ($rreasons | split(", "))} else {} end)') np_scope_affordance "$meter" - local restarts_label="restarts" + local restarts_label="restarts" detail_clause="" [ "$restart_total" -eq 1 ] && restarts_label="restart" - if [ "$UNHEALTHY_POD_COUNT" -gt 0 ]; then + [ -n "$WAIT_REAL_DETAIL" ] && detail_clause=": $WAIT_REAL_DETAIL" + if [ "$problem_count" -gt 0 ]; then np_scope_explain --title "$WAIT_TITLE" --severity warn \ - --what "Waiting for $ready_now/$desired_now instances to be healthy — $UNHEALTHY_POD_COUNT failing health checks${UNHEALTHY_POD_REASONS:+ ($UNHEALTHY_POD_REASONS)}" \ + --what "Waiting for $ready_now/$desired_now instances to be healthy — $problem_count with ${problem_reasons:-failing health checks}$detail_clause" \ --impact "The deployment fails if the instances don't become healthy before the health-check timeout." elif [ "$restart_total" -gt 0 ]; then if [ "$all_healthy" = "true" ]; then np_scope_explain --title "$WAIT_TITLE" --severity warn \ --what "All $desired_now instances healthy — after $restart_total $restarts_label$restart_clause" \ - --impact "Repeated restarts during startup usually mean crashes or out-of-memory kills; check the instance logs and memory limits." + --impact "The instances crashed on the way here; the last crash output is attached to this step." else np_scope_explain --title "$WAIT_TITLE" --severity warn \ - --what "Waiting for $ready_now/$desired_now instances to be healthy — $restart_total $restarts_label so far$restart_clause" \ - --impact "Repeated restarts during startup usually mean crashes or out-of-memory kills; check the instance logs and memory limits." + --what "Waiting for $ready_now/$desired_now instances to be healthy — $restart_total $restarts_label so far$restart_clause$detail_clause" \ + --impact "The last crash output is attached to this step." fi elif [ "$all_healthy" = "true" ]; then np_scope_explain --title "$WAIT_TITLE" --what "All $desired_now instances healthy" @@ -180,14 +234,23 @@ while true; do source "$SERVICE_PATH/deployment/print_failed_deployment_hints" if command -v np_scope_step_timeout >/dev/null 2>&1; then - # The terminal narrative replaces the polling one: what it gave up - # with, at ERROR severity, with the structured counts as evidence. + # The terminal leads with the CAUSE — kubernetes' own words or the + # app's last crash line — and the mechanism (the timeout) follows + # in parentheses. ONE message everywhere: facets fold last-writer- + # wins, so the close must not overwrite the cause with the + # mechanism. + timeout_cause="${WAIT_REAL_DETAIL:-}" + timeout_reasons="${WAIT_EFFECTIVE_REASONS:-$UNHEALTHY_POD_REASONS}" + timeout_message="deployment '$K8S_DEPLOYMENT_NAME' not active after ${TIMEOUT}s" + if [ -n "$timeout_cause" ]; then + timeout_message="$timeout_cause ($timeout_message)" + fi np_scope_explain --title "$WAIT_TITLE" --severity error \ - --what "Gave up with ${ready:-0}/${desired:-0} instances healthy${UNHEALTHY_POD_REASONS:+ ($UNHEALTHY_POD_REASONS)}" - np_scope_error "deployment '$K8S_DEPLOYMENT_NAME' not active after ${TIMEOUT}s" \ - "$(jq -nc --argjson h "${ready:-0}" --argjson l "${launched:-0}" --argjson d "${desired:-0}" \ - '{instances: {healthy: $h, launched: $l, desired: $d}}')" - np_scope_step_timeout "deployment '$K8S_DEPLOYMENT_NAME' not active after ${TIMEOUT}s" + --what "Gave up with ${ready:-0}/${desired:-0} instances healthy${timeout_reasons:+ — $timeout_reasons}${timeout_cause:+: $timeout_cause}" + np_scope_error "$timeout_message" \ + "$(jq -nc --argjson h "${ready:-0}" --argjson l "${launched:-0}" --argjson d "${desired:-0}" --arg reasons "$timeout_reasons" \ + '{instances: {healthy: $h, launched: $l, desired: $d}} + (if $reasons != "" then {reasons: ($reasons | split(", "))} else {} end)')" + np_scope_step_timeout "$timeout_message" fi exit 1 fi @@ -226,6 +289,11 @@ while true; do report_instance_counts "$desired" "$launched" "$ready" + # Realtime WHY: the narrative re-evaluates every poll and emits only when + # the situation changes — a new problem appears on the trace within one + # poll interval, not at the next heartbeat. + report_wait_narrative "$ready" "$desired" "$launched" + if [ "$desired" = "$current" ] && [ "$desired" = "$updated" ] && [ "$desired" = "$ready" ] && [ "$desired" -gt 0 ]; then log debug "" log info "✅ All pods in deployment '$K8S_DEPLOYMENT_NAME' are available and ready!" @@ -275,7 +343,6 @@ while true; do "wait.desired=$desired" "wait.launched=$launched" \ "wait.ready=$ready" "wait.available=$current" "wait.updated=$updated" np_scope_progress "$ready" "$desired" instances - report_wait_narrative "$ready" "$desired" "$launched" fi fi diff --git a/k8s/logging b/k8s/logging index 6a14dca1..e6d0b9e7 100644 --- a/k8s/logging +++ b/k8s/logging @@ -92,15 +92,36 @@ _np_scopes_node() { } # Record an observed failure on the current step (or open sub-step). +# +# A failure is usually a BURST of log errors: the cause first, then the +# "possible causes" and "how to fix" hint lines. Facets fold last-writer-wins, +# so naively recording each line would leave the LAST HINT as the step's +# error and bury the cause. Instead: the burst's FIRST message is the error, +# and every later line of the same step accumulates into the error's +# structured details.hints — the cause leads, the guidance travels with it. _np_scopes_trace_error() { local _lt_node _lt_node=$(_np_scopes_node) || return 0 + if [ "${_NP_SCOPES_ERRED_ON:-}" = "${NP_TRACE:-}" ] && [ -n "${_NP_SCOPES_ERR_MESSAGE:-}" ]; then + # Same step, later line: a hint. Re-emit the whole error with the + # original cause as the message and the growing hint list as evidence. + if [ -n "${_NP_SCOPES_ERR_HINTS:-}" ]; then + _NP_SCOPES_ERR_HINTS="$_NP_SCOPES_ERR_HINTS,$(np__json_str "$1")" + else + _NP_SCOPES_ERR_HINTS="$(np__json_str "$1")" + fi + np_trace_error "$_lt_node" --message "$_NP_SCOPES_ERR_MESSAGE" \ + --details "{\"hints\":[$_NP_SCOPES_ERR_HINTS]}" + return 0 + fi np_trace_error "$_lt_node" --message "$1" ${2:+--code "$2"} # Remember which step already carries a real message (so the exit trap does # not shadow it with a generic one) AND the message itself — on a fatal # exit, the run-level mirror repeats the real diagnosis, not the mechanism. _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" _NP_SCOPES_LAST_REASON="$1" + _NP_SCOPES_ERR_MESSAGE="$1" + _NP_SCOPES_ERR_HINTS="" return 0 } diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index 74e00324..248fca8a 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -387,3 +387,28 @@ secret/sec-1 created" [ "$status" -eq 0 ] [ "$(echo "$output" | grep -c '"root failure"')" -eq 1 ] } + +@test "a hint burst never shadows the cause: first message wins, hints ride as evidence" { + run_logged ' + log error "❌ HostedZone not found (AccessDenied)" + log error "💡 Possible causes:" + log error " • The role lacks route53:ListHostedZones" + ' + [ "$status" -eq 0 ] + # every emission of the burst carries the CAUSE as the message + ! echo "$output" | grep '"tracing.error"' | grep -q '"message":"💡' + echo "$output" | grep -q '"message":"❌ HostedZone not found (AccessDenied)","details":{"hints":\["💡 Possible causes:"," • The role lacks route53:ListHostedZones"\]}' +} + +@test "a new step starts a new error burst" { + run_logged ' + log error "first cause" + export NP_TRACE="1|trace-9|scope-provision-42~create-dns@0.0" + log error "second cause" + log error "a hint for the second" + ' + [ "$status" -eq 0 ] + echo "$output" | grep 'create-dns@0.0' | grep '"tracing.error"' | grep -q '"message":"second cause"' + echo "$output" | grep -q '"hints":\["a hint for the second"\]' + ! echo "$output" | grep 'create-dns@0.0' | grep -q '"message":"first cause"' +} diff --git a/scheduled_task/logging b/scheduled_task/logging index 6a14dca1..e6d0b9e7 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -92,15 +92,36 @@ _np_scopes_node() { } # Record an observed failure on the current step (or open sub-step). +# +# A failure is usually a BURST of log errors: the cause first, then the +# "possible causes" and "how to fix" hint lines. Facets fold last-writer-wins, +# so naively recording each line would leave the LAST HINT as the step's +# error and bury the cause. Instead: the burst's FIRST message is the error, +# and every later line of the same step accumulates into the error's +# structured details.hints — the cause leads, the guidance travels with it. _np_scopes_trace_error() { local _lt_node _lt_node=$(_np_scopes_node) || return 0 + if [ "${_NP_SCOPES_ERRED_ON:-}" = "${NP_TRACE:-}" ] && [ -n "${_NP_SCOPES_ERR_MESSAGE:-}" ]; then + # Same step, later line: a hint. Re-emit the whole error with the + # original cause as the message and the growing hint list as evidence. + if [ -n "${_NP_SCOPES_ERR_HINTS:-}" ]; then + _NP_SCOPES_ERR_HINTS="$_NP_SCOPES_ERR_HINTS,$(np__json_str "$1")" + else + _NP_SCOPES_ERR_HINTS="$(np__json_str "$1")" + fi + np_trace_error "$_lt_node" --message "$_NP_SCOPES_ERR_MESSAGE" \ + --details "{\"hints\":[$_NP_SCOPES_ERR_HINTS]}" + return 0 + fi np_trace_error "$_lt_node" --message "$1" ${2:+--code "$2"} # Remember which step already carries a real message (so the exit trap does # not shadow it with a generic one) AND the message itself — on a fatal # exit, the run-level mirror repeats the real diagnosis, not the mechanism. _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" _NP_SCOPES_LAST_REASON="$1" + _NP_SCOPES_ERR_MESSAGE="$1" + _NP_SCOPES_ERR_HINTS="" return 0 } From 35291db83554f7f7ebc2270f00f3c521489aa6fd Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Thu, 13 Aug 2026 15:06:57 -0300 Subject: [PATCH 16/52] fix(k8s): surface the provider's real error in IAM and resource lookups End-to-end verification (real CLI executing the real create workflow against a wire sink) caught scripts that capture the provider's error output and then log a generic message: build_service_account, create_role (OIDC, account id, create-role, policy attach), delete_role and require_resource. Each now carries the captured reason on the traced error, apply_templates-style. Trace error messages also drop the console decoration (indentation, the marker emoji) so the cause reads clean on the trace while console output stays exactly as it was. --- k8s/logging | 17 +++++++++++------ k8s/scope/iam/build_service_account | 8 ++++---- k8s/scope/iam/create_role | 25 +++++++++++++++---------- k8s/scope/iam/delete_role | 2 +- k8s/scope/require_resource | 3 +-- k8s/utils/tests/trace_logging.bats | 14 ++++++++++++-- scheduled_task/logging | 17 +++++++++++------ 7 files changed, 55 insertions(+), 31 deletions(-) diff --git a/k8s/logging b/k8s/logging index e6d0b9e7..64f808f2 100644 --- a/k8s/logging +++ b/k8s/logging @@ -100,27 +100,32 @@ _np_scopes_node() { # and every later line of the same step accumulates into the error's # structured details.hints — the cause leads, the guidance travels with it. _np_scopes_trace_error() { - local _lt_node + local _lt_node _lt_message _lt_node=$(_np_scopes_node) || return 0 + # The console line arrives verbatim; the trace carries the CAUSE, so strip + # the console decoration (leading indentation and the ❌ marker) first. + _lt_message="$1" + _lt_message="${_lt_message#"${_lt_message%%[![:space:]]*}"}" + case "$_lt_message" in "❌ "*) _lt_message="${_lt_message#❌ }" ;; esac if [ "${_NP_SCOPES_ERRED_ON:-}" = "${NP_TRACE:-}" ] && [ -n "${_NP_SCOPES_ERR_MESSAGE:-}" ]; then # Same step, later line: a hint. Re-emit the whole error with the # original cause as the message and the growing hint list as evidence. if [ -n "${_NP_SCOPES_ERR_HINTS:-}" ]; then - _NP_SCOPES_ERR_HINTS="$_NP_SCOPES_ERR_HINTS,$(np__json_str "$1")" + _NP_SCOPES_ERR_HINTS="$_NP_SCOPES_ERR_HINTS,$(np__json_str "$_lt_message")" else - _NP_SCOPES_ERR_HINTS="$(np__json_str "$1")" + _NP_SCOPES_ERR_HINTS="$(np__json_str "$_lt_message")" fi np_trace_error "$_lt_node" --message "$_NP_SCOPES_ERR_MESSAGE" \ --details "{\"hints\":[$_NP_SCOPES_ERR_HINTS]}" return 0 fi - np_trace_error "$_lt_node" --message "$1" ${2:+--code "$2"} + np_trace_error "$_lt_node" --message "$_lt_message" ${2:+--code "$2"} # Remember which step already carries a real message (so the exit trap does # not shadow it with a generic one) AND the message itself — on a fatal # exit, the run-level mirror repeats the real diagnosis, not the mechanism. _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" - _NP_SCOPES_LAST_REASON="$1" - _NP_SCOPES_ERR_MESSAGE="$1" + _NP_SCOPES_LAST_REASON="$_lt_message" + _NP_SCOPES_ERR_MESSAGE="$_lt_message" _NP_SCOPES_ERR_HINTS="" return 0 } diff --git a/k8s/scope/iam/build_service_account b/k8s/scope/iam/build_service_account index 64a7511b..a3abb200 100644 --- a/k8s/scope/iam/build_service_account +++ b/k8s/scope/iam/build_service_account @@ -21,7 +21,7 @@ ROLE_ARN=$(aws iam get-role --role-name "$SERVICE_ACCOUNT_NAME" --query 'Role.Ar return 0 fi - log error " ❌ Failed to find IAM role '$SERVICE_ACCOUNT_NAME'" + log error " ❌ Failed to find IAM role '$SERVICE_ACCOUNT_NAME'${ROLE_ARN:+: $ROLE_ARN}" log error "" log error "💡 Possible causes:" log error " The IAM role may not exist or the agent lacks IAM permissions" @@ -39,10 +39,10 @@ SERVICE_ACCOUNT_PATH="$OUTPUT_DIR/service_account-$SCOPE_ID.yaml" echo "$CONTEXT" | jq --arg role_arn "$ROLE_ARN" --arg service_account_name "$SERVICE_ACCOUNT_NAME" '. + {role_arn: $role_arn, service_account_name: $service_account_name}' > "$CONTEXT_PATH" log debug "📝 Building service account template: $SERVICE_ACCOUNT_TEMPLATE" -gomplate -c .="$CONTEXT_PATH" \ +GOMPLATE_OUT=$(gomplate -c .="$CONTEXT_PATH" \ --file "$SERVICE_ACCOUNT_TEMPLATE" \ - --out "$SERVICE_ACCOUNT_PATH" || { - log error " ❌ Failed to build service account template" + --out "$SERVICE_ACCOUNT_PATH" 2>&1) || { + log error " ❌ Failed to build service account template${GOMPLATE_OUT:+: $GOMPLATE_OUT}" log error "" log error "💡 Possible causes:" log error " The template file may be missing or contain invalid gomplate syntax" diff --git a/k8s/scope/iam/create_role b/k8s/scope/iam/create_role index 2307c703..bec4ef3b 100644 --- a/k8s/scope/iam/create_role +++ b/k8s/scope/iam/create_role @@ -17,7 +17,7 @@ ROLE_PATH="/nullplatform/custom-scopes/" NAMESPACE=$(echo "$CONTEXT" | jq -r .k8s_namespace) log debug "🔍 Getting EKS OIDC provider for cluster: $CLUSTER_NAME" OIDC_PROVIDER=$(aws eks describe-cluster --name "$CLUSTER_NAME" --query "cluster.identity.oidc.issuer" --output text 2>&1 | sed -e "s/^https:\/\///") || { - log error " ❌ Failed to get OIDC provider for EKS cluster '$CLUSTER_NAME'" + log error " ❌ Failed to get OIDC provider for EKS cluster '$CLUSTER_NAME'${OIDC_PROVIDER:+: $OIDC_PROVIDER}" log error "" log error "💡 Possible causes:" log error " The OIDC provider may not be configured for this EKS cluster" @@ -31,7 +31,7 @@ OIDC_PROVIDER=$(aws eks describe-cluster --name "$CLUSTER_NAME" --query "cluster log debug "🔍 Getting AWS account ID..." AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text 2>&1) || { - log error " ❌ Failed to get AWS account ID" + log error " ❌ Failed to get AWS account ID${AWS_ACCOUNT_ID:+: $AWS_ACCOUNT_ID}" log error "" log error "💡 Possible causes:" log error " AWS credentials may not be configured or have expired" @@ -100,7 +100,7 @@ if [[ -n "$DIMENSIONS" && "$DIMENSIONS" != "null" ]]; then fi create_role_error() { - log error " ❌ Failed to create IAM role '$ROLE_NAME'" + log error " ❌ Failed to create IAM role '$ROLE_NAME'${CREATE_ROLE_OUT:+: $CREATE_ROLE_OUT}" log error "" log error "💡 Possible causes:" log error " The role may already exist or the agent lacks IAM permissions" @@ -127,7 +127,10 @@ if [[ -n "$BOUNDARY_ARN" && "$BOUNDARY_ARN" != "null" ]]; then CREATE_ROLE_ARGS+=(--permissions-boundary "$BOUNDARY_ARN") fi -aws iam create-role "${CREATE_ROLE_ARGS[@]}" || create_role_error +# Captured (with stderr) so a failure carries AWS's actual reason onto the +# trace; on success the response is echoed exactly as before. +CREATE_ROLE_OUT=$(aws iam create-role "${CREATE_ROLE_ARGS[@]}" 2>&1) || create_role_error +[[ -n "$CREATE_ROLE_OUT" ]] && echo "$CREATE_ROLE_OUT" log info " ✅ IAM role created successfully" rm "$TRUST_POLICY_PATH" @@ -141,10 +144,10 @@ for ((i=0; i<$POLICIES_COUNT; i++)); do if [[ "$POLICY_TYPE" == "arn" ]]; then log debug "📝 Attaching managed policy: $POLICY_VALUE" - aws iam attach-role-policy \ + ATTACH_POLICY_OUT=$(aws iam attach-role-policy \ --role-name "$ROLE_NAME" \ - --policy-arn "$POLICY_VALUE" || { - log error " ❌ Failed to attach managed policy: $POLICY_VALUE" + --policy-arn "$POLICY_VALUE" 2>&1) || { + log error " ❌ Failed to attach managed policy: $POLICY_VALUE${ATTACH_POLICY_OUT:+ — $ATTACH_POLICY_OUT}" log error "" log error "💡 Possible causes:" log error " The policy ARN may be invalid or the agent lacks IAM permissions" @@ -155,6 +158,7 @@ for ((i=0; i<$POLICIES_COUNT; i++)); do log error "" exit 1 } + [[ -n "$ATTACH_POLICY_OUT" ]] && echo "$ATTACH_POLICY_OUT" log info " ✅ Successfully attached managed policy: $POLICY_VALUE" elif [[ "$POLICY_TYPE" == "inline" ]]; then @@ -164,11 +168,11 @@ for ((i=0; i<$POLICIES_COUNT; i++)); do TEMP_POLICY_FILE="$OUTPUT_DIR/inline-policy-$i.json" echo "$POLICY_VALUE" > "$TEMP_POLICY_FILE" - aws iam put-role-policy \ + PUT_POLICY_OUT=$(aws iam put-role-policy \ --role-name "$ROLE_NAME" \ --policy-name "$POLICY_NAME" \ - --policy-document "file://$TEMP_POLICY_FILE" || { - log error " ❌ Failed to attach inline policy: $POLICY_NAME" + --policy-document "file://$TEMP_POLICY_FILE" 2>&1) || { + log error " ❌ Failed to attach inline policy: $POLICY_NAME${PUT_POLICY_OUT:+ — $PUT_POLICY_OUT}" log error "" log error "💡 Possible causes:" log error " The inline policy JSON may be invalid or the agent lacks IAM permissions" @@ -180,6 +184,7 @@ for ((i=0; i<$POLICIES_COUNT; i++)); do rm -f "$TEMP_POLICY_FILE" exit 1 } + [[ -n "$PUT_POLICY_OUT" ]] && echo "$PUT_POLICY_OUT" log info " ✅ Successfully attached inline policy: $POLICY_NAME" rm -f "$TEMP_POLICY_FILE" diff --git a/k8s/scope/iam/delete_role b/k8s/scope/iam/delete_role index eac8dbaf..81a0f458 100755 --- a/k8s/scope/iam/delete_role +++ b/k8s/scope/iam/delete_role @@ -19,7 +19,7 @@ ROLE_ARN=$(aws iam get-role --role-name "$SERVICE_ACCOUNT_NAME" --query 'Role.Ar return 0 fi - log error " ❌ Failed to find IAM role '$SERVICE_ACCOUNT_NAME'" + log error " ❌ Failed to find IAM role '$SERVICE_ACCOUNT_NAME'${ROLE_ARN:+: $ROLE_ARN}" log error "" log error "💡 Possible causes:" log error " The IAM role may not exist or the agent lacks IAM permissions" diff --git a/k8s/scope/require_resource b/k8s/scope/require_resource index a3daa10a..5785d6cb 100644 --- a/k8s/scope/require_resource +++ b/k8s/scope/require_resource @@ -55,8 +55,7 @@ find_deployment_by_label() { log debug "🔍 Looking for deployment with label: $label" DEPLOYMENT=$(kubectl get deployment -n "$namespace" -l "$label" -o jsonpath="{.items[0].metadata.name}" 2>&1) || { - log error " ❌ Failed to find deployment with label '$label' in namespace '$namespace'" - log debug "📋 Kubectl error: $DEPLOYMENT" + log error " ❌ Failed to find deployment with label '$label' in namespace '$namespace'${DEPLOYMENT:+: $DEPLOYMENT}" DEPLOYMENT="" } diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index 248fca8a..9498b82e 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -395,9 +395,19 @@ secret/sec-1 created" log error " • The role lacks route53:ListHostedZones" ' [ "$status" -eq 0 ] - # every emission of the burst carries the CAUSE as the message + # every emission of the burst carries the CAUSE as the message — with the + # console decoration (indentation, ❌ marker) stripped for the trace ! echo "$output" | grep '"tracing.error"' | grep -q '"message":"💡' - echo "$output" | grep -q '"message":"❌ HostedZone not found (AccessDenied)","details":{"hints":\["💡 Possible causes:"," • The role lacks route53:ListHostedZones"\]}' + echo "$output" | grep -q '"message":"HostedZone not found (AccessDenied)","details":{"hints":\["💡 Possible causes:","• The role lacks route53:ListHostedZones"\]}' +} + +@test "trace errors carry the cause stripped of console decoration" { + run_logged ' + log error " ❌ Failed to find IAM role: An error occurred (NoSuchEntity)" + ' + [ "$status" -eq 0 ] + echo "$output" | grep -q '"message":"Failed to find IAM role: An error occurred (NoSuchEntity)"' + ! echo "$output" | grep '"tracing.error"' | grep -q '"message":" ❌' } @test "a new step starts a new error burst" { diff --git a/scheduled_task/logging b/scheduled_task/logging index e6d0b9e7..64f808f2 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -100,27 +100,32 @@ _np_scopes_node() { # and every later line of the same step accumulates into the error's # structured details.hints — the cause leads, the guidance travels with it. _np_scopes_trace_error() { - local _lt_node + local _lt_node _lt_message _lt_node=$(_np_scopes_node) || return 0 + # The console line arrives verbatim; the trace carries the CAUSE, so strip + # the console decoration (leading indentation and the ❌ marker) first. + _lt_message="$1" + _lt_message="${_lt_message#"${_lt_message%%[![:space:]]*}"}" + case "$_lt_message" in "❌ "*) _lt_message="${_lt_message#❌ }" ;; esac if [ "${_NP_SCOPES_ERRED_ON:-}" = "${NP_TRACE:-}" ] && [ -n "${_NP_SCOPES_ERR_MESSAGE:-}" ]; then # Same step, later line: a hint. Re-emit the whole error with the # original cause as the message and the growing hint list as evidence. if [ -n "${_NP_SCOPES_ERR_HINTS:-}" ]; then - _NP_SCOPES_ERR_HINTS="$_NP_SCOPES_ERR_HINTS,$(np__json_str "$1")" + _NP_SCOPES_ERR_HINTS="$_NP_SCOPES_ERR_HINTS,$(np__json_str "$_lt_message")" else - _NP_SCOPES_ERR_HINTS="$(np__json_str "$1")" + _NP_SCOPES_ERR_HINTS="$(np__json_str "$_lt_message")" fi np_trace_error "$_lt_node" --message "$_NP_SCOPES_ERR_MESSAGE" \ --details "{\"hints\":[$_NP_SCOPES_ERR_HINTS]}" return 0 fi - np_trace_error "$_lt_node" --message "$1" ${2:+--code "$2"} + np_trace_error "$_lt_node" --message "$_lt_message" ${2:+--code "$2"} # Remember which step already carries a real message (so the exit trap does # not shadow it with a generic one) AND the message itself — on a fatal # exit, the run-level mirror repeats the real diagnosis, not the mechanism. _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" - _NP_SCOPES_LAST_REASON="$1" - _NP_SCOPES_ERR_MESSAGE="$1" + _NP_SCOPES_LAST_REASON="$_lt_message" + _NP_SCOPES_ERR_MESSAGE="$_lt_message" _NP_SCOPES_ERR_HINTS="" return 0 } From e044c8033fc0a156bd38a2fbe7306dc5053830ec Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Thu, 13 Aug 2026 17:23:12 -0300 Subject: [PATCH 17/52] feat: consume catalog-tracing-sh as a git submodule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tracing SDK now rides as a submodule pinned to the SDK's release history instead of a hand-vendored copy of nptrace.sh. The logging hooks look for the submodule first and still fall back to a root-vendored nptrace.sh, so repo copies that lose the submodule (archive downloads, docker build contexts) keep working — and with neither present the scripts degrade to plain logging exactly as before. CI checks out submodules so the trace tests exercise the real SDK. --- .github/workflows/pr-checks.yml | 2 + .gitmodules | 4 + k8s/logging | 9 +- nptrace.sh | 2265 ------------------------------- scheduled_task/logging | 9 +- vendor/catalog-tracing-sh | 1 + 6 files changed, 21 insertions(+), 2269 deletions(-) create mode 100644 .gitmodules delete mode 100755 nptrace.sh create mode 160000 vendor/catalog-tracing-sh diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 69bc0e37..6a13381f 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -19,6 +19,8 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 + with: + submodules: true # catalog-tracing-sh — the tracing SDK the tests exercise - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y bats jq diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..7c2e3258 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "vendor/catalog-tracing-sh"] + path = vendor/catalog-tracing-sh + url = https://github.com/nullplatform/catalog-tracing-sh.git + branch = main diff --git a/k8s/logging b/k8s/logging index 64f808f2..dae4454c 100644 --- a/k8s/logging +++ b/k8s/logging @@ -407,12 +407,17 @@ _np_scopes_on_exit() { } _NP_SCOPES_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# The SDK ships as the catalog-tracing-sh submodule; a repo copy that lost the +# submodule (archive download, docker build context) may carry the single file +# vendored at the root instead. Neither present -> plain logging, untraced. +_NP_SCOPES_SDK="$_NP_SCOPES_ROOT/vendor/catalog-tracing-sh/nptrace.sh" +[ -f "$_NP_SCOPES_SDK" ] || _NP_SCOPES_SDK="$_NP_SCOPES_ROOT/nptrace.sh" if [ -z "${NP_TRACE_LOADED:-}" ] \ - && [ -f "$_NP_SCOPES_ROOT/nptrace.sh" ] \ + && [ -f "$_NP_SCOPES_SDK" ] \ && [ -n "${NP_API_KEY:-}" ] \ && [ -n "${NP_TRACE:-}" ]; then # shellcheck source=/dev/null - . "$_NP_SCOPES_ROOT/nptrace.sh" + . "$_NP_SCOPES_SDK" # --no-trap: the exit flush is ours, so the uncaught-failure report and the # flush share ONE trap in a defined order. np_trace_init --producer "nullplatform-scopes@1" --api-key "$NP_API_KEY" --no-trap diff --git a/nptrace.sh b/nptrace.sh deleted file mode 100755 index b26096db..00000000 --- a/nptrace.sh +++ /dev/null @@ -1,2265 +0,0 @@ -#!/bin/sh - -# ---- src/header.sh ---- -# nullplatform tracing for POSIX shell — producer SDK for the nullplatform -# tracing API. Zero runtime dependencies beyond curl and the POSIX toolset. -# -# Generated file: edit src/*.sh and run ./build.sh. - -if [ -n "${NP_TRACE_LOADED:-}" ]; then - return 0 2>/dev/null || exit 0 -fi -NP_TRACE_LOADED=1 -NP_TRACE_VERSION="0.1.0" - -# ---- src/compat.sh ---- -# compat.sh — portability shims. The ONLY place OS differences live. - -# Unix milliseconds. GNU date supports %N; busybox and BSD may not, and they -# fail in two DIFFERENT ways: -# -# busybox 1.38 / BSD -> "1786045823%3N" the format leaks through literally -# busybox 1.37 -> "1786045823" the format is silently DROPPED -# -# The second is the dangerous one: the result is clean digits that merely happen -# to be seconds, so a digits-only check accepts it and every timestamp is then -# 1000x too small — which silently destroys UUIDv7 ordering, since the seconds -# value lands in a 48-bit millisecond field and decodes to 1970. -# -# Length is what separates them: Unix milliseconds have been 13 digits since -# 2001-09-09 and stay 13 until 2286, while seconds are 10. Anything shorter than -# 13 is not milliseconds, whatever it looks like. -np__epoch_ms() { - _epoch_ms_ms=$(date -u +%s%3N 2>/dev/null) || _epoch_ms_ms='' - case "$_epoch_ms_ms" in - '' | *[!0-9]*) _epoch_ms_ms='' ;; - esac - if [ -n "$_epoch_ms_ms" ] && [ "${#_epoch_ms_ms}" -ge 13 ]; then - printf '%s' "$_epoch_ms_ms" - return 0 - fi - # Second precision. Event ids stay unique via their random bits. - printf '%s000' "$(date -u +%s)" -} - -# Exactly $1 lowercase hex characters from the kernel CSPRNG. -np__rand_hex() { - _rand_hex_want=$1 - _rand_hex_bytes=$(( (_rand_hex_want + 1) / 2 )) - od -An -tx1 -N"$_rand_hex_bytes" /dev/urandom | tr -d ' \n' | cut -c1-"$_rand_hex_want" -} - -# RFC 3339 UTC, second precision — the envelope `time` field. -np__iso8601() { - date -u +%Y-%m-%dT%H:%M:%SZ -} - -# ---- src/json.sh ---- -# json.sh — JSON emission. There is no parser here beyond one field extractor -# for the auth response; the SDK only ever WRITES JSON. - -# Escape a string for a JSON string body (no surrounding quotes). -# -# Fast path: a string made only of unmistakably safe characters is returned -# unchanged, so the common label/id case never forks an awk. The allowlist is -# deliberately conservative — routing an unusual string to the slow path is -# always correct, only slower. -# -# Slow path: awk under LC_ALL=C, so length/substr are BYTE oriented on every -# awk (gawk, mawk, busybox). UTF-8 sequences pass through byte for byte, which -# is valid JSON; only the seven shorthand escapes and C0 controls are rewritten. -# Records are read line by line and rejoined with \n rather than using a -# multi-character RS, whose behaviour POSIX leaves undefined. -np__json_escape() { - case "$1" in - *[!A-Za-z0-9\ ._:/@=+,-]*) ;; - *) printf '%s' "$1"; return 0 ;; - esac - printf '%s' "$1" | LC_ALL=C awk ' - function esc(s, i, c, n, o) { - o = "" - n = length(s) - for (i = 1; i <= n; i++) { - c = substr(s, i, 1) - if (c == "\\") { o = o "\\\\" } - else if (c == "\"") { o = o "\\\"" } - else if (c == "\t") { o = o "\\t" } - else if (c == "\r") { o = o "\\r" } - else if (c == "\b") { o = o "\\b" } - else if (c == "\f") { o = o "\\f" } - else if (c < " ") { o = o sprintf("\\u%04x", ORD[c]) } - else { o = o c } - } - return o - } - BEGIN { - ORS = "" - for (i = 0; i < 256; i++) { ORD[sprintf("%c", i)] = i } - out = "" - } - { - if (NR > 1) { out = out "\\n" } - out = out esc($0) - } - END { printf "%s", out } - ' -} - -# A complete quoted JSON string. -np__json_str() { - printf '"%s"' "$(np__json_escape "$1")" -} - -# A JSON object from alternating key/value arguments. Values are emitted as -# JSON strings. A pair whose key or value is empty is OMITTED — an absent -# optional is absent, never the string "". -np__json_obj() { - _json_obj_out='' - while [ "$#" -ge 2 ]; do - if [ -n "$1" ] && [ -n "$2" ]; then - if [ -n "$_json_obj_out" ]; then - _json_obj_out="$_json_obj_out," - fi - _json_obj_out="$_json_obj_out$(np__json_str "$1"):$(np__json_str "$2")" - fi - shift 2 - done - printf '{%s}' "$_json_obj_out" -} - -# As np__json_obj, but each value is already-formed JSON inserted verbatim. -# Use for nested objects, arrays, numbers, and booleans. -np__json_obj_raw() { - _json_obj_raw_out='' - while [ "$#" -ge 2 ]; do - if [ -n "$1" ] && [ -n "$2" ]; then - if [ -n "$_json_obj_raw_out" ]; then - _json_obj_raw_out="$_json_obj_raw_out," - fi - _json_obj_raw_out="$_json_obj_raw_out$(np__json_str "$1"):$2" - fi - shift 2 - done - printf '{%s}' "$_json_obj_raw_out" -} - -# A JSON array of strings from a comma-separated list ("a, b" → ["a","b"]). -# Surrounding whitespace per item is trimmed; empty items are omitted. -np__json_str_array_csv() { - _json_str_array_csv_out='' - _json_str_array_csv_rest=$1 - while [ -n "$_json_str_array_csv_rest" ]; do - case "$_json_str_array_csv_rest" in - *,*) _json_str_array_csv_item=${_json_str_array_csv_rest%%,*}; _json_str_array_csv_rest=${_json_str_array_csv_rest#*,} ;; - *) _json_str_array_csv_item=$_json_str_array_csv_rest; _json_str_array_csv_rest='' ;; - esac - _json_str_array_csv_item=$(printf '%s' "$_json_str_array_csv_item" | sed 's/^ *//; s/ *$//') - if [ -n "$_json_str_array_csv_item" ]; then - if [ -n "$_json_str_array_csv_out" ]; then - _json_str_array_csv_out="$_json_str_array_csv_out," - fi - _json_str_array_csv_out="$_json_str_array_csv_out$(np__json_str "$_json_str_array_csv_item")" - fi - done - printf '[%s]' "$_json_str_array_csv_out" -} - -# ---- src/uuid.sh ---- -# uuid.sh — UUIDv7. The event id MUST be a v7: the API derives the storage -# partition from its embedded millisecond timestamp and rejects anything else. -# -# Layout: 48-bit big-endian ms timestamp | version nibble 7 | 12 random bits -# | variant bits 10 | 62 random bits. - -np__uuidv7() { - _u7_ts=$(printf '%012x' "$(np__epoch_ms)") - _u7_r=$(np__rand_hex 19) - - # The variant nibble must be one of 8, 9, a, b. Fold a random hex digit into - # that range rather than drawing again. - case $(printf '%s' "$_u7_r" | cut -c1) in - 0 | 1 | 2 | 3) _u7_var=8 ;; - 4 | 5 | 6 | 7) _u7_var=9 ;; - 8 | 9 | a | b) _u7_var=a ;; - *) _u7_var=b ;; - esac - - printf '%s-%s-7%s-%s%s-%s\n' \ - "$(printf '%s' "$_u7_ts" | cut -c1-8)" \ - "$(printf '%s' "$_u7_ts" | cut -c9-12)" \ - "$(printf '%s' "$_u7_r" | cut -c2-4)" \ - "$_u7_var" \ - "$(printf '%s' "$_u7_r" | cut -c5-7)" \ - "$(printf '%s' "$_u7_r" | cut -c8-19)" -} - -# Mint a per-occurrence token for a repeatable operation's run_id. Time-ordered, -# so minted ids sort by creation time. -np_trace_occurrence() { - np__uuidv7 -} - -# ---- src/identity.sh ---- -# identity.sh — the node identity grammar. A hand-port of the tracing API's -# contract module; these functions and their tests are the drift safety net. -# -# child_run_id = parent_run_id "~" key "@" attempt "." iteration -# -# One charset covers every producer-authored segment: [A-Za-z0-9_.-]+. The -# delimiter '~' and the coordinate marker '@' sit outside it, which is what -# makes the grammar collision-proof — no named id can ever parse as a derived -# one. - -NP_ID_DELIMITER='~' -NP_MAX_RUN_ID_LENGTH=1024 -NP_MAX_KEY_LENGTH=256 -NP_MAX_TRACE_ID_LENGTH=256 - -np__is_identifier() { - case "${1:-}" in - '') return 1 ;; - *[!A-Za-z0-9_.-]*) return 1 ;; - *) return 0 ;; - esac -} - -# Print a reason and return 1, or return 0 silently. -np__identifier_violation() { - if [ -z "$1" ]; then - printf 'must be non-empty' - return 1 - fi - if [ "${#1}" -gt "$2" ]; then - printf 'exceeds %s chars' "$2" - return 1 - fi - if ! np__is_identifier "$1"; then - printf "must be identifier-charset: letters, digits, '_', '.', '-'" - return 1 - fi - return 0 -} - -np__key_violation() { - np__identifier_violation "${1:-}" "$NP_MAX_KEY_LENGTH" -} - -np__named_id_violation() { - np__identifier_violation "${1:-}" "$NP_MAX_RUN_ID_LENGTH" -} - -np__trace_id_violation() { - np__identifier_violation "${1:-}" "$NP_MAX_TRACE_ID_LENGTH" -} - -# The derived id of a keyed child. -np__derive_child_id() { - printf '%s%s%s@%s.%s' "$1" "$NP_ID_DELIMITER" "$2" "$3" "$4" -} - -# Everything before the FIRST delimiter — the nearest named ancestor. Every -# keyed descendant of a named run shares its scope root at any depth. -np__scope_root_of() { - case "$1" in - *"$NP_ID_DELIMITER"*) printf '%s' "${1%%"$NP_ID_DELIMITER"*}" ;; - *) printf '%s' "$1" ;; - esac -} - -# Parse the LAST hop of a derived id. Prints " ". -# Returns 1 for a named id (no delimiter) or a malformed tail. -np__parse_node_id() { - case "$1" in - *"$NP_ID_DELIMITER"*) ;; - *) return 1 ;; - esac - _parse_node_id_parent=${1%"$NP_ID_DELIMITER"*} - _parse_node_id_tail=${1##*"$NP_ID_DELIMITER"} - case "$_parse_node_id_tail" in - *@*.*) ;; - *) return 1 ;; - esac - _parse_node_id_key=${_parse_node_id_tail%%@*} - _parse_node_id_coord=${_parse_node_id_tail#*@} - _parse_node_id_attempt=${_parse_node_id_coord%%.*} - _parse_node_id_iteration=${_parse_node_id_coord#*.} - if [ -z "$_parse_node_id_parent" ] || [ -z "$_parse_node_id_key" ]; then - return 1 - fi - case "$_parse_node_id_attempt" in - '' | *[!0-9]*) return 1 ;; - esac - case "$_parse_node_id_iteration" in - '' | *[!0-9]*) return 1 ;; - esac - printf '%s %s %s %s' "$_parse_node_id_parent" "$_parse_node_id_key" "$_parse_node_id_attempt" "$_parse_node_id_iteration" -} - -# Join parts into a stable id, dropping empty parts. Use instead of -# hand-interpolation so an absent part never leaves a dangling separator. -# The joiner is '-', a charset character, so the result stays a legal named id. -np_trace_key() { - _k_out='' - for _k_part in "$@"; do - if [ -n "$_k_part" ]; then - if [ -n "$_k_out" ]; then - _k_out="$_k_out-" - fi - _k_out="$_k_out$_k_part" - fi - done - printf '%s' "$_k_out" -} - -# ---- src/wire.sh ---- -# wire.sh — contract constants, hand-ported from the tracing API's wire -# package. When the API's contract changes, this file and identity.sh are what -# must be re-ported; their tests are the safety net. - -NP_TYPE_NODE_RUN='node.run' -NP_TYPE_NODE_DATASET='node.dataset' -NP_TYPE_NODE_JOB='node.job' - -NP_TYPE_EDGE_PARENT='edge.parent' -NP_TYPE_EDGE_TRIGGERED_BY='edge.triggered_by' -NP_TYPE_EDGE_RETRY_OF='edge.retry_of' -NP_TYPE_EDGE_CONTINUES='edge.continues' -NP_TYPE_EDGE_CORRELATES='edge.correlates' -NP_TYPE_EDGE_COMPENSATES='edge.compensates' -NP_TYPE_EDGE_PRODUCES='edge.produces' -NP_TYPE_EDGE_CONSUMES='edge.consumes' -NP_TYPE_EDGE_INSTANCE_OF='edge.instance_of' - -NP_STATUS_STARTED='started' -NP_STATUS_COMPLETED='completed' -NP_STATUS_FAILED='failed' -NP_STATUS_CANCELLED='cancelled' -NP_STATUS_TIMED_OUT='timed_out' -NP_STATUS_SKIPPED='skipped' -NP_STATUS_WAITING='waiting' - -NP_FACET_ERROR='tracing.error' -NP_FACET_TIMING='tracing.timing' -NP_FACET_INPUT='tracing.input' -NP_FACET_OUTPUT='tracing.output' -NP_FACET_BINDING='tracing.binding' -NP_FACET_DECISION='tracing.decision' -NP_FACET_RETRY='tracing.retry' -NP_FACET_SIGNAL='tracing.signal' -NP_FACET_EXTERNAL_LINKS='tracing.externalLinks' -NP_FACET_PLAN='tracing.plan' -NP_FACET_ACTOR='tracing.actor' -NP_FACET_DROPPED='tracing.dropped' -NP_FACET_ENGINE_STATUS='tracing.engineStatus' -NP_FACET_AFFORDANCES='tracing.affordances' -NP_FACET_EXPLAIN='tracing.explain' -NP_FACET_PROGRESS='tracing.progress' - -NP_CORE_FACETS="$NP_FACET_ERROR $NP_FACET_TIMING $NP_FACET_INPUT $NP_FACET_OUTPUT \ -$NP_FACET_BINDING $NP_FACET_DECISION $NP_FACET_RETRY $NP_FACET_SIGNAL \ -$NP_FACET_EXTERNAL_LINKS $NP_FACET_PLAN $NP_FACET_ACTOR $NP_FACET_DROPPED \ -$NP_FACET_ENGINE_STATUS $NP_FACET_AFFORDANCES $NP_FACET_EXPLAIN $NP_FACET_PROGRESS" - -NP_RESERVED_FACET_PREFIX='tracing.' -NP_RESERVED_LABEL_PREFIX='tracing.io/' - -# The context carrier: ONE field whose value packs version, trace and run. -NP_CARRIER_KEY='np-trace' -NP_CARRIER_VERSION='1' -NP_CARRIER_DELIMITER='|' - -np__is_terminal_status() { - case "${1:-}" in - completed | failed | cancelled | timed_out | skipped) return 0 ;; - *) return 1 ;; - esac -} - -# ---- src/state.sh ---- -# state.sh — the on-disk node registry. State lives on disk rather than in -# shell memory so handles survive process boundaries: in CI every pipeline step -# is a fresh shell. - -# Create the state tree. If it cannot be created or written — a read-only -# filesystem, a full disk, a bad NP_TRACE_DIR — the SDK degrades to a REAL -# no-op rather than half-working: a half-initialised SDK whose next write fails -# would take down a caller running under `set -e`, which is exactly the failure -# mode tracing must never cause. -np__state_init() { - if [ -z "${NP_TRACE_DIR:-}" ]; then - NP_TRACE_DIR="${TMPDIR:-/tmp}/nptrace.$$" - fi - export NP_TRACE_DIR - if ! mkdir -p "$NP_TRACE_DIR/nodes" "$NP_TRACE_DIR/staged" \ - "$NP_TRACE_DIR/spool" "$NP_TRACE_DIR/failed" 2>/dev/null; then - NP_TRACE_ENABLED=0 - return 0 - fi - # Prove the tree is actually writable before trusting it. - if ! printf '0' > "$NP_TRACE_DIR/seq.probe" 2>/dev/null; then - NP_TRACE_ENABLED=0 - return 0 - fi - rm -f "$NP_TRACE_DIR/seq.probe" 2>/dev/null || : - if [ ! -f "$NP_TRACE_DIR/seq" ]; then - printf '0' > "$NP_TRACE_DIR/seq" 2>/dev/null || : - fi - return 0 -} - -# Allocate the next handle. Handles are opaque by contract: consumers never -# parse them. -np__handle_new() { - _handle_new_seq=$(cat "$NP_TRACE_DIR/seq" 2>/dev/null || printf '0') - case "$_handle_new_seq" in - '' | *[!0-9]*) _handle_new_seq=0 ;; - esac - _handle_new_seq=$((_handle_new_seq + 1)) - printf '%s' "$_handle_new_seq" > "$NP_TRACE_DIR/seq" - _handle_new_handle="n$_handle_new_seq" - : > "$NP_TRACE_DIR/nodes/$_handle_new_handle" - printf '%s' "$_handle_new_handle" -} - -# THE rule the whole public surface rests on: an argument is a handle iff it -# has the allocator's shape AND names an existing node file. The shape check -# comes first so a caller-supplied string can never traverse out of nodes/. -np__is_handle() { - case "${1:-}" in - n) return 1 ;; - n*) case "${1#n}" in '' | *[!0-9]*) return 1 ;; esac ;; - *) return 1 ;; - esac - [ -f "$NP_TRACE_DIR/nodes/$1" ] -} - -np__node_set() { - _node_set_file="$NP_TRACE_DIR/nodes/$1" - [ -f "$_node_set_file" ] || return 0 - # Drop any prior value for this key, then append the new one. The trailing - # '=' in the match means a key that is a prefix of another never collides. - if grep -q "^$2=" "$_node_set_file" 2>/dev/null; then - grep -v "^$2=" "$_node_set_file" > "$_node_set_file.tmp" 2>/dev/null || : > "$_node_set_file.tmp" - mv "$_node_set_file.tmp" "$_node_set_file" - fi - printf '%s=%s\n' "$2" "$3" >> "$_node_set_file" - return 0 -} - -np__node_get() { - _node_get_file="$NP_TRACE_DIR/nodes/$1" - [ -f "$_node_get_file" ] || return 0 - # Strip only the leading "key=", so a value containing '=' survives intact. - sed -n "s/^$2=//p" "$_node_get_file" 2>/dev/null | head -n 1 - return 0 -} - -# Ambient resolution, exactly two levels. There is deliberately no third, -# session-wide level: that is where concurrent writers race. -# -# 1. NP_TRACE_CURRENT — explicit, and what you export to cross a CI step. -# 2. current.$$ — auto-maintained within one process tree. POSIX $$ -# does not change in a subshell, so a handle created -# inside $(...) is visible to the caller. -np__ambient() { - if [ -n "${NP_TRACE_CURRENT:-}" ]; then - printf '%s' "$NP_TRACE_CURRENT" - return 0 - fi - cat "$NP_TRACE_DIR/current.$$" 2>/dev/null || printf '' - return 0 -} - -np__ambient_set() { - printf '%s' "$1" > "$NP_TRACE_DIR/current.$$" 2>/dev/null || return 0 - return 0 -} - -np__ambient_clear() { - # Only clear when the cleared handle IS current, so terminalizing an outer - # node cannot silently retarget an inner one. - if [ "$(np__ambient)" = "$1" ]; then - rm -f "$NP_TRACE_DIR/current.$$" 2>/dev/null || : - if [ -n "${NP_TRACE_CURRENT:-}" ] && [ "$NP_TRACE_CURRENT" = "$1" ]; then - NP_TRACE_CURRENT='' - fi - fi - return 0 -} - -# Every node-scoped public function starts here: use $1 when it is a handle, -# otherwise fall back to the ambient node. -np__resolve_handle() { - if np__is_handle "${1:-}"; then - printf '%s' "$1" - else - np__ambient - fi - return 0 -} - -# ---- src/spool.sh ---- -# spool.sh — the emit hot path. Every emit is a LOCAL FILE WRITE: the network -# is never touched here, which is what makes API downtime invisible to the -# caller. The spool file's NAME is the event id, so re-POSTing after a crash is -# idempotent — that is recover() for free. - -# np__spool -> prints the event id -np__spool() { - _spool_id=$(np__uuidv7) - _spool_env=$(np__json_obj_raw \ - id "$(np__json_str "$_spool_id")" \ - time "$(np__json_str "$(np__iso8601)")" \ - type "$(np__json_str "$1")" \ - nrn "$(if [ -n "$2" ]; then np__json_str "$2"; fi)" \ - producer "$(np__json_str "${NP_TRACE_PRODUCER:-}")" \ - data "$3") - - _spool_tmp="$NP_TRACE_DIR/spool/$_spool_id.json.tmp" - _spool_final="$NP_TRACE_DIR/spool/$_spool_id.json" - printf '%s' "$_spool_env" > "$_spool_tmp" 2>/dev/null || return 0 - # Create-then-rename: a concurrent flush never sees a half-written envelope. - mv "$_spool_tmp" "$_spool_final" 2>/dev/null || return 0 - printf '%s' "$_spool_id" - return 0 -} - -np__spool_count() { - _spool_count_n=0 - for _spool_count_f in "$NP_TRACE_DIR/spool"/*.json; do - [ -f "$_spool_count_f" ] || continue - _spool_count_n=$((_spool_count_n + 1)) - done - printf '%s' "$_spool_count_n" - return 0 -} - -# ---- src/http.sh ---- -# http.sh — the only module that touches the network. Every request is bounded -# by a connect AND a total timeout, so an unreachable or hanging API can never -# stall the caller. - -NP_TRACE_CONNECT_TIMEOUT="${NP_TRACE_CONNECT_TIMEOUT:-3}" -NP_TRACE_MAX_TIME="${NP_TRACE_MAX_TIME:-10}" -NP_TRACE_DEFAULT_BASE_URL='https://api.nullplatform.com/tracing' -NP_TRACE_DEFAULT_AUTH_URL='https://api.nullplatform.com' - -np__drop() { - printf '%s\t%s\t%s\n' "$(np__iso8601)" "$1" "$2" >> "$NP_TRACE_DIR/drops.log" 2>/dev/null || : - if [ -n "${NP_TRACE_ON_DROP:-}" ]; then - "$NP_TRACE_ON_DROP" "$1" "$2" 2>/dev/null || : - fi - if [ -n "${NP_TRACE_DEBUG:-}" ]; then - printf 'np-trace drop: %s (%s)\n' "$1" "$2" >&2 - fi - return 0 -} - -# Suppress xtrace for a credential-handling region, remembering whether it was -# on. CI scripts routinely `set -x`, and shell options are global — so without -# this a sourced SDK function would print the bearer token into the build log -# even though it never reaches curl's argv. Every credential path is bracketed -# by np__secret_begin / np__secret_end. -np__secret_begin() { - case "$-" in - *x*) NP_TRACE_XTRACE=1; set +x ;; - *) NP_TRACE_XTRACE='' ;; - esac -} - -np__secret_end() { - if [ -n "${NP_TRACE_XTRACE:-}" ]; then - NP_TRACE_XTRACE='' - set -x - fi - return 0 -} - -# A bearer token. A pre-issued NP_TRACE_TOKEN wins; otherwise exchange the api -# key, caching until shortly before expiry. Called LAZILY, at first flush — -# never at init, so a down auth endpoint cannot delay pipeline startup. -np__token() { - np__secret_begin - if [ -n "${NP_TRACE_TOKEN:-}" ]; then - printf '%s' "$NP_TRACE_TOKEN" - np__secret_end - return 0 - fi - np__token_exchange - np__secret_end - return 0 -} - -# The api-key exchange. Always called from inside a secret region. -np__token_exchange() { - if [ -z "${NP_TRACE_API_KEY:-}" ]; then - printf '' - return 0 - fi - - _token_exchange_cache="$NP_TRACE_DIR/token" - if [ -f "$_token_exchange_cache" ]; then - _token_exchange_exp=$(sed -n '1p' "$_token_exchange_cache" 2>/dev/null) - _token_exchange_val=$(sed -n '2p' "$_token_exchange_cache" 2>/dev/null) - case "$_token_exchange_exp" in - '' | *[!0-9]*) _token_exchange_exp=0 ;; - esac - if [ -n "$_token_exchange_val" ] && [ "$_token_exchange_exp" -gt "$(date +%s)" ]; then - printf '%s' "$_token_exchange_val" - return 0 - fi - fi - - _token_exchange_body=$(curl -sS -X POST \ - --connect-timeout "$NP_TRACE_CONNECT_TIMEOUT" --max-time "$NP_TRACE_MAX_TIME" \ - -H 'Content-Type: application/json' \ - -d "$(np__json_obj apiKey "$NP_TRACE_API_KEY")" \ - "${NP_TRACE_AUTH_URL:-$NP_TRACE_DEFAULT_AUTH_URL}/token" 2>/dev/null) || _token_exchange_body='' - - _token_exchange_new=$(printf '%s' "$_token_exchange_body" | - sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p') - if [ -z "$_token_exchange_new" ]; then - np__drop 'auth' 'token exchange failed' - printf '' - return 0 - fi - ( umask 077; printf '%s\n%s\n' "$(( $(date +%s) + 3540 ))" "$_token_exchange_new" > "$_token_exchange_cache" ) - printf '%s' "$_token_exchange_new" - return 0 -} - -# The auth header goes to curl via --config from a mode-600 file, NEVER as -H -# in argv: CI runs with `set -x`, and an argv-borne header prints the token -# straight into the build log. -np__auth_config() { - np__secret_begin - _auth_config_file="$NP_TRACE_DIR/curlcfg.$$" - ( umask 077; printf 'header = "Authorization: Bearer %s"\n' "$(np__token)" > "$_auth_config_file" ) - np__secret_end - printf '%s' "$_auth_config_file" - return 0 -} - -# POST one spool file. Prints the HTTP status code, or 000 on a network failure. -np__post_event() { - _post_event_cfg=$(np__auth_config) - _post_event_code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ - --config "$_post_event_cfg" \ - --connect-timeout "$NP_TRACE_CONNECT_TIMEOUT" --max-time "$NP_TRACE_MAX_TIME" \ - -H 'Content-Type: application/json' \ - --data-binary "@$1" \ - "${NP_TRACE_BASE_URL:-$NP_TRACE_DEFAULT_BASE_URL}/events" 2>/dev/null) || _post_event_code='000' - rm -f "$_post_event_cfg" 2>/dev/null || : - case "$_post_event_code" in - '' | *[!0-9]*) _post_event_code='000' ;; - esac - printf '%s' "$_post_event_code" - return 0 -} - -# ---- src/flush.sh ---- -# flush.sh — the spool drain. Bounded by a wall-clock budget so a dead API can -# never hang process exit; every path returns 0. - -NP_TRACE_FLUSH_TIMEOUT="${NP_TRACE_FLUSH_TIMEOUT:-10}" -NP_TRACE_MAX_RETRIES="${NP_TRACE_MAX_RETRIES:-3}" - -np__attempts_of() { - _attempts_of_n=$(cat "$1.attempts" 2>/dev/null || printf '0') - case "$_attempts_of_n" in - '' | *[!0-9]*) _attempts_of_n=0 ;; - esac - printf '%s' "$_attempts_of_n" -} - -np__fail_event() { - mv "$1" "$NP_TRACE_DIR/failed/" 2>/dev/null || rm -f "$1" 2>/dev/null || : - rm -f "$1.attempts" 2>/dev/null || : - np__drop "${1##*/}" "$2" - return 0 -} - -np_trace_flush() { - [ -n "${NP_TRACE_DIR:-}" ] || return 0 - [ -d "$NP_TRACE_DIR/spool" ] || return 0 - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _flush_deadline=$(( $(date +%s) + NP_TRACE_FLUSH_TIMEOUT )) - - for _flush_file in "$NP_TRACE_DIR/spool"/*.json; do - [ -f "$_flush_file" ] || continue - if [ "$(date +%s)" -ge "$_flush_deadline" ]; then - # Budget spent. Remaining events stay on disk for the next flush or a - # later np_trace_recover; the process exits on time regardless. This is - # the guarantee that a dead API cannot hang a build. - return 0 - fi - - _flush_code=$(np__post_event "$_flush_file") - case "$_flush_code" in - 201 | 200) - # 200 is an idempotent re-POST of an already-accepted event. - rm -f "$_flush_file" "$_flush_file.attempts" 2>/dev/null || : - ;; - 400) - # A contract violation. Never retried — retrying cannot change it. - np__fail_event "$_flush_file" "rejected 400" - ;; - 401 | 403) - rm -f "$NP_TRACE_DIR/token" 2>/dev/null || : - np__fail_event "$_flush_file" "unauthorized $_flush_code" - ;; - *) - _flush_n=$(( $(np__attempts_of "$_flush_file") + 1 )) - if [ "$_flush_n" -gt "$NP_TRACE_MAX_RETRIES" ]; then - np__fail_event "$_flush_file" "gave up after $_flush_n attempts (last status $_flush_code)" - else - printf '%s' "$_flush_n" > "$_flush_file.attempts" 2>/dev/null || : - fi - ;; - esac - done - return 0 -} - -np_trace_shutdown() { - np_trace_flush - if [ -n "${NP_TRACE_DIR:-}" ] && [ "${NP_TRACE_KEEP_STATE:-0}" != '1' ]; then - rm -rf "$NP_TRACE_DIR" 2>/dev/null || : - fi - return 0 -} - -# Re-deliver a previous process's leftover spool. Idempotent by construction: -# the spool file name IS the event id, so the API answers a re-POST with -# 200 duplicate. -np_trace_recover() { - np_trace_flush - return 0 -} - -np__install_trap() { - if [ -z "${NP_TRACE_NO_TRAP:-}" ]; then - trap 'np_trace_flush' EXIT - trap 'np_trace_flush' INT - trap 'np_trace_flush' TERM - fi - return 0 -} - -# ---- src/propagation.sh ---- -# --------------------------------------------------------------------------- -# Propagation -# -# Cross-process trace context, wire-identical to the Go and JS SDKs: a single -# carrier value packing "||". The '|' delimiter is -# reserved, so the value splits unambiguously even though a run_id may itself -# contain '~' and '@'. -# -# The carrier travels in the NP_TRACE environment variable. Note that this is -# deliberately OUTSIDE the NP_TRACE_* configuration namespace the SDK reads for -# its own settings: NP_TRACE is context handed to us by a caller, not something -# a user configures. -# --------------------------------------------------------------------------- - -# np_trace_inject [handle] -# -# Print the carrier value for a handle (defaults to the ambient node), for -# handing to a child process. Prints nothing when there is no node to inject, -# so `NP_TRACE=$(np_trace_inject)` is always safe. -np_trace_inject() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _inject_h=$(np__resolve_handle "${1:-}") - np__is_handle "$_inject_h" || return 0 - printf '%s%s%s%s%s' \ - "$NP_CARRIER_VERSION" "$NP_CARRIER_DELIMITER" \ - "$(np__node_get "$_inject_h" trace_id)" "$NP_CARRIER_DELIMITER" \ - "$(np__node_get "$_inject_h" run_id)" - return 0 -} - -# np_trace_extract [carrier] -# -# Parse a carrier value (defaults to $NP_TRACE) and print " ". -# Returns 1 when there is no usable context, so callers can branch: -# -# if ctx=$(np_trace_extract); then set -- $ctx; fi -# -# When only a trace id is present it is used for both, matching the Go SDK, so -# the result is always a usable pair. -np_trace_extract() { - _extract_raw=${1-${NP_TRACE:-}} - [ -n "$_extract_raw" ] || return 1 - - case "$_extract_raw" in - "$NP_CARRIER_VERSION$NP_CARRIER_DELIMITER"*) ;; - *) return 1 ;; - esac - _extract_rest=${_extract_raw#*"$NP_CARRIER_DELIMITER"} - - # trace_id is up to the next delimiter; run_id is the whole remainder, which - # may itself contain '~' and '@' but never a delimiter. - case "$_extract_rest" in - *"$NP_CARRIER_DELIMITER"*) - _extract_trace=${_extract_rest%%"$NP_CARRIER_DELIMITER"*} - _extract_run=${_extract_rest#*"$NP_CARRIER_DELIMITER"} - ;; - *) - _extract_trace=$_extract_rest - _extract_run=$_extract_rest - ;; - esac - [ -n "$_extract_trace" ] || return 1 - [ -n "$_extract_run" ] || _extract_run=$_extract_trace - - printf '%s %s' "$_extract_trace" "$_extract_run" - return 0 -} - -# np_trace_adopt [carrier] -# -# Attach to an upstream node and return a handle standing in for it, so work -# started here nests UNDERNEATH it: -# -# parent=$(np_trace_adopt) || parent=$(np_trace_run --run-id "$(np_trace_occurrence)") -# step=$(np_trace_step "$parent" build) -# -# The adopted node belongs to whoever created it — typically the np CLI, which -# exports NP_TRACE per workflow step. We hold its ids so children derive -# correctly, but must never speak for it: it is marked foreign, so it emits no -# node event of its own and the terminal verbs refuse to close it. Children -# hanging off it still emit their own containment edges, which IS ours to say. -# -# Returns 1 when there is no upstream context, leaving the caller to open a root -# run instead. -np_trace_adopt() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 1 - _adopt_ctx=$(np_trace_extract "${1-${NP_TRACE:-}}") || return 1 - _adopt_trace=${_adopt_ctx%% *} - _adopt_run=${_adopt_ctx#* } - - if ! _adopt_why=$(np__trace_id_violation "$_adopt_trace"); then - np__drop 'adopt' "trace_id $_adopt_why" - return 1 - fi - # An upstream run_id is commonly a DERIVED path (parent~key@attempt.iteration) - # rather than a named id — the np CLI hands us the step it is running. Accept - # either: parse it as a node path first, and only fall back to the named-id - # rules when it has no delimiter. - if ! np__parse_node_id "$_adopt_run" >/dev/null 2>&1; then - if ! _adopt_why=$(np__named_id_violation "$_adopt_run"); then - np__drop 'adopt' "run_id $_adopt_why" - return 1 - fi - fi - - _adopt_h=$(np__handle_new) - np__node_set "$_adopt_h" kind run - np__node_set "$_adopt_h" trace_id "$_adopt_trace" - np__node_set "$_adopt_h" run_id "$_adopt_run" - np__node_set "$_adopt_h" nrn "${NP_TRACE_NRN:-}" - np__node_set "$_adopt_h" foreign 1 - # started=1 suppresses the lazy `started` emit; closed=0 keeps it usable as a - # parent for the whole script. - np__node_set "$_adopt_h" started 1 - np__node_set "$_adopt_h" closed 0 - np__ambient_set "$_adopt_h" - printf '%s' "$_adopt_h" - return 0 -} - -# True when a handle stands in for a node owned by another process. -np__is_foreign() { - [ "$(np__node_get "$1" foreign)" = '1' ] -} - -# ---- src/api.sh ---- -# api.sh — the public producer surface. Every function here returns 0, always: -# tracing must never fail the caller. -# -# Every node-scoped function takes an OPTIONAL leading handle. This is one -# function with a defaulted argument, not two ways to say the same thing: when -# the first argument is not a handle it falls back to the innermost open node. - -np_trace_init() { - while [ "$#" -gt 0 ]; do - case "$1" in - --producer) NP_TRACE_PRODUCER=${2:-}; shift 2 ;; - --base-url) NP_TRACE_BASE_URL=${2:-}; shift 2 ;; - --auth-url) NP_TRACE_AUTH_URL=${2:-}; shift 2 ;; - --api-key) NP_TRACE_API_KEY=${2:-}; shift 2 ;; - --token) NP_TRACE_TOKEN=${2:-}; shift 2 ;; - --nrn) NP_TRACE_NRN=${2:-}; shift 2 ;; - --enabled) NP_TRACE_ENABLED=${2:-1}; shift 2 ;; - --no-trap) NP_TRACE_NO_TRAP=1; shift ;; - *) shift ;; - esac - done - NP_TRACE_ENABLED="${NP_TRACE_ENABLED:-1}" - np__state_init - # No network call here, deliberately: a down auth endpoint must never delay - # the start of a pipeline. The token is fetched lazily, at first flush. - np__install_trap - return 0 -} - -# --------------------------------------------------------------------------- -# Emission -# --------------------------------------------------------------------------- - -# Emit the node event for a handle at the given status, carrying whatever -# context is currently staged. -np__emit_node() { - _emit_node_h=$1 - _emit_node_status=$2 - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - - _emit_node_labels=$(np__node_get "$_emit_node_h" labels) - _emit_node_facets=$(np__node_get "$_emit_node_h" facets) - _emit_node_key=$(np__node_get "$_emit_node_h" key) - _emit_node_schema=$(np__node_get "$_emit_node_h" schema_url) - - if [ -n "$_emit_node_key" ]; then - _emit_node_data=$(np__json_obj_raw \ - trace_id "$(np__json_str "$(np__node_get "$_emit_node_h" trace_id)")" \ - run_id "$(np__json_str "$(np__node_get "$_emit_node_h" run_id)")" \ - key "$(np__json_str "$_emit_node_key")" \ - attempt "$(np__node_get "$_emit_node_h" attempt)" \ - iteration "$(np__node_get "$_emit_node_h" iteration)" \ - status "$(np__json_str "$_emit_node_status")" \ - labels "$_emit_node_labels" \ - facets "$_emit_node_facets" \ - schema_url "$(if [ -n "$_emit_node_schema" ]; then np__json_str "$_emit_node_schema"; fi)") - else - _emit_node_data=$(np__json_obj_raw \ - trace_id "$(np__json_str "$(np__node_get "$_emit_node_h" trace_id)")" \ - run_id "$(np__json_str "$(np__node_get "$_emit_node_h" run_id)")" \ - status "$(np__json_str "$_emit_node_status")" \ - labels "$_emit_node_labels" \ - facets "$_emit_node_facets" \ - schema_url "$(if [ -n "$_emit_node_schema" ]; then np__json_str "$_emit_node_schema"; fi)") - fi - - np__spool "$NP_TYPE_NODE_RUN" "$(np__node_get "$_emit_node_h" nrn)" "$_emit_node_data" >/dev/null - return 0 -} - -# A run ref for a handle — the self-describing address used on edge endpoints. -np__ref_of() { - np__json_obj \ - type run \ - trace_id "$(np__node_get "$1" trace_id)" \ - run_id "$(np__node_get "$1" run_id)" -} - -np__emit_parent_edge() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _emit_parent_edge_data=$(np__json_obj_raw from "$(np__ref_of "$1")" to "$(np__ref_of "$2")") - np__spool "$NP_TYPE_EDGE_PARENT" "$(np__node_get "$1" nrn)" "$_emit_parent_edge_data" >/dev/null - return 0 -} - -# Force the lazy `started`. Idempotent. -# -# Shell has no microtask, so `started` is emitted at the first event that must -# follow it — a terminal, a child open, an explicit call, or flush. Context -# staged before that lands on `started`; context staged after lands on the -# terminal. Same observable semantics as the JS and Go SDKs, without a timer. -np_trace_start() { - _start_h=$(np__resolve_handle "${1:-}") - np__is_handle "$_start_h" || return 0 - if [ "$(np__node_get "$_start_h" started)" = '1' ]; then - return 0 - fi - np__node_set "$_start_h" started 1 - np__emit_node "$_start_h" "$NP_STATUS_STARTED" - return 0 -} - -# --------------------------------------------------------------------------- -# Nodes -# --------------------------------------------------------------------------- - -np_trace_run() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _run_trace='' - _run_run='' - _run_nrn="${NP_TRACE_NRN:-}" - while [ "$#" -gt 0 ]; do - case "$1" in - --trace-id) _run_trace=${2:-}; shift 2 ;; - --run-id) _run_run=${2:-}; shift 2 ;; - --nrn) _run_nrn=${2:-}; shift 2 ;; - *) shift ;; - esac - done - # A lone root run's trace_id defaults to its run_id, and vice versa. - [ -n "$_run_trace" ] || _run_trace=$_run_run - [ -n "$_run_run" ] || _run_run=$_run_trace - - if ! _run_why=$(np__trace_id_violation "$_run_trace"); then - np__drop 'run' "trace_id $_run_why" - return 0 - fi - if ! _run_why=$(np__named_id_violation "$_run_run"); then - np__drop 'run' "run_id $_run_why" - return 0 - fi - - _run_h=$(np__handle_new) - np__node_set "$_run_h" kind run - np__node_set "$_run_h" trace_id "$_run_trace" - np__node_set "$_run_h" run_id "$_run_run" - np__node_set "$_run_h" nrn "$_run_nrn" - np__node_set "$_run_h" auto_started_at "$(np__iso8601)" - np__node_set "$_run_h" started 0 - np__node_set "$_run_h" closed 0 - np__ambient_set "$_run_h" - printf '%s' "$_run_h" - return 0 -} - -# np_trace_step [handle] [--attempt N] [--iteration N] -np_trace_step() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _step_parent=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - _step_key=${1:-} - if [ "$#" -gt 0 ]; then - shift - fi - _step_attempt=0 - _step_iteration=0 - while [ "$#" -gt 0 ]; do - case "$1" in - --attempt) _step_attempt=${2:-0}; shift 2 ;; - --iteration) _step_iteration=${2:-0}; shift 2 ;; - *) shift ;; - esac - done - - if ! np__is_handle "$_step_parent"; then - np__drop 'step' 'no parent node in scope' - return 0 - fi - if ! _step_why=$(np__key_violation "$_step_key"); then - np__drop 'step' "key $_step_why" - return 0 - fi - case "$_step_attempt$_step_iteration" in - '' | *[!0-9]*) np__drop 'step' 'attempt and iteration must be integers'; return 0 ;; - esac - - # Opening a child forces the parent's started: a parent edge must not point - # at a node the read model has never seen. - np_trace_start "$_step_parent" - - _step_id=$(np__derive_child_id "$(np__node_get "$_step_parent" run_id)" \ - "$_step_key" "$_step_attempt" "$_step_iteration") - - _step_h=$(np__handle_new) - np__node_set "$_step_h" kind step - np__node_set "$_step_h" trace_id "$(np__node_get "$_step_parent" trace_id)" - np__node_set "$_step_h" run_id "$_step_id" - np__node_set "$_step_h" nrn "$(np__node_get "$_step_parent" nrn)" - np__node_set "$_step_h" key "$_step_key" - np__node_set "$_step_h" attempt "$_step_attempt" - np__node_set "$_step_h" iteration "$_step_iteration" - np__node_set "$_step_h" parent "$_step_parent" - np__node_set "$_step_h" auto_started_at "$(np__iso8601)" - np__node_set "$_step_h" started 0 - np__node_set "$_step_h" closed 0 - - np_trace_start "$_step_h" - np__emit_parent_edge "$_step_parent" "$_step_h" - np__ambient_set "$_step_h" - printf '%s' "$_step_h" - return 0 -} - -# A named child run — a new scope under the same trace. -np_trace_child() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _child_parent=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - _child_run='' - while [ "$#" -gt 0 ]; do - case "$1" in - --run-id) _child_run=${2:-}; shift 2 ;; - *) shift ;; - esac - done - if ! np__is_handle "$_child_parent"; then - np__drop 'child' 'no parent node in scope' - return 0 - fi - if ! _child_why=$(np__named_id_violation "$_child_run"); then - np__drop 'child' "run_id $_child_why" - return 0 - fi - np_trace_start "$_child_parent" - _child_h=$(np_trace_run --trace-id "$(np__node_get "$_child_parent" trace_id)" \ - --run-id "$_child_run" \ - --nrn "$(np__node_get "$_child_parent" nrn)") - np__is_handle "$_child_h" || return 0 - np__node_set "$_child_h" parent "$_child_parent" - np_trace_start "$_child_h" - np__emit_parent_edge "$_child_parent" "$_child_h" - np__ambient_set "$_child_h" - printf '%s' "$_child_h" - return 0 -} - -# --------------------------------------------------------------------------- -# Staging context -# --------------------------------------------------------------------------- - -# Merge a pre-formed `"key":value` fragment into the node's staged labels. -np__stage_label() { - _stage_label_cur=$(np__node_get "$1" labels) - if [ -z "$_stage_label_cur" ] || [ "$_stage_label_cur" = '{}' ]; then - np__node_set "$1" labels "{$2}" - else - np__node_set "$1" labels "${_stage_label_cur%\}},$2}" - fi - return 0 -} - -np__stage_facet() { - _stage_facet_cur=$(np__node_get "$1" facets) - _stage_facet_entry="$(np__json_str "$2"):$3" - if [ -z "$_stage_facet_cur" ] || [ "$_stage_facet_cur" = '{}' ]; then - np__node_set "$1" facets "{$_stage_facet_entry}" - else - # Last write wins per namespace: drop any prior entry for this facet. - np__node_set "$1" facets "${_stage_facet_cur%\}},$_stage_facet_entry}" - fi - return 0 -} - -# Staged context normally rides the node's NEXT lifecycle emit. A FOREIGN -# (adopted) node never has one here — its owner closes it in another process — -# so anything staged on it would die in local state. Re-emit `started` with the -# full current bag instead (additive, the same shape the JS SDK's -# late-enrichment flush produces): the fold keeps the node's real outcome (the -# owner's terminal is later by time) and gains the facts this process observed. -np__flush_foreign() { - [ "$(np__node_get "$1" foreign)" = '1' ] || return 0 - np__emit_node "$1" "$NP_STATUS_STARTED" - return 0 -} - -# np_trace_labels [handle] key=value ... -np_trace_labels() { - _labels_h=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_labels_h" || return 0 - for _labels_pair in "$@"; do - case "$_labels_pair" in - *=*) ;; - *) continue ;; - esac - _labels_k=${_labels_pair%%=*} - _labels_v=${_labels_pair#*=} - # An absent optional is omitted, never recorded as the string "null". - if [ -n "$_labels_k" ] && [ -n "$_labels_v" ]; then - np__stage_label "$_labels_h" "$(np__json_str "$_labels_k"):$(np__json_str "$_labels_v")" - fi - done - np__flush_foreign "$_labels_h" - return 0 -} - -# np_trace_facet [handle] — your own namespace. -np_trace_facet() { - _facet_h=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_facet_h" || return 0 - if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then - return 0 - fi - np__stage_facet "$_facet_h" "$1" "$2" - np__flush_foreign "$_facet_h" - return 0 -} - -np_trace_schema() { - _schema_h=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_schema_h" || return 0 - np__node_set "$_schema_h" schema_url "${1:-}" - return 0 -} - -np_trace_explain() { - _explain_h=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_explain_h" || return 0 - _explain_title='' - _explain_what='' - _explain_why='' - _explain_impact='' - _explain_next='' - _explain_sev='' - while [ "$#" -gt 0 ]; do - case "$1" in - --title) _explain_title=${2:-}; shift 2 ;; - --what) _explain_what=${2:-}; shift 2 ;; - --why) _explain_why=${2:-}; shift 2 ;; - --impact) _explain_impact=${2:-}; shift 2 ;; - --next) _explain_next=${2:-}; shift 2 ;; - --severity) _explain_sev=${2:-}; shift 2 ;; - *) shift ;; - esac - done - if [ -z "$_explain_title" ]; then - np__drop 'explain' 'title is required' - return 0 - fi - np__stage_facet "$_explain_h" "$NP_FACET_EXPLAIN" \ - "$(np__json_obj title "$_explain_title" severity "$_explain_sev" what "$_explain_what" \ - why "$_explain_why" impact "$_explain_impact" next "$_explain_next")" - np__flush_foreign "$_explain_h" - return 0 -} - -np_trace_error() { - _error_h=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_error_h" || return 0 - _error_msg='' - _error_code='' - _error_stack='' - _error_details='' - while [ "$#" -gt 0 ]; do - case "$1" in - --message) _error_msg=${2:-}; shift 2 ;; - --code) _error_code=${2:-}; shift 2 ;; - --stack-trace) _error_stack=${2:-}; shift 2 ;; - # A JSON object with the diagnosis's structured evidence (counts, the - # failing probe, ...) — the sibling SDKs' error `details`. - --details) _error_details=${2:-}; shift 2 ;; - *) - if [ -z "$_error_msg" ]; then - _error_msg=$1 - fi - shift - ;; - esac - done - [ -n "$_error_msg" ] || return 0 - case "$_error_details" in - '' | \{*) ;; - *) _error_details='' ;; - esac - np__stage_facet "$_error_h" "$NP_FACET_ERROR" \ - "$(np__json_obj_raw \ - message "$(np__json_str "$_error_msg")" \ - code "$(if [ -n "$_error_code" ]; then np__json_str "$_error_code"; fi)" \ - stack_trace "$(if [ -n "$_error_stack" ]; then np__json_str "$_error_stack"; fi)" \ - details "$_error_details")" - np__flush_foreign "$_error_h" - return 0 -} - -np_trace_timing() { - _timing_h=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_timing_h" || return 0 - while [ "$#" -gt 0 ]; do - case "$1" in - --started-at) np__node_set "$_timing_h" started_at "${2:-}"; shift 2 ;; - --ended-at) np__node_set "$_timing_h" ended_at "${2:-}"; shift 2 ;; - *) shift ;; - esac - done - return 0 -} - -# Stamp the auto timing facet, letting any manual override win per field. -np__stage_timing() { - _stage_timing_started=$(np__node_get "$1" started_at) - _stage_timing_ended=$(np__node_get "$1" ended_at) - [ -n "$_stage_timing_started" ] || _stage_timing_started=$(np__node_get "$1" auto_started_at) - [ -n "$_stage_timing_ended" ] || _stage_timing_ended=$2 - np__stage_facet "$1" "$NP_FACET_TIMING" \ - "$(np__json_obj started_at "$_stage_timing_started" ended_at "$_stage_timing_ended")" - return 0 -} - -# --------------------------------------------------------------------------- -# Lineage — produces/consumes edges with io pointers -# --------------------------------------------------------------------------- - -# A dataset ref for an edge endpoint. The id is the CANONICAL dataset id — the -# exact string a producer and a consumer must both name for lineage to join -# them by value (an ARN, an FQDN, `:` for an asset) — never a -# synthesised id. -np__dataset_ref() { - np__json_obj type dataset id "$1" -} - -# Append one io descriptor to a direction's list; the facet is re-staged -# whole each time (last write wins per namespace), so the array only ever -# grows. $1 handle, $2 facet namespace, $3 descriptor store key, $4 the -# already-formed descriptor JSON. -np__append_io_descriptor() { - _append_io_descriptor_descriptors=$(np__node_get "$1" "$3") - if [ -n "$_append_io_descriptor_descriptors" ]; then - _append_io_descriptor_descriptors="$_append_io_descriptor_descriptors,$4" - else - _append_io_descriptor_descriptors=$4 - fi - np__node_set "$1" "$3" "$_append_io_descriptor_descriptors" - np__stage_facet "$1" "$2" "[$_append_io_descriptor_descriptors]" - return 0 -} - -# Build one io descriptor from its parsed parts, choosing the kind by which -# parts are present: a uri is a POINTER (large data referenced, not inlined), -# a source+external-id is a REF (an entity in an external catalog), a JSON -# value is INLINE (carried in the event itself). Prints the descriptor, or -# nothing (with a drop) when the parts don't form one. -# $1 verb (for drop records), $2 name, $3 inline JSON, $4 uri, $5 ref source, -# $6 ref external id, $7 ref version. -np__build_io_descriptor() { - _build_io_descriptor_verb=$1 - _build_io_descriptor_name=$2 - _build_io_descriptor_inline=$3 - _build_io_descriptor_uri=$4 - _build_io_descriptor_ref_source=$5 - _build_io_descriptor_ref_id=$6 - _build_io_descriptor_ref_version=$7 - if [ -z "$_build_io_descriptor_name" ]; then - np__drop "$_build_io_descriptor_verb" 'a descriptor name is required' - return 1 - fi - if [ -n "$_build_io_descriptor_uri" ]; then - np__json_obj kind pointer name "$_build_io_descriptor_name" uri "$_build_io_descriptor_uri" - return 0 - fi - if [ -n "$_build_io_descriptor_ref_source" ] && [ -n "$_build_io_descriptor_ref_id" ]; then - np__json_obj kind ref name "$_build_io_descriptor_name" source "$_build_io_descriptor_ref_source" \ - external_id "$_build_io_descriptor_ref_id" version "$_build_io_descriptor_ref_version" - return 0 - fi - if [ -n "$_build_io_descriptor_inline" ]; then - case "$_build_io_descriptor_inline" in - \{* | \[* | \"* | [0-9-]* | true | false | null) - np__json_obj_raw kind '"inline"' name "$(np__json_str "$_build_io_descriptor_name")" value "$_build_io_descriptor_inline" - return 0 - ;; - esac - np__drop "$_build_io_descriptor_verb" 'value must be JSON' - return 1 - fi - np__drop "$_build_io_descriptor_verb" 'a JSON value, --uri, or --source + --external-id is required' - return 1 -} - -# The shared body of np_trace_output / np_trace_input. -# $1 direction (out|in), $2 verb, then the caller's argv: -# [handle] [] [--uri U] [--source S --external-id E [--version V]] -np__declare_io() { - _declare_io_direction=$1 - _declare_io_verb=$2 - shift 2 - _declare_io_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_declare_io_handle" || { np__drop "$_declare_io_verb" 'no node in scope'; return 0; } - _declare_io_name=${1:-} - if [ "$#" -gt 0 ]; then - shift - fi - _declare_io_inline='' - _declare_io_uri='' - _declare_io_ref_source='' - _declare_io_ref_id='' - _declare_io_ref_version='' - while [ "$#" -gt 0 ]; do - case "$1" in - --uri) _declare_io_uri=${2:-}; shift 2 ;; - --source) _declare_io_ref_source=${2:-}; shift 2 ;; - --external-id) _declare_io_ref_id=${2:-}; shift 2 ;; - --version) _declare_io_ref_version=${2:-}; shift 2 ;; - *) - if [ -z "$_declare_io_inline" ]; then - _declare_io_inline=$1 - fi - shift - ;; - esac - done - _declare_io_descriptor=$(np__build_io_descriptor "$_declare_io_verb" "$_declare_io_name" "$_declare_io_inline" \ - "$_declare_io_uri" "$_declare_io_ref_source" "$_declare_io_ref_id" "$_declare_io_ref_version") || return 0 - if [ "$_declare_io_direction" = 'out' ]; then - np__append_io_descriptor "$_declare_io_handle" "$NP_FACET_OUTPUT" io_output "$_declare_io_descriptor" - else - np__append_io_descriptor "$_declare_io_handle" "$NP_FACET_INPUT" io_input "$_declare_io_descriptor" - fi - np__flush_foreign "$_declare_io_handle" - return 0 -} - -# np_trace_output [handle] [] [--uri U] [--source S --external-id E [--version V]] -# -# Record what this node PRODUCED: an inline value carried in the event -# (`np_trace_output instances '{"healthy":2}'`), a pointer to large data -# (`--uri`), or a ref to an external catalog entity (`--source`/`--external-id`). -# For an artifact that should ALSO join the lineage graph, prefer -# np_trace_produces (descriptor + edge in one call). -np_trace_output() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - np__declare_io out output "$@" - return 0 -} - -# np_trace_input [handle] [] [--uri U] [--source S --external-id E [--version V]] -# -# Record what this node CONSUMED; see np_trace_output. -np_trace_input() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - np__declare_io in input "$@" - return 0 -} - -# np__emit_io_edge [descriptor-json] -# -# Emit one lineage edge. The direction decides everything else: `out` is -# edge.produces + tracing.output, `in` is edge.consumes + tracing.input. -# -# With a descriptor the io is declared ONCE: it accumulates into the node's -# io facet AND becomes the edge's tracing.binding — the same single-source -# rule as the sibling SDKs. Without one, the edge records lineage only. -# -# On a FOREIGN (adopted) node this is an observed fact, exactly like -# np_trace_error: the edge is ours to say, and the staged io facet reaches the -# wire through the foreign re-emit. -np__emit_io_edge() { - _emit_io_edge_handle=$1 - _emit_io_edge_direction=$2 - _emit_io_edge_dataset_id=$3 - - if [ "$_emit_io_edge_direction" = 'out' ]; then - _emit_io_edge_edge_type=$NP_TYPE_EDGE_PRODUCES - _emit_io_edge_facet_namespace=$NP_FACET_OUTPUT - _emit_io_edge_descriptor_store=io_output - else - _emit_io_edge_edge_type=$NP_TYPE_EDGE_CONSUMES - _emit_io_edge_facet_namespace=$NP_FACET_INPUT - _emit_io_edge_descriptor_store=io_input - fi - - _emit_io_edge_binding=$4 - - if [ -n "$_emit_io_edge_binding" ]; then - np__append_io_descriptor "$_emit_io_edge_handle" "$_emit_io_edge_facet_namespace" "$_emit_io_edge_descriptor_store" "$_emit_io_edge_binding" - fi - - # An edge must not point FROM a node the read model has never seen. - np_trace_start "$_emit_io_edge_handle" - - if [ -n "$_emit_io_edge_binding" ]; then - _emit_io_edge_edge_data=$(np__json_obj_raw \ - from "$(np__ref_of "$_emit_io_edge_handle")" \ - to "$(np__dataset_ref "$_emit_io_edge_dataset_id")" \ - facets "{$(np__json_str "$NP_FACET_BINDING"):$_emit_io_edge_binding}") - else - _emit_io_edge_edge_data=$(np__json_obj_raw \ - from "$(np__ref_of "$_emit_io_edge_handle")" \ - to "$(np__dataset_ref "$_emit_io_edge_dataset_id")") - fi - np__spool "$_emit_io_edge_edge_type" "$(np__node_get "$_emit_io_edge_handle" nrn)" "$_emit_io_edge_edge_data" >/dev/null - np__flush_foreign "$_emit_io_edge_handle" - return 0 -} - -# The shared argv handling of np_trace_produces / np_trace_consumes: -# resolve the optional leading handle, take the dataset id, parse the -# pointer flags, and hand off to np__emit_io_edge. -# $1 direction (out|in), $2 verb name for drop records, then the caller's argv. -np__declare_lineage() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _declare_lineage_direction=$1 - _declare_lineage_verb=$2 - shift 2 - - _declare_lineage_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_declare_lineage_handle" || { np__drop "$_declare_lineage_verb" 'no node in scope'; return 0; } - - _declare_lineage_dataset_id=${1:-} - if [ "$#" -gt 0 ]; then - shift - fi - if [ -z "$_declare_lineage_dataset_id" ]; then - np__drop "$_declare_lineage_verb" 'dataset id is required' - return 0 - fi - - _declare_lineage_name='' - _declare_lineage_inline='' - _declare_lineage_uri='' - _declare_lineage_ref_source='' - _declare_lineage_ref_id='' - _declare_lineage_ref_version='' - while [ "$#" -gt 0 ]; do - case "$1" in - --name) _declare_lineage_name=${2:-}; shift 2 ;; - --uri) _declare_lineage_uri=${2:-}; shift 2 ;; - --value) _declare_lineage_inline=${2:-}; shift 2 ;; - --source) _declare_lineage_ref_source=${2:-}; shift 2 ;; - --external-id) _declare_lineage_ref_id=${2:-}; shift 2 ;; - --version) _declare_lineage_ref_version=${2:-}; shift 2 ;; - *) shift ;; - esac - done - - _declare_lineage_binding='' - if [ -n "$_declare_lineage_name" ]; then - _declare_lineage_binding=$(np__build_io_descriptor "$_declare_lineage_verb" "$_declare_lineage_name" "$_declare_lineage_inline" \ - "$_declare_lineage_uri" "$_declare_lineage_ref_source" "$_declare_lineage_ref_id" "$_declare_lineage_ref_version") || return 0 - fi - - np__emit_io_edge "$_declare_lineage_handle" "$_declare_lineage_direction" "$_declare_lineage_dataset_id" "$_declare_lineage_binding" - return 0 -} - -# np_trace_produces [handle] [--name (--uri U | --value JSON | --source S --external-id E [--version V])] -# -# Declare this node WROTE the dataset. With `--name` the io is declared once -# — a pointer (`--uri`, the artifact's address), an inline value (`--value`), -# or a catalog ref (`--source`/`--external-id`) — on both the node and the -# edge's binding. Bare form records lineage only. -np_trace_produces() { - np__declare_lineage out produces "$@" - return 0 -} - -# np_trace_consumes [handle] [--name (--uri U | --value JSON | --source S --external-id E [--version V])] -# -# Declare this node READ the dataset; see np_trace_produces. -np_trace_consumes() { - np__declare_lineage in consumes "$@" - return 0 -} - -# --------------------------------------------------------------------------- -# Run-to-run edges — how operations relate across the graph -# --------------------------------------------------------------------------- - -# Resolve an edge target: a handle from this process, or a PACKED CARRIER -# ("1||") — the natural address in shell, where the other -# end of an edge usually arrived via an env var. Prints the target's ref. -np__edge_target_ref() { - if np__is_handle "$1"; then - np__ref_of "$1" - return 0 - fi - _edge_target_ref_context=$(np_trace_extract "$1") || return 1 - _edge_target_ref_trace=${_edge_target_ref_context%% *} - _edge_target_ref_run=${_edge_target_ref_context#* } - np__json_obj type run trace_id "$_edge_target_ref_trace" run_id "$_edge_target_ref_run" - return 0 -} - -# Emit one relationship edge from a node this process holds. -# $1 handle, $2 edge type, $3 target ref JSON, $4 verb for drop records. -np__emit_ref_edge() { - _emit_ref_edge_from=$(np__ref_of "$1") - if [ "$_emit_ref_edge_from" = "$3" ]; then - np__drop "$4" 'self-edge forbidden' - return 0 - fi - # An edge must not point FROM a node the read model has never seen. - np_trace_start "$1" - _emit_ref_edge_data=$(np__json_obj_raw from "$_emit_ref_edge_from" to "$3") - np__spool "$2" "$(np__node_get "$1" nrn)" "$_emit_ref_edge_data" >/dev/null - np__flush_foreign "$1" - return 0 -} - -# The shared argv handling of the run-to-run edge verbs. -# $1 edge type, $2 verb, then the caller's argv: [handle] . -np__declare_relation() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _declare_relation_type=$1 - _declare_relation_verb=$2 - shift 2 - _declare_relation_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_declare_relation_handle" || { np__drop "$_declare_relation_verb" 'no node in scope'; return 0; } - if [ -z "${1:-}" ]; then - np__drop "$_declare_relation_verb" 'a target (handle or packed carrier) is required' - return 0 - fi - _declare_relation_target=$(np__edge_target_ref "$1") || { - np__drop "$_declare_relation_verb" 'target is not a handle or a valid carrier' - return 0 - } - np__emit_ref_edge "$_declare_relation_handle" "$_declare_relation_type" "$_declare_relation_target" "$_declare_relation_verb" - return 0 -} - -# np_trace_triggered_by [handle] -# -# The operation that CAUSED this one — a cross-trace fact (the target is -# usually another trace's run, addressed by its packed carrier). -np_trace_triggered_by() { - np__declare_relation "$NP_TYPE_EDGE_TRIGGERED_BY" triggered_by "$@" - return 0 -} - -# np_trace_retry_of [handle] — this run retries that one. -np_trace_retry_of() { - np__declare_relation "$NP_TYPE_EDGE_RETRY_OF" retry_of "$@" - return 0 -} - -# np_trace_continues [handle] — this run resumes that one's work. -np_trace_continues() { - np__declare_relation "$NP_TYPE_EDGE_CONTINUES" continues "$@" - return 0 -} - -# np_trace_correlates [handle] — related, with no causal claim. -np_trace_correlates() { - np__declare_relation "$NP_TYPE_EDGE_CORRELATES" correlates "$@" - return 0 -} - -# np_trace_compensates [handle] — this run undoes that one's effect. -np_trace_compensates() { - np__declare_relation "$NP_TYPE_EDGE_COMPENSATES" compensates "$@" - return 0 -} - -# np_trace_link [handle] -# -# Escape hatch over the named verbs — emit any known edge type. Prefer the -# named functions when one fits. -np_trace_link() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _link_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_link_handle" || { np__drop 'link' 'no node in scope'; return 0; } - _link_type=${1:-} - case "$_link_type" in - "$NP_TYPE_EDGE_TRIGGERED_BY" | "$NP_TYPE_EDGE_RETRY_OF" | "$NP_TYPE_EDGE_CONTINUES" \ - | "$NP_TYPE_EDGE_CORRELATES" | "$NP_TYPE_EDGE_COMPENSATES" | "$NP_TYPE_EDGE_PARENT") ;; - *) np__drop 'link' "unknown edge type '${_link_type}'"; return 0 ;; - esac - if [ -z "${2:-}" ]; then - np__drop 'link' 'a target (handle or packed carrier) is required' - return 0 - fi - _link_target=$(np__edge_target_ref "$2") || { - np__drop 'link' 'target is not a handle or a valid carrier' - return 0 - } - np__emit_ref_edge "$_link_handle" "$_link_type" "$_link_target" link - return 0 -} - -# np_trace_instance_of [handle] [--nrn N] -# -# This run instantiates a reusable JOB definition — the read model resolves -# the run's plan from the definition. Emit the definition itself with -# np_trace_job. -np_trace_instance_of() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _instance_of_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_instance_of_handle" || { np__drop 'instance_of' 'no node in scope'; return 0; } - _instance_of_namespace=${1:-} - _instance_of_name=${2:-} - _instance_of_version=${3:-} - if [ "$#" -ge 3 ]; then - shift 3 - fi - _instance_of_nrn='' - while [ "$#" -gt 0 ]; do - case "$1" in - --nrn) _instance_of_nrn=${2:-}; shift 2 ;; - *) shift ;; - esac - done - if [ -z "$_instance_of_namespace" ] || [ -z "$_instance_of_name" ] || [ -z "$_instance_of_version" ]; then - np__drop 'instance_of' 'namespace, name and version are required' - return 0 - fi - _instance_of_target=$(np__json_obj type job namespace "$_instance_of_namespace" \ - name "$_instance_of_name" version "$_instance_of_version" nrn "$_instance_of_nrn") - np__emit_ref_edge "$_instance_of_handle" "$NP_TYPE_EDGE_INSTANCE_OF" "$_instance_of_target" instance_of - return 0 -} - -# --------------------------------------------------------------------------- -# Definition nodes — identities, not executions -# --------------------------------------------------------------------------- - -# np_trace_dataset [--nrn N] -# -# Emit a dataset node — an identity a lineage edge can point at. The id is -# the CANONICAL address (see np_trace_produces); edges to an unemitted -# dataset still resolve, so this is only needed to carry the node itself. -np_trace_dataset() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _dataset_id=${1:-} - if [ "$#" -gt 0 ]; then - shift - fi - _dataset_nrn='' - while [ "$#" -gt 0 ]; do - case "$1" in - --nrn) _dataset_nrn=${2:-}; shift 2 ;; - *) shift ;; - esac - done - if [ -z "$_dataset_id" ]; then - np__drop 'dataset' 'an id is required' - return 0 - fi - np__spool "$NP_TYPE_NODE_DATASET" "$_dataset_nrn" "$(np__json_obj id "$_dataset_id")" >/dev/null - return 0 -} - -# np_trace_job [--nrn N] [--plan JSON] -# -# Emit a job definition node — the reusable spec runs link instance_of, with -# its expected step plan (previewable before any run exists). -np_trace_job() { - [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 - _job_namespace=${1:-} - _job_name=${2:-} - _job_version=${3:-} - if [ "$#" -ge 3 ]; then - shift 3 - fi - _job_nrn='' - _job_plan='' - while [ "$#" -gt 0 ]; do - case "$1" in - --nrn) _job_nrn=${2:-}; shift 2 ;; - --plan) _job_plan=${2:-}; shift 2 ;; - *) shift ;; - esac - done - if [ -z "$_job_namespace" ] || [ -z "$_job_name" ] || [ -z "$_job_version" ]; then - np__drop 'job' 'namespace, name and version are required' - return 0 - fi - case "$_job_plan" in - '' | \[*) ;; - *) np__drop 'job' 'the plan must be a JSON array of steps'; return 0 ;; - esac - if [ -n "$_job_plan" ]; then - _job_data=$(np__json_obj_raw \ - namespace "$(np__json_str "$_job_namespace")" \ - name "$(np__json_str "$_job_name")" \ - version "$(np__json_str "$_job_version")" \ - facets "{$(np__json_str "$NP_FACET_PLAN"):$_job_plan}") - else - _job_data=$(np__json_obj namespace "$_job_namespace" name "$_job_name" version "$_job_version") - fi - np__spool "$NP_TYPE_NODE_JOB" "$_job_nrn" "$_job_data" >/dev/null - return 0 -} - -# --------------------------------------------------------------------------- -# The remaining core-facet setters -# --------------------------------------------------------------------------- - -# np_trace_actor [handle] [--source S] -# -# WHO acted. The sibling SDKs also accept a bearer JWT and decode it; that -# sugar needs base64, which this SDK's runtime toolset excludes — pass the -# identity explicitly (the np CLI stamps the actor on workflow runs already). -np_trace_actor() { - _actor_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_actor_handle" || return 0 - _actor_kind=${1:-} - _actor_id=${2:-} - if [ "$#" -ge 2 ]; then - shift 2 - fi - _actor_source='' - while [ "$#" -gt 0 ]; do - case "$1" in - --source) _actor_source=${2:-}; shift 2 ;; - *) shift ;; - esac - done - case "$_actor_kind" in - user | service) ;; - *) np__drop 'actor' "kind must be user or service, got '${_actor_kind}'"; return 0 ;; - esac - if [ -z "$_actor_id" ]; then - np__drop 'actor' 'an id is required' - return 0 - fi - np__stage_facet "$_actor_handle" "$NP_FACET_ACTOR" \ - "$(np__json_obj kind "$_actor_kind" id "$_actor_id" source "$_actor_source")" - np__flush_foreign "$_actor_handle" - return 0 -} - -# np_trace_decision [handle] [--available a,b,c] [--expression E] -# -# The branch(es) this node chose, with the option set and the human-readable -# expression when known. -np_trace_decision() { - _decision_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_decision_handle" || return 0 - _decision_chosen=${1:-} - if [ "$#" -gt 0 ]; then - shift - fi - _decision_available='' - _decision_expression='' - while [ "$#" -gt 0 ]; do - case "$1" in - --available) _decision_available=${2:-}; shift 2 ;; - --expression) _decision_expression=${2:-}; shift 2 ;; - *) shift ;; - esac - done - if [ -z "$_decision_chosen" ]; then - np__drop 'decision' 'at least one chosen branch is required' - return 0 - fi - np__stage_facet "$_decision_handle" "$NP_FACET_DECISION" \ - "$(np__json_obj_raw \ - chosen "$(np__json_str_array_csv "$_decision_chosen")" \ - available "$(if [ -n "$_decision_available" ]; then np__json_str_array_csv "$_decision_available"; fi)" \ - expression "$(if [ -n "$_decision_expression" ]; then np__json_str "$_decision_expression"; fi)")" - np__flush_foreign "$_decision_handle" - return 0 -} - -# np_trace_retry [handle] [--next-attempt N] [--delay-ms MS] -np_trace_retry() { - _retry_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_retry_handle" || return 0 - _retry_attempt=${1:-} - if [ "$#" -gt 0 ]; then - shift - fi - _retry_next='' - _retry_delay='' - while [ "$#" -gt 0 ]; do - case "$1" in - --next-attempt) _retry_next=${2:-}; shift 2 ;; - --delay-ms) _retry_delay=${2:-}; shift 2 ;; - *) shift ;; - esac - done - case "$_retry_attempt$_retry_next$_retry_delay" in - '' | *[!0-9]*) np__drop 'retry' 'attempt, next-attempt and delay-ms must be non-negative integers'; return 0 ;; - esac - np__stage_facet "$_retry_handle" "$NP_FACET_RETRY" \ - "$(np__json_obj_raw attempt "$_retry_attempt" next_attempt "$_retry_next" delay_ms "$_retry_delay")" - np__flush_foreign "$_retry_handle" - return 0 -} - -# np_trace_signal [handle] [--timeout-ms MS] -np_trace_signal() { - _signal_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_signal_handle" || return 0 - _signal_name=${1:-} - _signal_direction=${2:-} - if [ "$#" -ge 2 ]; then - shift 2 - fi - _signal_timeout='' - while [ "$#" -gt 0 ]; do - case "$1" in - --timeout-ms) _signal_timeout=${2:-}; shift 2 ;; - *) shift ;; - esac - done - if [ -z "$_signal_name" ]; then - np__drop 'signal' 'a name is required' - return 0 - fi - case "$_signal_direction" in - wait | received) ;; - *) np__drop 'signal' "direction must be wait or received, got '${_signal_direction}'"; return 0 ;; - esac - case "$_signal_timeout" in - '' | *[!0-9]*) - if [ -n "$_signal_timeout" ]; then - np__drop 'signal' 'timeout-ms must be a non-negative integer' - return 0 - fi - ;; - esac - np__stage_facet "$_signal_handle" "$NP_FACET_SIGNAL" \ - "$(np__json_obj_raw \ - name "$(np__json_str "$_signal_name")" \ - direction "$(np__json_str "$_signal_direction")" \ - timeout_ms "$_signal_timeout")" - np__flush_foreign "$_signal_handle" - return 0 -} - -# np_trace_external_links [handle] [--label L] -# -# One off-platform link (a CI run, a dashboard). Accumulates: call once per -# link, the facet is the array of everything declared so far. -np_trace_external_links() { - _external_links_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_external_links_handle" || return 0 - _external_links_rel=${1:-} - _external_links_uri=${2:-} - if [ "$#" -ge 2 ]; then - shift 2 - fi - _external_links_label='' - while [ "$#" -gt 0 ]; do - case "$1" in - --label) _external_links_label=${2:-}; shift 2 ;; - *) shift ;; - esac - done - if [ -z "$_external_links_rel" ] || [ -z "$_external_links_uri" ]; then - np__drop 'external_links' 'rel and uri are required' - return 0 - fi - _external_links_link=$(np__json_obj rel "$_external_links_rel" uri "$_external_links_uri" label "$_external_links_label") - _external_links_links=$(np__node_get "$_external_links_handle" external_links) - if [ -n "$_external_links_links" ]; then - _external_links_links="$_external_links_links,$_external_links_link" - else - _external_links_links=$_external_links_link - fi - np__node_set "$_external_links_handle" external_links "$_external_links_links" - np__stage_facet "$_external_links_handle" "$NP_FACET_EXTERNAL_LINKS" "[$_external_links_links]" - np__flush_foreign "$_external_links_handle" - return 0 -} - -# np_trace_engine_status [handle] [--raw JSON] -# -# The underlying engine's own view of this node (a k8s rollout's status, a -# queue's verdict), verbatim. -np_trace_engine_status() { - _engine_status_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_engine_status_handle" || return 0 - _engine_status_engine=${1:-} - _engine_status_state=${2:-} - if [ "$#" -ge 2 ]; then - shift 2 - fi - _engine_status_raw='' - while [ "$#" -gt 0 ]; do - case "$1" in - --raw) _engine_status_raw=${2:-}; shift 2 ;; - *) shift ;; - esac - done - if [ -z "$_engine_status_engine" ] || [ -z "$_engine_status_state" ]; then - np__drop 'engine_status' 'engine and state are required' - return 0 - fi - case "$_engine_status_raw" in - '' | \{*) ;; - *) np__drop 'engine_status' 'raw must be a JSON object'; return 0 ;; - esac - np__stage_facet "$_engine_status_handle" "$NP_FACET_ENGINE_STATUS" \ - "$(np__json_obj_raw \ - engine "$(np__json_str "$_engine_status_engine")" \ - state "$(np__json_str "$_engine_status_state")" \ - raw "$_engine_status_raw")" - np__flush_foreign "$_engine_status_handle" - return 0 -} - -# np_trace_dropped [handle] -# -# A record of data intentionally dropped — pair with np_trace_skip. -np_trace_dropped() { - _dropped_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_dropped_handle" || return 0 - if [ -z "${1:-}" ]; then - np__drop 'dropped' 'a reason is required' - return 0 - fi - np__stage_facet "$_dropped_handle" "$NP_FACET_DROPPED" "$(np__json_obj reason "$1")" - np__flush_foreign "$_dropped_handle" - return 0 -} - -# np_trace_plan [handle] -# -# Declare the node's EXPECTED step plan ([{"key":...,"title":...}, ...]) so -# the read model reports expected-vs-observed progress. On a reusable -# definition, prefer np_trace_job --plan. -np_trace_plan() { - _plan_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_plan_handle" || return 0 - case "${1:-}" in - \[*) ;; - *) np__drop 'plan' 'the plan must be a JSON array of steps'; return 0 ;; - esac - np__stage_facet "$_plan_handle" "$NP_FACET_PLAN" "$1" - np__flush_foreign "$_plan_handle" - return 0 -} - -# np_trace_affordances [handle] -# -# What this node OFFERS a human to do — a declared fact the UI renders as a -# control (view live logs, switch traffic). One affordance object -# ('{"kind":"deploy-log",...}') or a bare array of them; the wire form is -# always the array. -np_trace_affordances() { - _affordances_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_affordances_handle" || return 0 - _affordances_body=${1:-} - case "$_affordances_body" in - \[*) ;; - \{*) _affordances_body="[$_affordances_body]" ;; - *) np__drop 'affordances' 'body must be a JSON object or array'; return 0 ;; - esac - np__stage_facet "$_affordances_handle" "$NP_FACET_AFFORDANCES" "$_affordances_body" - np__flush_foreign "$_affordances_handle" - return 0 -} - -# np_trace_progress [handle] [unit] -# -# How far a CONVERGING phase has advanced toward its declared target — -# instances 3 of 10, traffic 40 of 100. Non-negative integers; the optional -# unit names what is counted ("percent", "instances"). -np_trace_progress() { - _progress_handle=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_progress_handle" || return 0 - _progress_current=${1:-} - _progress_target=${2:-} - _progress_unit=${3:-} - if [ -z "$_progress_current" ] || [ -z "$_progress_target" ]; then - np__drop 'progress' 'current and target must be non-negative integers' - return 0 - fi - case "$_progress_current$_progress_target" in - *[!0-9]*) np__drop 'progress' 'current and target must be non-negative integers'; return 0 ;; - esac - np__stage_facet "$_progress_handle" "$NP_FACET_PROGRESS" \ - "$(np__json_obj_raw current "$_progress_current" target "$_progress_target" \ - unit "$(if [ -n "$_progress_unit" ]; then np__json_str "$_progress_unit"; fi)")" - np__flush_foreign "$_progress_handle" - return 0 -} - -# --------------------------------------------------------------------------- -# Lifecycle terminals -# --------------------------------------------------------------------------- - -# The shared terminal path. $1 = handle, $2 = status. -np__terminalize() { - np__is_handle "$1" || return 0 - if [ "$(np__node_get "$1" closed)" = '1' ]; then - return 0 - fi - # An adopted node belongs to the process that created it. Its owner decides - # its outcome; emitting a terminal here would assert a state we did not - # observe, and would race the owner's own terminal event. - if np__is_foreign "$1"; then - np__drop 'terminal' 'refusing to close an adopted node' - return 0 - fi - np_trace_start "$1" - np__stage_timing "$1" "$(np__iso8601)" - np__node_set "$1" closed 1 - np__emit_node "$1" "$2" - np__ambient_clear "$1" - # Restore the parent as ambient so a sibling opened next lands correctly. - _terminalize_parent=$(np__node_get "$1" parent) - if [ -n "$_terminalize_parent" ] && np__is_handle "$_terminalize_parent"; then - if [ "$(np__node_get "$_terminalize_parent" closed)" != '1' ]; then - np__ambient_set "$_terminalize_parent" - fi - fi - return 0 -} - -np_trace_complete() { - np__terminalize "$(np__resolve_handle "${1:-}")" "$NP_STATUS_COMPLETED" - return 0 -} - -# An idempotent completing close. -np_trace_end() { - np_trace_complete "$@" - return 0 -} - -np_trace_fail() { - _fail_h=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - # Refuse a foreign fail WHOLE, before the message stages: half-applying it - # (error facet emitted via the foreign flush, close refused) would smear an - # unowned outcome onto the node. Recording an observed fact on a foreign - # node is np_trace_error, deliberately. - if np__is_foreign "$_fail_h"; then - np__drop 'terminal' 'refusing to close an adopted node' - return 0 - fi - if [ -n "${1:-}" ]; then - np_trace_error "$_fail_h" --message "$1" - fi - # fail cascades to still-open child steps; complete deliberately does not — - # auto-completing an open child would assert a success the SDK cannot vouch - # for, and back-date its duration. - np__cascade_fail "$_fail_h" "${1:-}" - np__terminalize "$_fail_h" "$NP_STATUS_FAILED" - return 0 -} - -# True when $1 is a descendant of $2, by walking the parent chain upward. -# Deliberately NOT recursive: POSIX sh has no `local`, so a recursive walk -# clobbers its caller's loop variables — which silently skipped intermediate -# nodes in the cascade. -np__is_descendant_of() { - _is_descendant_of_cur=$(np__node_get "$1" parent) - _is_descendant_of_guard=0 - while [ -n "$_is_descendant_of_cur" ] && [ "$_is_descendant_of_guard" -lt 64 ]; do - if [ "$_is_descendant_of_cur" = "$2" ]; then - return 0 - fi - _is_descendant_of_cur=$(np__node_get "$_is_descendant_of_cur" parent) - _is_descendant_of_guard=$((_is_descendant_of_guard + 1)) - done - return 1 -} - -# Fail every still-open descendant. One flat pass over the registry, deepest -# first, so a node is closed before anything reads it as a parent. -np__cascade_fail() { - _cascade_fail_depth=64 - while [ "$_cascade_fail_depth" -ge 0 ]; do - for _cascade_fail_file in "$NP_TRACE_DIR/nodes"/*; do - [ -f "$_cascade_fail_file" ] || continue - _cascade_fail_h=${_cascade_fail_file##*/} - [ "$_cascade_fail_h" = "$1" ] && continue - [ "$(np__node_get "$_cascade_fail_h" closed)" = '1' ] && continue - np__is_descendant_of "$_cascade_fail_h" "$1" || continue - [ "$(np__depth_of "$_cascade_fail_h")" -eq "$_cascade_fail_depth" ] || continue - if [ -n "$2" ]; then - np_trace_error "$_cascade_fail_h" --message "$2" - fi - np__terminalize "$_cascade_fail_h" "$NP_STATUS_FAILED" - done - _cascade_fail_depth=$((_cascade_fail_depth - 1)) - done - return 0 -} - -# How many parent links sit above this node. -np__depth_of() { - _depth_of_cur=$(np__node_get "$1" parent) - _depth_of_n=0 - while [ -n "$_depth_of_cur" ] && [ "$_depth_of_n" -lt 64 ]; do - _depth_of_n=$((_depth_of_n + 1)) - _depth_of_cur=$(np__node_get "$_depth_of_cur" parent) - done - printf '%s' "$_depth_of_n" -} - -np_trace_skip() { - _skip_h=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__is_handle "$_skip_h" || return 0 - if [ -n "${1:-}" ]; then - np__stage_facet "$_skip_h" "$NP_FACET_DROPPED" "$(np__json_obj reason "$1")" - fi - np__terminalize "$_skip_h" "$NP_STATUS_SKIPPED" - return 0 -} - -np_trace_cancel() { - _cancel_h=$(np__resolve_handle "${1:-}") - if np__is_handle "${1:-}"; then - shift - fi - np__terminalize "$_cancel_h" "$NP_STATUS_CANCELLED" - return 0 -} - -np_trace_timeout() { - np__terminalize "$(np__resolve_handle "${1:-}")" "$NP_STATUS_TIMED_OUT" - return 0 -} - -# Non-terminal: the node stays open. -np_trace_waiting() { - _waiting_h=$(np__resolve_handle "${1:-}") - np__is_handle "$_waiting_h" || return 0 - np_trace_start "$_waiting_h" - np__emit_node "$_waiting_h" "$NP_STATUS_WAITING" - return 0 -} - -# ---- src/cli.sh ---- -# cli.sh — argv to function shim (Phase 2). diff --git a/scheduled_task/logging b/scheduled_task/logging index 64f808f2..dae4454c 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -407,12 +407,17 @@ _np_scopes_on_exit() { } _NP_SCOPES_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# The SDK ships as the catalog-tracing-sh submodule; a repo copy that lost the +# submodule (archive download, docker build context) may carry the single file +# vendored at the root instead. Neither present -> plain logging, untraced. +_NP_SCOPES_SDK="$_NP_SCOPES_ROOT/vendor/catalog-tracing-sh/nptrace.sh" +[ -f "$_NP_SCOPES_SDK" ] || _NP_SCOPES_SDK="$_NP_SCOPES_ROOT/nptrace.sh" if [ -z "${NP_TRACE_LOADED:-}" ] \ - && [ -f "$_NP_SCOPES_ROOT/nptrace.sh" ] \ + && [ -f "$_NP_SCOPES_SDK" ] \ && [ -n "${NP_API_KEY:-}" ] \ && [ -n "${NP_TRACE:-}" ]; then # shellcheck source=/dev/null - . "$_NP_SCOPES_ROOT/nptrace.sh" + . "$_NP_SCOPES_SDK" # --no-trap: the exit flush is ours, so the uncaught-failure report and the # flush share ONE trap in a defined order. np_trace_init --producer "nullplatform-scopes@1" --api-key "$NP_API_KEY" --no-trap diff --git a/vendor/catalog-tracing-sh b/vendor/catalog-tracing-sh new file mode 160000 index 00000000..882ba1f4 --- /dev/null +++ b/vendor/catalog-tracing-sh @@ -0,0 +1 @@ +Subproject commit 882ba1f42cd17387554be648a940d56be795ff85 From 2792c19f63b71984647353bb1185460e10efeaf0 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Thu, 13 Aug 2026 17:54:14 -0300 Subject: [PATCH 18/52] feat(k8s): state each error's cause to the workflow engine via np_step_error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the engine's preamble publishes np_step_error, the first log error cause of a step is stated through it, so the engine's own failure terminal carries the real diagnosis instead of a bare exit status — one message on the wire no matter which side reports last. The exit trap's synthetic mechanism message never states: it must not outrank the engine's own evidence (the shell's last stderr words). --- k8s/logging | 11 +++++++++++ k8s/utils/tests/trace_logging.bats | 25 +++++++++++++++++++++++++ scheduled_task/logging | 11 +++++++++++ 3 files changed, 47 insertions(+) diff --git a/k8s/logging b/k8s/logging index dae4454c..fb68e8ee 100644 --- a/k8s/logging +++ b/k8s/logging @@ -120,6 +120,15 @@ _np_scopes_trace_error() { return 0 fi np_trace_error "$_lt_node" --message "$_lt_message" ${2:+--code "$2"} + # State the cause to the workflow engine too (np_step_error is published by + # the engine's script preamble): if this step later exits nonzero, the + # engine's own failure report carries THIS message instead of a bare exit + # status — one diagnosis on the wire, no matter which side reports last. + # Only REAL diagnoses state: the exit trap's synthetic mechanism message + # must not outrank the engine's own evidence (the shell's stderr words). + if [ -z "${_NP_SCOPES_TRAP_REPORT:-}" ] && command -v np_step_error >/dev/null 2>&1; then + np_step_error "$_lt_message" + fi # Remember which step already carries a real message (so the exit trap does # not shadow it with a generic one) AND the message itself — on a fatal # exit, the run-level mirror repeats the real diagnosis, not the mechanism. @@ -395,7 +404,9 @@ _np_scopes_on_exit() { local _ex_rc="${1:-0}" local _ex_reason="${_NP_SCOPES_LAST_REASON:-${_NP_SCOPES_LAST_ERR:-workflow shell exited with status $_ex_rc}}" if [ "$_ex_rc" -ne 0 ] && [ "${_NP_SCOPES_ERRED_ON:-}" != "${NP_TRACE:-}" ]; then + _NP_SCOPES_TRAP_REPORT=1 _np_scopes_trace_error "$_ex_reason" || true + _NP_SCOPES_TRAP_REPORT="" fi if [ "$_ex_rc" -ne 0 ]; then _np_scopes_error_on_run "$_ex_reason" || true diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index 9498b82e..2f6e52da 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -401,6 +401,31 @@ secret/sec-1 created" echo "$output" | grep -q '"message":"HostedZone not found (AccessDenied)","details":{"hints":\["💡 Possible causes:","• The role lacks route53:ListHostedZones"\]}' } +@test "the first error cause is stated to the workflow engine via np_step_error" { + STATED="$BATS_TEST_TMPDIR/stated" + export STATED + run_logged ' + np_step_error() { printf "%s\n" "$1" >>"$STATED"; } + log error "❌ HostedZone not found (AccessDenied)" + log error "💡 a hint, not a cause" + ' + [ "$status" -eq 0 ] + run cat "$STATED" + assert_equal "$output" "HostedZone not found (AccessDenied)" +} + +@test "the exit trap's synthetic message is never stated as a cause" { + STATED="$BATS_TEST_TMPDIR/stated" + export STATED + run_logged ' + np_step_error() { printf "%s\n" "$1" >>"$STATED"; } + ( exit 3 ) # arm LAST_ERR without any log error, then die unhandled + exit 3 + ' + [ "$status" -eq 3 ] + [ ! -s "$STATED" ] +} + @test "trace errors carry the cause stripped of console decoration" { run_logged ' log error " ❌ Failed to find IAM role: An error occurred (NoSuchEntity)" diff --git a/scheduled_task/logging b/scheduled_task/logging index dae4454c..fb68e8ee 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -120,6 +120,15 @@ _np_scopes_trace_error() { return 0 fi np_trace_error "$_lt_node" --message "$_lt_message" ${2:+--code "$2"} + # State the cause to the workflow engine too (np_step_error is published by + # the engine's script preamble): if this step later exits nonzero, the + # engine's own failure report carries THIS message instead of a bare exit + # status — one diagnosis on the wire, no matter which side reports last. + # Only REAL diagnoses state: the exit trap's synthetic mechanism message + # must not outrank the engine's own evidence (the shell's stderr words). + if [ -z "${_NP_SCOPES_TRAP_REPORT:-}" ] && command -v np_step_error >/dev/null 2>&1; then + np_step_error "$_lt_message" + fi # Remember which step already carries a real message (so the exit trap does # not shadow it with a generic one) AND the message itself — on a fatal # exit, the run-level mirror repeats the real diagnosis, not the mechanism. @@ -395,7 +404,9 @@ _np_scopes_on_exit() { local _ex_rc="${1:-0}" local _ex_reason="${_NP_SCOPES_LAST_REASON:-${_NP_SCOPES_LAST_ERR:-workflow shell exited with status $_ex_rc}}" if [ "$_ex_rc" -ne 0 ] && [ "${_NP_SCOPES_ERRED_ON:-}" != "${NP_TRACE:-}" ]; then + _NP_SCOPES_TRAP_REPORT=1 _np_scopes_trace_error "$_ex_reason" || true + _NP_SCOPES_TRAP_REPORT="" fi if [ "$_ex_rc" -ne 0 ]; then _np_scopes_error_on_run "$_ex_reason" || true From 1f3aabae0b03ccf89c46a935657facec3afe9a29 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Tue, 25 Aug 2026 20:28:29 -0300 Subject: [PATCH 19/52] feat(k8s): a traced run missing the bundled SDK says so, loudly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A build that loses the tracing submodule (archive download, docker build context) used to degrade to plain logging in SILENCE — the CLI side traces, the structure looks fine, and the scope-side story (sub-steps, live waits, real errors, lineage, affordances) is just absent, with nothing anywhere saying why. Exactly the failure nobody sees. Now, when the engine is tracing the run (NP_TRACE) and credentials are present (NP_API_KEY) but nptrace.sh is not in the bundle, the workflow log carries one loud warning naming the missing file. Untraced runs stay silent as before. --- k8s/logging | 10 ++++++++++ k8s/utils/tests/trace_logging.bats | 29 +++++++++++++++++++++++++++++ scheduled_task/logging | 10 ++++++++++ 3 files changed, 49 insertions(+) diff --git a/k8s/logging b/k8s/logging index fb68e8ee..21812f2b 100644 --- a/k8s/logging +++ b/k8s/logging @@ -434,4 +434,14 @@ if [ -z "${NP_TRACE_LOADED:-}" ] \ np_trace_init --producer "nullplatform-scopes@1" --api-key "$NP_API_KEY" --no-trap trap '_np_scopes_on_err $?' ERR trap '_np_scopes_on_exit $?' EXIT +elif [ -z "${NP_TRACE_LOADED:-}" ] \ + && [ -n "${NP_TRACE:-}" ] \ + && [ -n "${NP_API_KEY:-}" ] \ + && [ ! -f "$_NP_SCOPES_SDK" ]; then + # The engine is tracing this run and the credentials are present, yet the shell + # SDK is not in this bundle: the scope-side story (sub-steps, live waits, real + # errors, lineage, affordances) will be ABSENT while everything else looks fine. + # A build that loses the submodule (archive download, docker build context) is + # exactly the failure nobody sees — so say it, once, on the workflow's own log. + log warn "⚠️ tracing SDK not bundled (vendor/catalog-tracing-sh/nptrace.sh missing) — scope-side tracing disabled for this run" fi diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index 2f6e52da..d986c009 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -447,3 +447,32 @@ secret/sec-1 created" echo "$output" | grep -q '"hints":\["a hint for the second"\]' ! echo "$output" | grep 'create-dns@0.0' | grep -q '"message":"first cause"' } + +@test "a traced run with credentials but NO bundled SDK warns loudly instead of degrading silently" { + run "$BASH" -c ' + export NP_TRACE="1|trace-1|run-1@0.0" + export NP_API_KEY="key" + NP_SCOPES_TEST_ROOT="$BATS_TEST_TMPDIR/empty-bundle" + mkdir -p "$NP_SCOPES_TEST_ROOT/k8s" + cp "'"$LOGGING"'" "$NP_SCOPES_TEST_ROOT/k8s/logging" + source "$NP_SCOPES_TEST_ROOT/k8s/logging" + log info "workflow proceeds" + ' + [ "$status" -eq 0 ] + echo "$output" | grep -q "tracing SDK not bundled" + echo "$output" | grep -q "workflow proceeds" +} + +@test "an untraced run (no NP_TRACE) stays silent about the SDK" { + run "$BASH" -c ' + unset NP_TRACE + export NP_API_KEY="key" + NP_SCOPES_TEST_ROOT="$BATS_TEST_TMPDIR/empty-bundle2" + mkdir -p "$NP_SCOPES_TEST_ROOT/k8s" + cp "'"$LOGGING"'" "$NP_SCOPES_TEST_ROOT/k8s/logging" + source "$NP_SCOPES_TEST_ROOT/k8s/logging" + log info "plain logging" + ' + [ "$status" -eq 0 ] + ! echo "$output" | grep -q "tracing SDK not bundled" +} diff --git a/scheduled_task/logging b/scheduled_task/logging index fb68e8ee..21812f2b 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -434,4 +434,14 @@ if [ -z "${NP_TRACE_LOADED:-}" ] \ np_trace_init --producer "nullplatform-scopes@1" --api-key "$NP_API_KEY" --no-trap trap '_np_scopes_on_err $?' ERR trap '_np_scopes_on_exit $?' EXIT +elif [ -z "${NP_TRACE_LOADED:-}" ] \ + && [ -n "${NP_TRACE:-}" ] \ + && [ -n "${NP_API_KEY:-}" ] \ + && [ ! -f "$_NP_SCOPES_SDK" ]; then + # The engine is tracing this run and the credentials are present, yet the shell + # SDK is not in this bundle: the scope-side story (sub-steps, live waits, real + # errors, lineage, affordances) will be ABSENT while everything else looks fine. + # A build that loses the submodule (archive download, docker build context) is + # exactly the failure nobody sees — so say it, once, on the workflow's own log. + log warn "⚠️ tracing SDK not bundled (vendor/catalog-tracing-sh/nptrace.sh missing) — scope-side tracing disabled for this run" fi From 6bdbcf016c41b9ae61945066b72712e223d9a83c Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 06:39:52 -0300 Subject: [PATCH 20/52] fix(k8s): each workflow's story lands on its OWN station MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated against real blue/green deployments on a custom scope, side by side with the native experience: - finalize declares its own group — its steps used to file under Setting up / Waiting for instances, twins of the deploy workflow's stations, and the ending never got a station of its own. The entity ending overlay names it (Finalized / Rolled back). - rollback likewise: one Finalize group — a rollback's wrap-up is the ending station's story. - switch_traffic scopes ALL its steps under Switching traffic: per-increment runs fold there as attempts (the native reading) instead of re-lighting Setting up / Waiting on every increment. - the wait steps get workflow-distinct keys (finalize-instances-check, switch-instances-check): three workflows declared the same wait-for-instances key, so the fold merged semantically different waits into one row across runs. --- k8s/deployment/workflows/finalize.yaml | 7 +++---- k8s/deployment/workflows/rollback.yaml | 3 +-- k8s/deployment/workflows/switch_traffic.yaml | 10 ++++------ 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index ff1e06de..7e6315fd 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -6,8 +6,7 @@ trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] job: k8s-deployment-finalize groups: - - {key: setting-up, title: Setting up} - - {key: waiting-instances, title: Waiting for instances to be healthy} + - {key: finalize, title: Finalize} steps: - name: load logging type: script @@ -55,9 +54,9 @@ steps: type: script file: "$SERVICE_PATH/deployment/wait_deployment_active" trace: - key: wait-for-instances + key: finalize-instances-check title: Instance health check - group: waiting-instances + group: finalize configuration: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS SKIP_DEPLOYMENT_STATUS_CHECK: true diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index 4db8f0d3..70c4d271 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -6,8 +6,7 @@ trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] job: k8s-deployment-rollback groups: - - {key: setting-up, title: Setting up} - - {key: waiting-instances, title: Waiting for instances to be healthy} + - {key: finalize, title: Finalize} steps: - name: load logging type: script diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index bfdf8023..a05c24b0 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -8,8 +8,6 @@ trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] job: k8s-deployment-switch-traffic groups: - - {key: setting-up, title: Setting up} - - {key: waiting-instances, title: Waiting for instances to be healthy} - {key: switching-traffic, title: Switching traffic} steps: - name: load logging @@ -55,15 +53,15 @@ steps: file: "$SERVICE_PATH/deployment/scale_deployments" trace: title: Scale green deployment - group: setting-up + group: switching-traffic post: name: wait deployment active type: script file: "$SERVICE_PATH/deployment/wait_deployment_active" trace: - key: wait-for-instances + key: switch-instances-check title: Instance health check - group: waiting-instances + group: switching-traffic configuration: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS - name: route traffic @@ -81,7 +79,7 @@ steps: type: script file: "$SERVICE_PATH/deployment/scale_deployments" trace: - group: setting-up + group: switching-traffic - name: apply traffic type: script file: "$SERVICE_PATH/apply_templates" From 0865820f541883760bd91ce89e099161062cb105 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 12:58:33 -0300 Subject: [PATCH 21/52] feat(k8s): user-facing step titles everywhere; plumbing hidden MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every step a user sees carries a clear title in the native tone (Validate load balancer capacity, Promote new deployment, Restore traffic routing, Remove previous deployment); build/template plumbing is trace: false. rollback gains its full trace pass (it had none). Steps injected by the overrides repo keep their keys — the dashboard humanizes an untitled key rather than showing snake_case. --- k8s/deployment/workflows/blue_green.yaml | 2 ++ k8s/deployment/workflows/finalize.yaml | 12 ++++++++++++ k8s/deployment/workflows/initial.yaml | 3 +++ k8s/deployment/workflows/rollback.yaml | 14 ++++++++++++++ k8s/deployment/workflows/switch_traffic.yaml | 3 +++ 5 files changed, 34 insertions(+) diff --git a/k8s/deployment/workflows/blue_green.yaml b/k8s/deployment/workflows/blue_green.yaml index e1bb8c86..5173bcb9 100644 --- a/k8s/deployment/workflows/blue_green.yaml +++ b/k8s/deployment/workflows/blue_green.yaml @@ -12,4 +12,6 @@ steps: - name: update blue deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" + trace: + title: Prepare previous deployment after: create deployment \ No newline at end of file diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index 7e6315fd..7a65c0fb 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -47,6 +47,8 @@ steps: - name: BLUE_DEPLOYMENT_ID type: environment - name: build green deployment + trace: + title: Promote new deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" post: @@ -61,11 +63,15 @@ steps: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS SKIP_DEPLOYMENT_STATUS_CHECK: true - name: route traffic + trace: + title: Configure ingress type: script file: "$SERVICE_PATH/deployment/networking/gateway/route_traffic" configuration: TEMPLATE: "$INGRESS_TEMPLATE" - name: apply traffic + trace: + title: Apply final routing type: script file: "$SERVICE_PATH/apply_templates" configuration: @@ -80,6 +86,8 @@ steps: type: workflow steps: - name: verify_networking_reconciliation + trace: + title: Verify networking type: script file: "$SERVICE_PATH/deployment/verify_networking_reconciliation" configuration: @@ -87,9 +95,11 @@ steps: # blue deployment is deleted below. Weights cannot express this state. EXPECT_SINGLE_TARGET_GROUP: true - name: publish_alb_metrics + trace: false type: script file: "$SERVICE_PATH/deployment/publish_alb_metrics" - name: build deployment + trace: false type: script file: "$SERVICE_PATH/deployment/build_blue_deployment" output: @@ -106,6 +116,8 @@ steps: type: file file: "$OUTPUT_DIR/service-$SCOPE_ID-$BLUE_DEPLOYMENT_ID.yaml" - name: delete deployment + trace: + title: Remove previous deployment type: script file: "$SERVICE_PATH/apply_templates" configuration: diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index 49961d36..cbcd93ef 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -54,6 +54,7 @@ steps: type: script file: "$SERVICE_PATH/deployment/validate_alb_target_group_capacity" trace: + title: Validate load balancer capacity group: setting-up flavors: [route53] - name: route traffic @@ -104,6 +105,7 @@ steps: type: script file: "$SERVICE_PATH/deployment/verify_networking_reconciliation" trace: + title: Verify networking group: setting-up flavors: [route53, external_dns] configuration: @@ -112,6 +114,7 @@ steps: type: script file: "$SERVICE_PATH/deployment/publish_alb_metrics" trace: + title: Publish load balancer metrics flavors: [route53] - name: wait deployment active type: script diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index 70c4d271..ec25a33f 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -49,7 +49,13 @@ steps: - name: update blue deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" + trace: + title: Restore previous deployment + group: finalize - name: rollback traffic + trace: + title: Restore traffic routing + group: finalize type: script file: "$SERVICE_PATH/deployment/networking/gateway/rollback_traffic" configuration: @@ -59,6 +65,9 @@ steps: type: file file: "$OUTPUT_DIR/ingress-$SCOPE_ID-$BLUE_DEPLOYMENT_ID.yaml" - name: apply traffic + trace: + title: Apply routing + group: finalize type: script file: "$SERVICE_PATH/apply_templates" configuration: @@ -73,6 +82,7 @@ steps: # the failed deployment is deleted below. Weights cannot express this state. EXPECT_SINGLE_TARGET_GROUP: true - name: build deployment + trace: false type: script file: "$SERVICE_PATH/deployment/build_deployment" output: @@ -89,6 +99,9 @@ steps: type: file file: "$OUTPUT_DIR/service-$SCOPE_ID-$DEPLOYMENT_ID.yaml" - name: delete deployment + trace: + title: Remove new deployment + group: finalize type: script file: "$SERVICE_PATH/apply_templates" configuration: @@ -101,5 +114,6 @@ steps: configuration: DEPLOYMENT: green - name: print_deployment_error_hints + trace: false type: script file: "$SERVICE_PATH/deployment/print_failed_deployment_hints" diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index a05c24b0..17ca9aa0 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -79,6 +79,7 @@ steps: type: script file: "$SERVICE_PATH/deployment/scale_deployments" trace: + title: Scale previous deployment group: switching-traffic - name: apply traffic type: script @@ -99,6 +100,7 @@ steps: type: script file: "$SERVICE_PATH/deployment/verify_networking_reconciliation" trace: + title: Verify networking group: switching-traffic flavors: [route53, external_dns] configuration: @@ -107,4 +109,5 @@ steps: type: script file: "$SERVICE_PATH/deployment/publish_alb_metrics" trace: + title: Publish load balancer metrics flavors: [route53] From 8884739f25110358da54ce987d8c8746f7179fa9 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 14:30:16 -0300 Subject: [PATCH 22/52] =?UTF-8?q?fix(tracing):=20vendor=20nptrace.sh=20at?= =?UTF-8?q?=20the=20repo=20root=20=E2=80=94=20agent=20clones=20carry=20no?= =?UTF-8?q?=20submodules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent materializes the package source with a plain clone, so the catalog-tracing-sh submodule arrives empty and every run logged 'tracing SDK not bundled' (observed live: zero step io on three deployments that declared it). The logging loader already prefers the submodule and falls back to a root copy — ship the copy. Kept in sync by the existing re-vendor chore. Also: blue_green/initial declare placeholder groups (switching-traffic, finalize) so the whole journey renders pending from the first paint — paired with the CLI emitting an optional placeholder step per empty declared group. --- k8s/deployment/workflows/blue_green.yaml | 5 + k8s/deployment/workflows/initial.yaml | 2 + nptrace.sh | 2265 ++++++++++++++++++++++ 3 files changed, 2272 insertions(+) create mode 100755 nptrace.sh diff --git a/k8s/deployment/workflows/blue_green.yaml b/k8s/deployment/workflows/blue_green.yaml index 5173bcb9..d66193ec 100644 --- a/k8s/deployment/workflows/blue_green.yaml +++ b/k8s/deployment/workflows/blue_green.yaml @@ -8,6 +8,11 @@ trace: groups: - {key: setting-up, title: Setting up} - {key: waiting-instances, title: Waiting for instances to be healthy} + # Declared with no steps here on purpose: PLACEHOLDER stations the later + # workflows (switch_traffic, finalize) fill — the line shows the whole + # journey pending from the first paint, exactly like a native scope. + - {key: switching-traffic, title: Switching traffic} + - {key: finalize, title: Finalize} steps: - name: update blue deployment type: script diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index cbcd93ef..a706455b 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -11,6 +11,8 @@ trace: groups: - {key: setting-up, title: Setting up} - {key: waiting-instances, title: Waiting for instances to be healthy} + # Placeholder: the finalize workflow fills this station later. + - {key: finalize, title: Finalize} steps: - name: load logging type: script diff --git a/nptrace.sh b/nptrace.sh new file mode 100755 index 00000000..b26096db --- /dev/null +++ b/nptrace.sh @@ -0,0 +1,2265 @@ +#!/bin/sh + +# ---- src/header.sh ---- +# nullplatform tracing for POSIX shell — producer SDK for the nullplatform +# tracing API. Zero runtime dependencies beyond curl and the POSIX toolset. +# +# Generated file: edit src/*.sh and run ./build.sh. + +if [ -n "${NP_TRACE_LOADED:-}" ]; then + return 0 2>/dev/null || exit 0 +fi +NP_TRACE_LOADED=1 +NP_TRACE_VERSION="0.1.0" + +# ---- src/compat.sh ---- +# compat.sh — portability shims. The ONLY place OS differences live. + +# Unix milliseconds. GNU date supports %N; busybox and BSD may not, and they +# fail in two DIFFERENT ways: +# +# busybox 1.38 / BSD -> "1786045823%3N" the format leaks through literally +# busybox 1.37 -> "1786045823" the format is silently DROPPED +# +# The second is the dangerous one: the result is clean digits that merely happen +# to be seconds, so a digits-only check accepts it and every timestamp is then +# 1000x too small — which silently destroys UUIDv7 ordering, since the seconds +# value lands in a 48-bit millisecond field and decodes to 1970. +# +# Length is what separates them: Unix milliseconds have been 13 digits since +# 2001-09-09 and stay 13 until 2286, while seconds are 10. Anything shorter than +# 13 is not milliseconds, whatever it looks like. +np__epoch_ms() { + _epoch_ms_ms=$(date -u +%s%3N 2>/dev/null) || _epoch_ms_ms='' + case "$_epoch_ms_ms" in + '' | *[!0-9]*) _epoch_ms_ms='' ;; + esac + if [ -n "$_epoch_ms_ms" ] && [ "${#_epoch_ms_ms}" -ge 13 ]; then + printf '%s' "$_epoch_ms_ms" + return 0 + fi + # Second precision. Event ids stay unique via their random bits. + printf '%s000' "$(date -u +%s)" +} + +# Exactly $1 lowercase hex characters from the kernel CSPRNG. +np__rand_hex() { + _rand_hex_want=$1 + _rand_hex_bytes=$(( (_rand_hex_want + 1) / 2 )) + od -An -tx1 -N"$_rand_hex_bytes" /dev/urandom | tr -d ' \n' | cut -c1-"$_rand_hex_want" +} + +# RFC 3339 UTC, second precision — the envelope `time` field. +np__iso8601() { + date -u +%Y-%m-%dT%H:%M:%SZ +} + +# ---- src/json.sh ---- +# json.sh — JSON emission. There is no parser here beyond one field extractor +# for the auth response; the SDK only ever WRITES JSON. + +# Escape a string for a JSON string body (no surrounding quotes). +# +# Fast path: a string made only of unmistakably safe characters is returned +# unchanged, so the common label/id case never forks an awk. The allowlist is +# deliberately conservative — routing an unusual string to the slow path is +# always correct, only slower. +# +# Slow path: awk under LC_ALL=C, so length/substr are BYTE oriented on every +# awk (gawk, mawk, busybox). UTF-8 sequences pass through byte for byte, which +# is valid JSON; only the seven shorthand escapes and C0 controls are rewritten. +# Records are read line by line and rejoined with \n rather than using a +# multi-character RS, whose behaviour POSIX leaves undefined. +np__json_escape() { + case "$1" in + *[!A-Za-z0-9\ ._:/@=+,-]*) ;; + *) printf '%s' "$1"; return 0 ;; + esac + printf '%s' "$1" | LC_ALL=C awk ' + function esc(s, i, c, n, o) { + o = "" + n = length(s) + for (i = 1; i <= n; i++) { + c = substr(s, i, 1) + if (c == "\\") { o = o "\\\\" } + else if (c == "\"") { o = o "\\\"" } + else if (c == "\t") { o = o "\\t" } + else if (c == "\r") { o = o "\\r" } + else if (c == "\b") { o = o "\\b" } + else if (c == "\f") { o = o "\\f" } + else if (c < " ") { o = o sprintf("\\u%04x", ORD[c]) } + else { o = o c } + } + return o + } + BEGIN { + ORS = "" + for (i = 0; i < 256; i++) { ORD[sprintf("%c", i)] = i } + out = "" + } + { + if (NR > 1) { out = out "\\n" } + out = out esc($0) + } + END { printf "%s", out } + ' +} + +# A complete quoted JSON string. +np__json_str() { + printf '"%s"' "$(np__json_escape "$1")" +} + +# A JSON object from alternating key/value arguments. Values are emitted as +# JSON strings. A pair whose key or value is empty is OMITTED — an absent +# optional is absent, never the string "". +np__json_obj() { + _json_obj_out='' + while [ "$#" -ge 2 ]; do + if [ -n "$1" ] && [ -n "$2" ]; then + if [ -n "$_json_obj_out" ]; then + _json_obj_out="$_json_obj_out," + fi + _json_obj_out="$_json_obj_out$(np__json_str "$1"):$(np__json_str "$2")" + fi + shift 2 + done + printf '{%s}' "$_json_obj_out" +} + +# As np__json_obj, but each value is already-formed JSON inserted verbatim. +# Use for nested objects, arrays, numbers, and booleans. +np__json_obj_raw() { + _json_obj_raw_out='' + while [ "$#" -ge 2 ]; do + if [ -n "$1" ] && [ -n "$2" ]; then + if [ -n "$_json_obj_raw_out" ]; then + _json_obj_raw_out="$_json_obj_raw_out," + fi + _json_obj_raw_out="$_json_obj_raw_out$(np__json_str "$1"):$2" + fi + shift 2 + done + printf '{%s}' "$_json_obj_raw_out" +} + +# A JSON array of strings from a comma-separated list ("a, b" → ["a","b"]). +# Surrounding whitespace per item is trimmed; empty items are omitted. +np__json_str_array_csv() { + _json_str_array_csv_out='' + _json_str_array_csv_rest=$1 + while [ -n "$_json_str_array_csv_rest" ]; do + case "$_json_str_array_csv_rest" in + *,*) _json_str_array_csv_item=${_json_str_array_csv_rest%%,*}; _json_str_array_csv_rest=${_json_str_array_csv_rest#*,} ;; + *) _json_str_array_csv_item=$_json_str_array_csv_rest; _json_str_array_csv_rest='' ;; + esac + _json_str_array_csv_item=$(printf '%s' "$_json_str_array_csv_item" | sed 's/^ *//; s/ *$//') + if [ -n "$_json_str_array_csv_item" ]; then + if [ -n "$_json_str_array_csv_out" ]; then + _json_str_array_csv_out="$_json_str_array_csv_out," + fi + _json_str_array_csv_out="$_json_str_array_csv_out$(np__json_str "$_json_str_array_csv_item")" + fi + done + printf '[%s]' "$_json_str_array_csv_out" +} + +# ---- src/uuid.sh ---- +# uuid.sh — UUIDv7. The event id MUST be a v7: the API derives the storage +# partition from its embedded millisecond timestamp and rejects anything else. +# +# Layout: 48-bit big-endian ms timestamp | version nibble 7 | 12 random bits +# | variant bits 10 | 62 random bits. + +np__uuidv7() { + _u7_ts=$(printf '%012x' "$(np__epoch_ms)") + _u7_r=$(np__rand_hex 19) + + # The variant nibble must be one of 8, 9, a, b. Fold a random hex digit into + # that range rather than drawing again. + case $(printf '%s' "$_u7_r" | cut -c1) in + 0 | 1 | 2 | 3) _u7_var=8 ;; + 4 | 5 | 6 | 7) _u7_var=9 ;; + 8 | 9 | a | b) _u7_var=a ;; + *) _u7_var=b ;; + esac + + printf '%s-%s-7%s-%s%s-%s\n' \ + "$(printf '%s' "$_u7_ts" | cut -c1-8)" \ + "$(printf '%s' "$_u7_ts" | cut -c9-12)" \ + "$(printf '%s' "$_u7_r" | cut -c2-4)" \ + "$_u7_var" \ + "$(printf '%s' "$_u7_r" | cut -c5-7)" \ + "$(printf '%s' "$_u7_r" | cut -c8-19)" +} + +# Mint a per-occurrence token for a repeatable operation's run_id. Time-ordered, +# so minted ids sort by creation time. +np_trace_occurrence() { + np__uuidv7 +} + +# ---- src/identity.sh ---- +# identity.sh — the node identity grammar. A hand-port of the tracing API's +# contract module; these functions and their tests are the drift safety net. +# +# child_run_id = parent_run_id "~" key "@" attempt "." iteration +# +# One charset covers every producer-authored segment: [A-Za-z0-9_.-]+. The +# delimiter '~' and the coordinate marker '@' sit outside it, which is what +# makes the grammar collision-proof — no named id can ever parse as a derived +# one. + +NP_ID_DELIMITER='~' +NP_MAX_RUN_ID_LENGTH=1024 +NP_MAX_KEY_LENGTH=256 +NP_MAX_TRACE_ID_LENGTH=256 + +np__is_identifier() { + case "${1:-}" in + '') return 1 ;; + *[!A-Za-z0-9_.-]*) return 1 ;; + *) return 0 ;; + esac +} + +# Print a reason and return 1, or return 0 silently. +np__identifier_violation() { + if [ -z "$1" ]; then + printf 'must be non-empty' + return 1 + fi + if [ "${#1}" -gt "$2" ]; then + printf 'exceeds %s chars' "$2" + return 1 + fi + if ! np__is_identifier "$1"; then + printf "must be identifier-charset: letters, digits, '_', '.', '-'" + return 1 + fi + return 0 +} + +np__key_violation() { + np__identifier_violation "${1:-}" "$NP_MAX_KEY_LENGTH" +} + +np__named_id_violation() { + np__identifier_violation "${1:-}" "$NP_MAX_RUN_ID_LENGTH" +} + +np__trace_id_violation() { + np__identifier_violation "${1:-}" "$NP_MAX_TRACE_ID_LENGTH" +} + +# The derived id of a keyed child. +np__derive_child_id() { + printf '%s%s%s@%s.%s' "$1" "$NP_ID_DELIMITER" "$2" "$3" "$4" +} + +# Everything before the FIRST delimiter — the nearest named ancestor. Every +# keyed descendant of a named run shares its scope root at any depth. +np__scope_root_of() { + case "$1" in + *"$NP_ID_DELIMITER"*) printf '%s' "${1%%"$NP_ID_DELIMITER"*}" ;; + *) printf '%s' "$1" ;; + esac +} + +# Parse the LAST hop of a derived id. Prints " ". +# Returns 1 for a named id (no delimiter) or a malformed tail. +np__parse_node_id() { + case "$1" in + *"$NP_ID_DELIMITER"*) ;; + *) return 1 ;; + esac + _parse_node_id_parent=${1%"$NP_ID_DELIMITER"*} + _parse_node_id_tail=${1##*"$NP_ID_DELIMITER"} + case "$_parse_node_id_tail" in + *@*.*) ;; + *) return 1 ;; + esac + _parse_node_id_key=${_parse_node_id_tail%%@*} + _parse_node_id_coord=${_parse_node_id_tail#*@} + _parse_node_id_attempt=${_parse_node_id_coord%%.*} + _parse_node_id_iteration=${_parse_node_id_coord#*.} + if [ -z "$_parse_node_id_parent" ] || [ -z "$_parse_node_id_key" ]; then + return 1 + fi + case "$_parse_node_id_attempt" in + '' | *[!0-9]*) return 1 ;; + esac + case "$_parse_node_id_iteration" in + '' | *[!0-9]*) return 1 ;; + esac + printf '%s %s %s %s' "$_parse_node_id_parent" "$_parse_node_id_key" "$_parse_node_id_attempt" "$_parse_node_id_iteration" +} + +# Join parts into a stable id, dropping empty parts. Use instead of +# hand-interpolation so an absent part never leaves a dangling separator. +# The joiner is '-', a charset character, so the result stays a legal named id. +np_trace_key() { + _k_out='' + for _k_part in "$@"; do + if [ -n "$_k_part" ]; then + if [ -n "$_k_out" ]; then + _k_out="$_k_out-" + fi + _k_out="$_k_out$_k_part" + fi + done + printf '%s' "$_k_out" +} + +# ---- src/wire.sh ---- +# wire.sh — contract constants, hand-ported from the tracing API's wire +# package. When the API's contract changes, this file and identity.sh are what +# must be re-ported; their tests are the safety net. + +NP_TYPE_NODE_RUN='node.run' +NP_TYPE_NODE_DATASET='node.dataset' +NP_TYPE_NODE_JOB='node.job' + +NP_TYPE_EDGE_PARENT='edge.parent' +NP_TYPE_EDGE_TRIGGERED_BY='edge.triggered_by' +NP_TYPE_EDGE_RETRY_OF='edge.retry_of' +NP_TYPE_EDGE_CONTINUES='edge.continues' +NP_TYPE_EDGE_CORRELATES='edge.correlates' +NP_TYPE_EDGE_COMPENSATES='edge.compensates' +NP_TYPE_EDGE_PRODUCES='edge.produces' +NP_TYPE_EDGE_CONSUMES='edge.consumes' +NP_TYPE_EDGE_INSTANCE_OF='edge.instance_of' + +NP_STATUS_STARTED='started' +NP_STATUS_COMPLETED='completed' +NP_STATUS_FAILED='failed' +NP_STATUS_CANCELLED='cancelled' +NP_STATUS_TIMED_OUT='timed_out' +NP_STATUS_SKIPPED='skipped' +NP_STATUS_WAITING='waiting' + +NP_FACET_ERROR='tracing.error' +NP_FACET_TIMING='tracing.timing' +NP_FACET_INPUT='tracing.input' +NP_FACET_OUTPUT='tracing.output' +NP_FACET_BINDING='tracing.binding' +NP_FACET_DECISION='tracing.decision' +NP_FACET_RETRY='tracing.retry' +NP_FACET_SIGNAL='tracing.signal' +NP_FACET_EXTERNAL_LINKS='tracing.externalLinks' +NP_FACET_PLAN='tracing.plan' +NP_FACET_ACTOR='tracing.actor' +NP_FACET_DROPPED='tracing.dropped' +NP_FACET_ENGINE_STATUS='tracing.engineStatus' +NP_FACET_AFFORDANCES='tracing.affordances' +NP_FACET_EXPLAIN='tracing.explain' +NP_FACET_PROGRESS='tracing.progress' + +NP_CORE_FACETS="$NP_FACET_ERROR $NP_FACET_TIMING $NP_FACET_INPUT $NP_FACET_OUTPUT \ +$NP_FACET_BINDING $NP_FACET_DECISION $NP_FACET_RETRY $NP_FACET_SIGNAL \ +$NP_FACET_EXTERNAL_LINKS $NP_FACET_PLAN $NP_FACET_ACTOR $NP_FACET_DROPPED \ +$NP_FACET_ENGINE_STATUS $NP_FACET_AFFORDANCES $NP_FACET_EXPLAIN $NP_FACET_PROGRESS" + +NP_RESERVED_FACET_PREFIX='tracing.' +NP_RESERVED_LABEL_PREFIX='tracing.io/' + +# The context carrier: ONE field whose value packs version, trace and run. +NP_CARRIER_KEY='np-trace' +NP_CARRIER_VERSION='1' +NP_CARRIER_DELIMITER='|' + +np__is_terminal_status() { + case "${1:-}" in + completed | failed | cancelled | timed_out | skipped) return 0 ;; + *) return 1 ;; + esac +} + +# ---- src/state.sh ---- +# state.sh — the on-disk node registry. State lives on disk rather than in +# shell memory so handles survive process boundaries: in CI every pipeline step +# is a fresh shell. + +# Create the state tree. If it cannot be created or written — a read-only +# filesystem, a full disk, a bad NP_TRACE_DIR — the SDK degrades to a REAL +# no-op rather than half-working: a half-initialised SDK whose next write fails +# would take down a caller running under `set -e`, which is exactly the failure +# mode tracing must never cause. +np__state_init() { + if [ -z "${NP_TRACE_DIR:-}" ]; then + NP_TRACE_DIR="${TMPDIR:-/tmp}/nptrace.$$" + fi + export NP_TRACE_DIR + if ! mkdir -p "$NP_TRACE_DIR/nodes" "$NP_TRACE_DIR/staged" \ + "$NP_TRACE_DIR/spool" "$NP_TRACE_DIR/failed" 2>/dev/null; then + NP_TRACE_ENABLED=0 + return 0 + fi + # Prove the tree is actually writable before trusting it. + if ! printf '0' > "$NP_TRACE_DIR/seq.probe" 2>/dev/null; then + NP_TRACE_ENABLED=0 + return 0 + fi + rm -f "$NP_TRACE_DIR/seq.probe" 2>/dev/null || : + if [ ! -f "$NP_TRACE_DIR/seq" ]; then + printf '0' > "$NP_TRACE_DIR/seq" 2>/dev/null || : + fi + return 0 +} + +# Allocate the next handle. Handles are opaque by contract: consumers never +# parse them. +np__handle_new() { + _handle_new_seq=$(cat "$NP_TRACE_DIR/seq" 2>/dev/null || printf '0') + case "$_handle_new_seq" in + '' | *[!0-9]*) _handle_new_seq=0 ;; + esac + _handle_new_seq=$((_handle_new_seq + 1)) + printf '%s' "$_handle_new_seq" > "$NP_TRACE_DIR/seq" + _handle_new_handle="n$_handle_new_seq" + : > "$NP_TRACE_DIR/nodes/$_handle_new_handle" + printf '%s' "$_handle_new_handle" +} + +# THE rule the whole public surface rests on: an argument is a handle iff it +# has the allocator's shape AND names an existing node file. The shape check +# comes first so a caller-supplied string can never traverse out of nodes/. +np__is_handle() { + case "${1:-}" in + n) return 1 ;; + n*) case "${1#n}" in '' | *[!0-9]*) return 1 ;; esac ;; + *) return 1 ;; + esac + [ -f "$NP_TRACE_DIR/nodes/$1" ] +} + +np__node_set() { + _node_set_file="$NP_TRACE_DIR/nodes/$1" + [ -f "$_node_set_file" ] || return 0 + # Drop any prior value for this key, then append the new one. The trailing + # '=' in the match means a key that is a prefix of another never collides. + if grep -q "^$2=" "$_node_set_file" 2>/dev/null; then + grep -v "^$2=" "$_node_set_file" > "$_node_set_file.tmp" 2>/dev/null || : > "$_node_set_file.tmp" + mv "$_node_set_file.tmp" "$_node_set_file" + fi + printf '%s=%s\n' "$2" "$3" >> "$_node_set_file" + return 0 +} + +np__node_get() { + _node_get_file="$NP_TRACE_DIR/nodes/$1" + [ -f "$_node_get_file" ] || return 0 + # Strip only the leading "key=", so a value containing '=' survives intact. + sed -n "s/^$2=//p" "$_node_get_file" 2>/dev/null | head -n 1 + return 0 +} + +# Ambient resolution, exactly two levels. There is deliberately no third, +# session-wide level: that is where concurrent writers race. +# +# 1. NP_TRACE_CURRENT — explicit, and what you export to cross a CI step. +# 2. current.$$ — auto-maintained within one process tree. POSIX $$ +# does not change in a subshell, so a handle created +# inside $(...) is visible to the caller. +np__ambient() { + if [ -n "${NP_TRACE_CURRENT:-}" ]; then + printf '%s' "$NP_TRACE_CURRENT" + return 0 + fi + cat "$NP_TRACE_DIR/current.$$" 2>/dev/null || printf '' + return 0 +} + +np__ambient_set() { + printf '%s' "$1" > "$NP_TRACE_DIR/current.$$" 2>/dev/null || return 0 + return 0 +} + +np__ambient_clear() { + # Only clear when the cleared handle IS current, so terminalizing an outer + # node cannot silently retarget an inner one. + if [ "$(np__ambient)" = "$1" ]; then + rm -f "$NP_TRACE_DIR/current.$$" 2>/dev/null || : + if [ -n "${NP_TRACE_CURRENT:-}" ] && [ "$NP_TRACE_CURRENT" = "$1" ]; then + NP_TRACE_CURRENT='' + fi + fi + return 0 +} + +# Every node-scoped public function starts here: use $1 when it is a handle, +# otherwise fall back to the ambient node. +np__resolve_handle() { + if np__is_handle "${1:-}"; then + printf '%s' "$1" + else + np__ambient + fi + return 0 +} + +# ---- src/spool.sh ---- +# spool.sh — the emit hot path. Every emit is a LOCAL FILE WRITE: the network +# is never touched here, which is what makes API downtime invisible to the +# caller. The spool file's NAME is the event id, so re-POSTing after a crash is +# idempotent — that is recover() for free. + +# np__spool -> prints the event id +np__spool() { + _spool_id=$(np__uuidv7) + _spool_env=$(np__json_obj_raw \ + id "$(np__json_str "$_spool_id")" \ + time "$(np__json_str "$(np__iso8601)")" \ + type "$(np__json_str "$1")" \ + nrn "$(if [ -n "$2" ]; then np__json_str "$2"; fi)" \ + producer "$(np__json_str "${NP_TRACE_PRODUCER:-}")" \ + data "$3") + + _spool_tmp="$NP_TRACE_DIR/spool/$_spool_id.json.tmp" + _spool_final="$NP_TRACE_DIR/spool/$_spool_id.json" + printf '%s' "$_spool_env" > "$_spool_tmp" 2>/dev/null || return 0 + # Create-then-rename: a concurrent flush never sees a half-written envelope. + mv "$_spool_tmp" "$_spool_final" 2>/dev/null || return 0 + printf '%s' "$_spool_id" + return 0 +} + +np__spool_count() { + _spool_count_n=0 + for _spool_count_f in "$NP_TRACE_DIR/spool"/*.json; do + [ -f "$_spool_count_f" ] || continue + _spool_count_n=$((_spool_count_n + 1)) + done + printf '%s' "$_spool_count_n" + return 0 +} + +# ---- src/http.sh ---- +# http.sh — the only module that touches the network. Every request is bounded +# by a connect AND a total timeout, so an unreachable or hanging API can never +# stall the caller. + +NP_TRACE_CONNECT_TIMEOUT="${NP_TRACE_CONNECT_TIMEOUT:-3}" +NP_TRACE_MAX_TIME="${NP_TRACE_MAX_TIME:-10}" +NP_TRACE_DEFAULT_BASE_URL='https://api.nullplatform.com/tracing' +NP_TRACE_DEFAULT_AUTH_URL='https://api.nullplatform.com' + +np__drop() { + printf '%s\t%s\t%s\n' "$(np__iso8601)" "$1" "$2" >> "$NP_TRACE_DIR/drops.log" 2>/dev/null || : + if [ -n "${NP_TRACE_ON_DROP:-}" ]; then + "$NP_TRACE_ON_DROP" "$1" "$2" 2>/dev/null || : + fi + if [ -n "${NP_TRACE_DEBUG:-}" ]; then + printf 'np-trace drop: %s (%s)\n' "$1" "$2" >&2 + fi + return 0 +} + +# Suppress xtrace for a credential-handling region, remembering whether it was +# on. CI scripts routinely `set -x`, and shell options are global — so without +# this a sourced SDK function would print the bearer token into the build log +# even though it never reaches curl's argv. Every credential path is bracketed +# by np__secret_begin / np__secret_end. +np__secret_begin() { + case "$-" in + *x*) NP_TRACE_XTRACE=1; set +x ;; + *) NP_TRACE_XTRACE='' ;; + esac +} + +np__secret_end() { + if [ -n "${NP_TRACE_XTRACE:-}" ]; then + NP_TRACE_XTRACE='' + set -x + fi + return 0 +} + +# A bearer token. A pre-issued NP_TRACE_TOKEN wins; otherwise exchange the api +# key, caching until shortly before expiry. Called LAZILY, at first flush — +# never at init, so a down auth endpoint cannot delay pipeline startup. +np__token() { + np__secret_begin + if [ -n "${NP_TRACE_TOKEN:-}" ]; then + printf '%s' "$NP_TRACE_TOKEN" + np__secret_end + return 0 + fi + np__token_exchange + np__secret_end + return 0 +} + +# The api-key exchange. Always called from inside a secret region. +np__token_exchange() { + if [ -z "${NP_TRACE_API_KEY:-}" ]; then + printf '' + return 0 + fi + + _token_exchange_cache="$NP_TRACE_DIR/token" + if [ -f "$_token_exchange_cache" ]; then + _token_exchange_exp=$(sed -n '1p' "$_token_exchange_cache" 2>/dev/null) + _token_exchange_val=$(sed -n '2p' "$_token_exchange_cache" 2>/dev/null) + case "$_token_exchange_exp" in + '' | *[!0-9]*) _token_exchange_exp=0 ;; + esac + if [ -n "$_token_exchange_val" ] && [ "$_token_exchange_exp" -gt "$(date +%s)" ]; then + printf '%s' "$_token_exchange_val" + return 0 + fi + fi + + _token_exchange_body=$(curl -sS -X POST \ + --connect-timeout "$NP_TRACE_CONNECT_TIMEOUT" --max-time "$NP_TRACE_MAX_TIME" \ + -H 'Content-Type: application/json' \ + -d "$(np__json_obj apiKey "$NP_TRACE_API_KEY")" \ + "${NP_TRACE_AUTH_URL:-$NP_TRACE_DEFAULT_AUTH_URL}/token" 2>/dev/null) || _token_exchange_body='' + + _token_exchange_new=$(printf '%s' "$_token_exchange_body" | + sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p') + if [ -z "$_token_exchange_new" ]; then + np__drop 'auth' 'token exchange failed' + printf '' + return 0 + fi + ( umask 077; printf '%s\n%s\n' "$(( $(date +%s) + 3540 ))" "$_token_exchange_new" > "$_token_exchange_cache" ) + printf '%s' "$_token_exchange_new" + return 0 +} + +# The auth header goes to curl via --config from a mode-600 file, NEVER as -H +# in argv: CI runs with `set -x`, and an argv-borne header prints the token +# straight into the build log. +np__auth_config() { + np__secret_begin + _auth_config_file="$NP_TRACE_DIR/curlcfg.$$" + ( umask 077; printf 'header = "Authorization: Bearer %s"\n' "$(np__token)" > "$_auth_config_file" ) + np__secret_end + printf '%s' "$_auth_config_file" + return 0 +} + +# POST one spool file. Prints the HTTP status code, or 000 on a network failure. +np__post_event() { + _post_event_cfg=$(np__auth_config) + _post_event_code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + --config "$_post_event_cfg" \ + --connect-timeout "$NP_TRACE_CONNECT_TIMEOUT" --max-time "$NP_TRACE_MAX_TIME" \ + -H 'Content-Type: application/json' \ + --data-binary "@$1" \ + "${NP_TRACE_BASE_URL:-$NP_TRACE_DEFAULT_BASE_URL}/events" 2>/dev/null) || _post_event_code='000' + rm -f "$_post_event_cfg" 2>/dev/null || : + case "$_post_event_code" in + '' | *[!0-9]*) _post_event_code='000' ;; + esac + printf '%s' "$_post_event_code" + return 0 +} + +# ---- src/flush.sh ---- +# flush.sh — the spool drain. Bounded by a wall-clock budget so a dead API can +# never hang process exit; every path returns 0. + +NP_TRACE_FLUSH_TIMEOUT="${NP_TRACE_FLUSH_TIMEOUT:-10}" +NP_TRACE_MAX_RETRIES="${NP_TRACE_MAX_RETRIES:-3}" + +np__attempts_of() { + _attempts_of_n=$(cat "$1.attempts" 2>/dev/null || printf '0') + case "$_attempts_of_n" in + '' | *[!0-9]*) _attempts_of_n=0 ;; + esac + printf '%s' "$_attempts_of_n" +} + +np__fail_event() { + mv "$1" "$NP_TRACE_DIR/failed/" 2>/dev/null || rm -f "$1" 2>/dev/null || : + rm -f "$1.attempts" 2>/dev/null || : + np__drop "${1##*/}" "$2" + return 0 +} + +np_trace_flush() { + [ -n "${NP_TRACE_DIR:-}" ] || return 0 + [ -d "$NP_TRACE_DIR/spool" ] || return 0 + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _flush_deadline=$(( $(date +%s) + NP_TRACE_FLUSH_TIMEOUT )) + + for _flush_file in "$NP_TRACE_DIR/spool"/*.json; do + [ -f "$_flush_file" ] || continue + if [ "$(date +%s)" -ge "$_flush_deadline" ]; then + # Budget spent. Remaining events stay on disk for the next flush or a + # later np_trace_recover; the process exits on time regardless. This is + # the guarantee that a dead API cannot hang a build. + return 0 + fi + + _flush_code=$(np__post_event "$_flush_file") + case "$_flush_code" in + 201 | 200) + # 200 is an idempotent re-POST of an already-accepted event. + rm -f "$_flush_file" "$_flush_file.attempts" 2>/dev/null || : + ;; + 400) + # A contract violation. Never retried — retrying cannot change it. + np__fail_event "$_flush_file" "rejected 400" + ;; + 401 | 403) + rm -f "$NP_TRACE_DIR/token" 2>/dev/null || : + np__fail_event "$_flush_file" "unauthorized $_flush_code" + ;; + *) + _flush_n=$(( $(np__attempts_of "$_flush_file") + 1 )) + if [ "$_flush_n" -gt "$NP_TRACE_MAX_RETRIES" ]; then + np__fail_event "$_flush_file" "gave up after $_flush_n attempts (last status $_flush_code)" + else + printf '%s' "$_flush_n" > "$_flush_file.attempts" 2>/dev/null || : + fi + ;; + esac + done + return 0 +} + +np_trace_shutdown() { + np_trace_flush + if [ -n "${NP_TRACE_DIR:-}" ] && [ "${NP_TRACE_KEEP_STATE:-0}" != '1' ]; then + rm -rf "$NP_TRACE_DIR" 2>/dev/null || : + fi + return 0 +} + +# Re-deliver a previous process's leftover spool. Idempotent by construction: +# the spool file name IS the event id, so the API answers a re-POST with +# 200 duplicate. +np_trace_recover() { + np_trace_flush + return 0 +} + +np__install_trap() { + if [ -z "${NP_TRACE_NO_TRAP:-}" ]; then + trap 'np_trace_flush' EXIT + trap 'np_trace_flush' INT + trap 'np_trace_flush' TERM + fi + return 0 +} + +# ---- src/propagation.sh ---- +# --------------------------------------------------------------------------- +# Propagation +# +# Cross-process trace context, wire-identical to the Go and JS SDKs: a single +# carrier value packing "||". The '|' delimiter is +# reserved, so the value splits unambiguously even though a run_id may itself +# contain '~' and '@'. +# +# The carrier travels in the NP_TRACE environment variable. Note that this is +# deliberately OUTSIDE the NP_TRACE_* configuration namespace the SDK reads for +# its own settings: NP_TRACE is context handed to us by a caller, not something +# a user configures. +# --------------------------------------------------------------------------- + +# np_trace_inject [handle] +# +# Print the carrier value for a handle (defaults to the ambient node), for +# handing to a child process. Prints nothing when there is no node to inject, +# so `NP_TRACE=$(np_trace_inject)` is always safe. +np_trace_inject() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _inject_h=$(np__resolve_handle "${1:-}") + np__is_handle "$_inject_h" || return 0 + printf '%s%s%s%s%s' \ + "$NP_CARRIER_VERSION" "$NP_CARRIER_DELIMITER" \ + "$(np__node_get "$_inject_h" trace_id)" "$NP_CARRIER_DELIMITER" \ + "$(np__node_get "$_inject_h" run_id)" + return 0 +} + +# np_trace_extract [carrier] +# +# Parse a carrier value (defaults to $NP_TRACE) and print " ". +# Returns 1 when there is no usable context, so callers can branch: +# +# if ctx=$(np_trace_extract); then set -- $ctx; fi +# +# When only a trace id is present it is used for both, matching the Go SDK, so +# the result is always a usable pair. +np_trace_extract() { + _extract_raw=${1-${NP_TRACE:-}} + [ -n "$_extract_raw" ] || return 1 + + case "$_extract_raw" in + "$NP_CARRIER_VERSION$NP_CARRIER_DELIMITER"*) ;; + *) return 1 ;; + esac + _extract_rest=${_extract_raw#*"$NP_CARRIER_DELIMITER"} + + # trace_id is up to the next delimiter; run_id is the whole remainder, which + # may itself contain '~' and '@' but never a delimiter. + case "$_extract_rest" in + *"$NP_CARRIER_DELIMITER"*) + _extract_trace=${_extract_rest%%"$NP_CARRIER_DELIMITER"*} + _extract_run=${_extract_rest#*"$NP_CARRIER_DELIMITER"} + ;; + *) + _extract_trace=$_extract_rest + _extract_run=$_extract_rest + ;; + esac + [ -n "$_extract_trace" ] || return 1 + [ -n "$_extract_run" ] || _extract_run=$_extract_trace + + printf '%s %s' "$_extract_trace" "$_extract_run" + return 0 +} + +# np_trace_adopt [carrier] +# +# Attach to an upstream node and return a handle standing in for it, so work +# started here nests UNDERNEATH it: +# +# parent=$(np_trace_adopt) || parent=$(np_trace_run --run-id "$(np_trace_occurrence)") +# step=$(np_trace_step "$parent" build) +# +# The adopted node belongs to whoever created it — typically the np CLI, which +# exports NP_TRACE per workflow step. We hold its ids so children derive +# correctly, but must never speak for it: it is marked foreign, so it emits no +# node event of its own and the terminal verbs refuse to close it. Children +# hanging off it still emit their own containment edges, which IS ours to say. +# +# Returns 1 when there is no upstream context, leaving the caller to open a root +# run instead. +np_trace_adopt() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 1 + _adopt_ctx=$(np_trace_extract "${1-${NP_TRACE:-}}") || return 1 + _adopt_trace=${_adopt_ctx%% *} + _adopt_run=${_adopt_ctx#* } + + if ! _adopt_why=$(np__trace_id_violation "$_adopt_trace"); then + np__drop 'adopt' "trace_id $_adopt_why" + return 1 + fi + # An upstream run_id is commonly a DERIVED path (parent~key@attempt.iteration) + # rather than a named id — the np CLI hands us the step it is running. Accept + # either: parse it as a node path first, and only fall back to the named-id + # rules when it has no delimiter. + if ! np__parse_node_id "$_adopt_run" >/dev/null 2>&1; then + if ! _adopt_why=$(np__named_id_violation "$_adopt_run"); then + np__drop 'adopt' "run_id $_adopt_why" + return 1 + fi + fi + + _adopt_h=$(np__handle_new) + np__node_set "$_adopt_h" kind run + np__node_set "$_adopt_h" trace_id "$_adopt_trace" + np__node_set "$_adopt_h" run_id "$_adopt_run" + np__node_set "$_adopt_h" nrn "${NP_TRACE_NRN:-}" + np__node_set "$_adopt_h" foreign 1 + # started=1 suppresses the lazy `started` emit; closed=0 keeps it usable as a + # parent for the whole script. + np__node_set "$_adopt_h" started 1 + np__node_set "$_adopt_h" closed 0 + np__ambient_set "$_adopt_h" + printf '%s' "$_adopt_h" + return 0 +} + +# True when a handle stands in for a node owned by another process. +np__is_foreign() { + [ "$(np__node_get "$1" foreign)" = '1' ] +} + +# ---- src/api.sh ---- +# api.sh — the public producer surface. Every function here returns 0, always: +# tracing must never fail the caller. +# +# Every node-scoped function takes an OPTIONAL leading handle. This is one +# function with a defaulted argument, not two ways to say the same thing: when +# the first argument is not a handle it falls back to the innermost open node. + +np_trace_init() { + while [ "$#" -gt 0 ]; do + case "$1" in + --producer) NP_TRACE_PRODUCER=${2:-}; shift 2 ;; + --base-url) NP_TRACE_BASE_URL=${2:-}; shift 2 ;; + --auth-url) NP_TRACE_AUTH_URL=${2:-}; shift 2 ;; + --api-key) NP_TRACE_API_KEY=${2:-}; shift 2 ;; + --token) NP_TRACE_TOKEN=${2:-}; shift 2 ;; + --nrn) NP_TRACE_NRN=${2:-}; shift 2 ;; + --enabled) NP_TRACE_ENABLED=${2:-1}; shift 2 ;; + --no-trap) NP_TRACE_NO_TRAP=1; shift ;; + *) shift ;; + esac + done + NP_TRACE_ENABLED="${NP_TRACE_ENABLED:-1}" + np__state_init + # No network call here, deliberately: a down auth endpoint must never delay + # the start of a pipeline. The token is fetched lazily, at first flush. + np__install_trap + return 0 +} + +# --------------------------------------------------------------------------- +# Emission +# --------------------------------------------------------------------------- + +# Emit the node event for a handle at the given status, carrying whatever +# context is currently staged. +np__emit_node() { + _emit_node_h=$1 + _emit_node_status=$2 + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + + _emit_node_labels=$(np__node_get "$_emit_node_h" labels) + _emit_node_facets=$(np__node_get "$_emit_node_h" facets) + _emit_node_key=$(np__node_get "$_emit_node_h" key) + _emit_node_schema=$(np__node_get "$_emit_node_h" schema_url) + + if [ -n "$_emit_node_key" ]; then + _emit_node_data=$(np__json_obj_raw \ + trace_id "$(np__json_str "$(np__node_get "$_emit_node_h" trace_id)")" \ + run_id "$(np__json_str "$(np__node_get "$_emit_node_h" run_id)")" \ + key "$(np__json_str "$_emit_node_key")" \ + attempt "$(np__node_get "$_emit_node_h" attempt)" \ + iteration "$(np__node_get "$_emit_node_h" iteration)" \ + status "$(np__json_str "$_emit_node_status")" \ + labels "$_emit_node_labels" \ + facets "$_emit_node_facets" \ + schema_url "$(if [ -n "$_emit_node_schema" ]; then np__json_str "$_emit_node_schema"; fi)") + else + _emit_node_data=$(np__json_obj_raw \ + trace_id "$(np__json_str "$(np__node_get "$_emit_node_h" trace_id)")" \ + run_id "$(np__json_str "$(np__node_get "$_emit_node_h" run_id)")" \ + status "$(np__json_str "$_emit_node_status")" \ + labels "$_emit_node_labels" \ + facets "$_emit_node_facets" \ + schema_url "$(if [ -n "$_emit_node_schema" ]; then np__json_str "$_emit_node_schema"; fi)") + fi + + np__spool "$NP_TYPE_NODE_RUN" "$(np__node_get "$_emit_node_h" nrn)" "$_emit_node_data" >/dev/null + return 0 +} + +# A run ref for a handle — the self-describing address used on edge endpoints. +np__ref_of() { + np__json_obj \ + type run \ + trace_id "$(np__node_get "$1" trace_id)" \ + run_id "$(np__node_get "$1" run_id)" +} + +np__emit_parent_edge() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _emit_parent_edge_data=$(np__json_obj_raw from "$(np__ref_of "$1")" to "$(np__ref_of "$2")") + np__spool "$NP_TYPE_EDGE_PARENT" "$(np__node_get "$1" nrn)" "$_emit_parent_edge_data" >/dev/null + return 0 +} + +# Force the lazy `started`. Idempotent. +# +# Shell has no microtask, so `started` is emitted at the first event that must +# follow it — a terminal, a child open, an explicit call, or flush. Context +# staged before that lands on `started`; context staged after lands on the +# terminal. Same observable semantics as the JS and Go SDKs, without a timer. +np_trace_start() { + _start_h=$(np__resolve_handle "${1:-}") + np__is_handle "$_start_h" || return 0 + if [ "$(np__node_get "$_start_h" started)" = '1' ]; then + return 0 + fi + np__node_set "$_start_h" started 1 + np__emit_node "$_start_h" "$NP_STATUS_STARTED" + return 0 +} + +# --------------------------------------------------------------------------- +# Nodes +# --------------------------------------------------------------------------- + +np_trace_run() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _run_trace='' + _run_run='' + _run_nrn="${NP_TRACE_NRN:-}" + while [ "$#" -gt 0 ]; do + case "$1" in + --trace-id) _run_trace=${2:-}; shift 2 ;; + --run-id) _run_run=${2:-}; shift 2 ;; + --nrn) _run_nrn=${2:-}; shift 2 ;; + *) shift ;; + esac + done + # A lone root run's trace_id defaults to its run_id, and vice versa. + [ -n "$_run_trace" ] || _run_trace=$_run_run + [ -n "$_run_run" ] || _run_run=$_run_trace + + if ! _run_why=$(np__trace_id_violation "$_run_trace"); then + np__drop 'run' "trace_id $_run_why" + return 0 + fi + if ! _run_why=$(np__named_id_violation "$_run_run"); then + np__drop 'run' "run_id $_run_why" + return 0 + fi + + _run_h=$(np__handle_new) + np__node_set "$_run_h" kind run + np__node_set "$_run_h" trace_id "$_run_trace" + np__node_set "$_run_h" run_id "$_run_run" + np__node_set "$_run_h" nrn "$_run_nrn" + np__node_set "$_run_h" auto_started_at "$(np__iso8601)" + np__node_set "$_run_h" started 0 + np__node_set "$_run_h" closed 0 + np__ambient_set "$_run_h" + printf '%s' "$_run_h" + return 0 +} + +# np_trace_step [handle] [--attempt N] [--iteration N] +np_trace_step() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _step_parent=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + _step_key=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + _step_attempt=0 + _step_iteration=0 + while [ "$#" -gt 0 ]; do + case "$1" in + --attempt) _step_attempt=${2:-0}; shift 2 ;; + --iteration) _step_iteration=${2:-0}; shift 2 ;; + *) shift ;; + esac + done + + if ! np__is_handle "$_step_parent"; then + np__drop 'step' 'no parent node in scope' + return 0 + fi + if ! _step_why=$(np__key_violation "$_step_key"); then + np__drop 'step' "key $_step_why" + return 0 + fi + case "$_step_attempt$_step_iteration" in + '' | *[!0-9]*) np__drop 'step' 'attempt and iteration must be integers'; return 0 ;; + esac + + # Opening a child forces the parent's started: a parent edge must not point + # at a node the read model has never seen. + np_trace_start "$_step_parent" + + _step_id=$(np__derive_child_id "$(np__node_get "$_step_parent" run_id)" \ + "$_step_key" "$_step_attempt" "$_step_iteration") + + _step_h=$(np__handle_new) + np__node_set "$_step_h" kind step + np__node_set "$_step_h" trace_id "$(np__node_get "$_step_parent" trace_id)" + np__node_set "$_step_h" run_id "$_step_id" + np__node_set "$_step_h" nrn "$(np__node_get "$_step_parent" nrn)" + np__node_set "$_step_h" key "$_step_key" + np__node_set "$_step_h" attempt "$_step_attempt" + np__node_set "$_step_h" iteration "$_step_iteration" + np__node_set "$_step_h" parent "$_step_parent" + np__node_set "$_step_h" auto_started_at "$(np__iso8601)" + np__node_set "$_step_h" started 0 + np__node_set "$_step_h" closed 0 + + np_trace_start "$_step_h" + np__emit_parent_edge "$_step_parent" "$_step_h" + np__ambient_set "$_step_h" + printf '%s' "$_step_h" + return 0 +} + +# A named child run — a new scope under the same trace. +np_trace_child() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _child_parent=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + _child_run='' + while [ "$#" -gt 0 ]; do + case "$1" in + --run-id) _child_run=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if ! np__is_handle "$_child_parent"; then + np__drop 'child' 'no parent node in scope' + return 0 + fi + if ! _child_why=$(np__named_id_violation "$_child_run"); then + np__drop 'child' "run_id $_child_why" + return 0 + fi + np_trace_start "$_child_parent" + _child_h=$(np_trace_run --trace-id "$(np__node_get "$_child_parent" trace_id)" \ + --run-id "$_child_run" \ + --nrn "$(np__node_get "$_child_parent" nrn)") + np__is_handle "$_child_h" || return 0 + np__node_set "$_child_h" parent "$_child_parent" + np_trace_start "$_child_h" + np__emit_parent_edge "$_child_parent" "$_child_h" + np__ambient_set "$_child_h" + printf '%s' "$_child_h" + return 0 +} + +# --------------------------------------------------------------------------- +# Staging context +# --------------------------------------------------------------------------- + +# Merge a pre-formed `"key":value` fragment into the node's staged labels. +np__stage_label() { + _stage_label_cur=$(np__node_get "$1" labels) + if [ -z "$_stage_label_cur" ] || [ "$_stage_label_cur" = '{}' ]; then + np__node_set "$1" labels "{$2}" + else + np__node_set "$1" labels "${_stage_label_cur%\}},$2}" + fi + return 0 +} + +np__stage_facet() { + _stage_facet_cur=$(np__node_get "$1" facets) + _stage_facet_entry="$(np__json_str "$2"):$3" + if [ -z "$_stage_facet_cur" ] || [ "$_stage_facet_cur" = '{}' ]; then + np__node_set "$1" facets "{$_stage_facet_entry}" + else + # Last write wins per namespace: drop any prior entry for this facet. + np__node_set "$1" facets "${_stage_facet_cur%\}},$_stage_facet_entry}" + fi + return 0 +} + +# Staged context normally rides the node's NEXT lifecycle emit. A FOREIGN +# (adopted) node never has one here — its owner closes it in another process — +# so anything staged on it would die in local state. Re-emit `started` with the +# full current bag instead (additive, the same shape the JS SDK's +# late-enrichment flush produces): the fold keeps the node's real outcome (the +# owner's terminal is later by time) and gains the facts this process observed. +np__flush_foreign() { + [ "$(np__node_get "$1" foreign)" = '1' ] || return 0 + np__emit_node "$1" "$NP_STATUS_STARTED" + return 0 +} + +# np_trace_labels [handle] key=value ... +np_trace_labels() { + _labels_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_labels_h" || return 0 + for _labels_pair in "$@"; do + case "$_labels_pair" in + *=*) ;; + *) continue ;; + esac + _labels_k=${_labels_pair%%=*} + _labels_v=${_labels_pair#*=} + # An absent optional is omitted, never recorded as the string "null". + if [ -n "$_labels_k" ] && [ -n "$_labels_v" ]; then + np__stage_label "$_labels_h" "$(np__json_str "$_labels_k"):$(np__json_str "$_labels_v")" + fi + done + np__flush_foreign "$_labels_h" + return 0 +} + +# np_trace_facet [handle] — your own namespace. +np_trace_facet() { + _facet_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_facet_h" || return 0 + if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then + return 0 + fi + np__stage_facet "$_facet_h" "$1" "$2" + np__flush_foreign "$_facet_h" + return 0 +} + +np_trace_schema() { + _schema_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_schema_h" || return 0 + np__node_set "$_schema_h" schema_url "${1:-}" + return 0 +} + +np_trace_explain() { + _explain_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_explain_h" || return 0 + _explain_title='' + _explain_what='' + _explain_why='' + _explain_impact='' + _explain_next='' + _explain_sev='' + while [ "$#" -gt 0 ]; do + case "$1" in + --title) _explain_title=${2:-}; shift 2 ;; + --what) _explain_what=${2:-}; shift 2 ;; + --why) _explain_why=${2:-}; shift 2 ;; + --impact) _explain_impact=${2:-}; shift 2 ;; + --next) _explain_next=${2:-}; shift 2 ;; + --severity) _explain_sev=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_explain_title" ]; then + np__drop 'explain' 'title is required' + return 0 + fi + np__stage_facet "$_explain_h" "$NP_FACET_EXPLAIN" \ + "$(np__json_obj title "$_explain_title" severity "$_explain_sev" what "$_explain_what" \ + why "$_explain_why" impact "$_explain_impact" next "$_explain_next")" + np__flush_foreign "$_explain_h" + return 0 +} + +np_trace_error() { + _error_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_error_h" || return 0 + _error_msg='' + _error_code='' + _error_stack='' + _error_details='' + while [ "$#" -gt 0 ]; do + case "$1" in + --message) _error_msg=${2:-}; shift 2 ;; + --code) _error_code=${2:-}; shift 2 ;; + --stack-trace) _error_stack=${2:-}; shift 2 ;; + # A JSON object with the diagnosis's structured evidence (counts, the + # failing probe, ...) — the sibling SDKs' error `details`. + --details) _error_details=${2:-}; shift 2 ;; + *) + if [ -z "$_error_msg" ]; then + _error_msg=$1 + fi + shift + ;; + esac + done + [ -n "$_error_msg" ] || return 0 + case "$_error_details" in + '' | \{*) ;; + *) _error_details='' ;; + esac + np__stage_facet "$_error_h" "$NP_FACET_ERROR" \ + "$(np__json_obj_raw \ + message "$(np__json_str "$_error_msg")" \ + code "$(if [ -n "$_error_code" ]; then np__json_str "$_error_code"; fi)" \ + stack_trace "$(if [ -n "$_error_stack" ]; then np__json_str "$_error_stack"; fi)" \ + details "$_error_details")" + np__flush_foreign "$_error_h" + return 0 +} + +np_trace_timing() { + _timing_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_timing_h" || return 0 + while [ "$#" -gt 0 ]; do + case "$1" in + --started-at) np__node_set "$_timing_h" started_at "${2:-}"; shift 2 ;; + --ended-at) np__node_set "$_timing_h" ended_at "${2:-}"; shift 2 ;; + *) shift ;; + esac + done + return 0 +} + +# Stamp the auto timing facet, letting any manual override win per field. +np__stage_timing() { + _stage_timing_started=$(np__node_get "$1" started_at) + _stage_timing_ended=$(np__node_get "$1" ended_at) + [ -n "$_stage_timing_started" ] || _stage_timing_started=$(np__node_get "$1" auto_started_at) + [ -n "$_stage_timing_ended" ] || _stage_timing_ended=$2 + np__stage_facet "$1" "$NP_FACET_TIMING" \ + "$(np__json_obj started_at "$_stage_timing_started" ended_at "$_stage_timing_ended")" + return 0 +} + +# --------------------------------------------------------------------------- +# Lineage — produces/consumes edges with io pointers +# --------------------------------------------------------------------------- + +# A dataset ref for an edge endpoint. The id is the CANONICAL dataset id — the +# exact string a producer and a consumer must both name for lineage to join +# them by value (an ARN, an FQDN, `:` for an asset) — never a +# synthesised id. +np__dataset_ref() { + np__json_obj type dataset id "$1" +} + +# Append one io descriptor to a direction's list; the facet is re-staged +# whole each time (last write wins per namespace), so the array only ever +# grows. $1 handle, $2 facet namespace, $3 descriptor store key, $4 the +# already-formed descriptor JSON. +np__append_io_descriptor() { + _append_io_descriptor_descriptors=$(np__node_get "$1" "$3") + if [ -n "$_append_io_descriptor_descriptors" ]; then + _append_io_descriptor_descriptors="$_append_io_descriptor_descriptors,$4" + else + _append_io_descriptor_descriptors=$4 + fi + np__node_set "$1" "$3" "$_append_io_descriptor_descriptors" + np__stage_facet "$1" "$2" "[$_append_io_descriptor_descriptors]" + return 0 +} + +# Build one io descriptor from its parsed parts, choosing the kind by which +# parts are present: a uri is a POINTER (large data referenced, not inlined), +# a source+external-id is a REF (an entity in an external catalog), a JSON +# value is INLINE (carried in the event itself). Prints the descriptor, or +# nothing (with a drop) when the parts don't form one. +# $1 verb (for drop records), $2 name, $3 inline JSON, $4 uri, $5 ref source, +# $6 ref external id, $7 ref version. +np__build_io_descriptor() { + _build_io_descriptor_verb=$1 + _build_io_descriptor_name=$2 + _build_io_descriptor_inline=$3 + _build_io_descriptor_uri=$4 + _build_io_descriptor_ref_source=$5 + _build_io_descriptor_ref_id=$6 + _build_io_descriptor_ref_version=$7 + if [ -z "$_build_io_descriptor_name" ]; then + np__drop "$_build_io_descriptor_verb" 'a descriptor name is required' + return 1 + fi + if [ -n "$_build_io_descriptor_uri" ]; then + np__json_obj kind pointer name "$_build_io_descriptor_name" uri "$_build_io_descriptor_uri" + return 0 + fi + if [ -n "$_build_io_descriptor_ref_source" ] && [ -n "$_build_io_descriptor_ref_id" ]; then + np__json_obj kind ref name "$_build_io_descriptor_name" source "$_build_io_descriptor_ref_source" \ + external_id "$_build_io_descriptor_ref_id" version "$_build_io_descriptor_ref_version" + return 0 + fi + if [ -n "$_build_io_descriptor_inline" ]; then + case "$_build_io_descriptor_inline" in + \{* | \[* | \"* | [0-9-]* | true | false | null) + np__json_obj_raw kind '"inline"' name "$(np__json_str "$_build_io_descriptor_name")" value "$_build_io_descriptor_inline" + return 0 + ;; + esac + np__drop "$_build_io_descriptor_verb" 'value must be JSON' + return 1 + fi + np__drop "$_build_io_descriptor_verb" 'a JSON value, --uri, or --source + --external-id is required' + return 1 +} + +# The shared body of np_trace_output / np_trace_input. +# $1 direction (out|in), $2 verb, then the caller's argv: +# [handle] [] [--uri U] [--source S --external-id E [--version V]] +np__declare_io() { + _declare_io_direction=$1 + _declare_io_verb=$2 + shift 2 + _declare_io_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_declare_io_handle" || { np__drop "$_declare_io_verb" 'no node in scope'; return 0; } + _declare_io_name=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + _declare_io_inline='' + _declare_io_uri='' + _declare_io_ref_source='' + _declare_io_ref_id='' + _declare_io_ref_version='' + while [ "$#" -gt 0 ]; do + case "$1" in + --uri) _declare_io_uri=${2:-}; shift 2 ;; + --source) _declare_io_ref_source=${2:-}; shift 2 ;; + --external-id) _declare_io_ref_id=${2:-}; shift 2 ;; + --version) _declare_io_ref_version=${2:-}; shift 2 ;; + *) + if [ -z "$_declare_io_inline" ]; then + _declare_io_inline=$1 + fi + shift + ;; + esac + done + _declare_io_descriptor=$(np__build_io_descriptor "$_declare_io_verb" "$_declare_io_name" "$_declare_io_inline" \ + "$_declare_io_uri" "$_declare_io_ref_source" "$_declare_io_ref_id" "$_declare_io_ref_version") || return 0 + if [ "$_declare_io_direction" = 'out' ]; then + np__append_io_descriptor "$_declare_io_handle" "$NP_FACET_OUTPUT" io_output "$_declare_io_descriptor" + else + np__append_io_descriptor "$_declare_io_handle" "$NP_FACET_INPUT" io_input "$_declare_io_descriptor" + fi + np__flush_foreign "$_declare_io_handle" + return 0 +} + +# np_trace_output [handle] [] [--uri U] [--source S --external-id E [--version V]] +# +# Record what this node PRODUCED: an inline value carried in the event +# (`np_trace_output instances '{"healthy":2}'`), a pointer to large data +# (`--uri`), or a ref to an external catalog entity (`--source`/`--external-id`). +# For an artifact that should ALSO join the lineage graph, prefer +# np_trace_produces (descriptor + edge in one call). +np_trace_output() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + np__declare_io out output "$@" + return 0 +} + +# np_trace_input [handle] [] [--uri U] [--source S --external-id E [--version V]] +# +# Record what this node CONSUMED; see np_trace_output. +np_trace_input() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + np__declare_io in input "$@" + return 0 +} + +# np__emit_io_edge [descriptor-json] +# +# Emit one lineage edge. The direction decides everything else: `out` is +# edge.produces + tracing.output, `in` is edge.consumes + tracing.input. +# +# With a descriptor the io is declared ONCE: it accumulates into the node's +# io facet AND becomes the edge's tracing.binding — the same single-source +# rule as the sibling SDKs. Without one, the edge records lineage only. +# +# On a FOREIGN (adopted) node this is an observed fact, exactly like +# np_trace_error: the edge is ours to say, and the staged io facet reaches the +# wire through the foreign re-emit. +np__emit_io_edge() { + _emit_io_edge_handle=$1 + _emit_io_edge_direction=$2 + _emit_io_edge_dataset_id=$3 + + if [ "$_emit_io_edge_direction" = 'out' ]; then + _emit_io_edge_edge_type=$NP_TYPE_EDGE_PRODUCES + _emit_io_edge_facet_namespace=$NP_FACET_OUTPUT + _emit_io_edge_descriptor_store=io_output + else + _emit_io_edge_edge_type=$NP_TYPE_EDGE_CONSUMES + _emit_io_edge_facet_namespace=$NP_FACET_INPUT + _emit_io_edge_descriptor_store=io_input + fi + + _emit_io_edge_binding=$4 + + if [ -n "$_emit_io_edge_binding" ]; then + np__append_io_descriptor "$_emit_io_edge_handle" "$_emit_io_edge_facet_namespace" "$_emit_io_edge_descriptor_store" "$_emit_io_edge_binding" + fi + + # An edge must not point FROM a node the read model has never seen. + np_trace_start "$_emit_io_edge_handle" + + if [ -n "$_emit_io_edge_binding" ]; then + _emit_io_edge_edge_data=$(np__json_obj_raw \ + from "$(np__ref_of "$_emit_io_edge_handle")" \ + to "$(np__dataset_ref "$_emit_io_edge_dataset_id")" \ + facets "{$(np__json_str "$NP_FACET_BINDING"):$_emit_io_edge_binding}") + else + _emit_io_edge_edge_data=$(np__json_obj_raw \ + from "$(np__ref_of "$_emit_io_edge_handle")" \ + to "$(np__dataset_ref "$_emit_io_edge_dataset_id")") + fi + np__spool "$_emit_io_edge_edge_type" "$(np__node_get "$_emit_io_edge_handle" nrn)" "$_emit_io_edge_edge_data" >/dev/null + np__flush_foreign "$_emit_io_edge_handle" + return 0 +} + +# The shared argv handling of np_trace_produces / np_trace_consumes: +# resolve the optional leading handle, take the dataset id, parse the +# pointer flags, and hand off to np__emit_io_edge. +# $1 direction (out|in), $2 verb name for drop records, then the caller's argv. +np__declare_lineage() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _declare_lineage_direction=$1 + _declare_lineage_verb=$2 + shift 2 + + _declare_lineage_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_declare_lineage_handle" || { np__drop "$_declare_lineage_verb" 'no node in scope'; return 0; } + + _declare_lineage_dataset_id=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + if [ -z "$_declare_lineage_dataset_id" ]; then + np__drop "$_declare_lineage_verb" 'dataset id is required' + return 0 + fi + + _declare_lineage_name='' + _declare_lineage_inline='' + _declare_lineage_uri='' + _declare_lineage_ref_source='' + _declare_lineage_ref_id='' + _declare_lineage_ref_version='' + while [ "$#" -gt 0 ]; do + case "$1" in + --name) _declare_lineage_name=${2:-}; shift 2 ;; + --uri) _declare_lineage_uri=${2:-}; shift 2 ;; + --value) _declare_lineage_inline=${2:-}; shift 2 ;; + --source) _declare_lineage_ref_source=${2:-}; shift 2 ;; + --external-id) _declare_lineage_ref_id=${2:-}; shift 2 ;; + --version) _declare_lineage_ref_version=${2:-}; shift 2 ;; + *) shift ;; + esac + done + + _declare_lineage_binding='' + if [ -n "$_declare_lineage_name" ]; then + _declare_lineage_binding=$(np__build_io_descriptor "$_declare_lineage_verb" "$_declare_lineage_name" "$_declare_lineage_inline" \ + "$_declare_lineage_uri" "$_declare_lineage_ref_source" "$_declare_lineage_ref_id" "$_declare_lineage_ref_version") || return 0 + fi + + np__emit_io_edge "$_declare_lineage_handle" "$_declare_lineage_direction" "$_declare_lineage_dataset_id" "$_declare_lineage_binding" + return 0 +} + +# np_trace_produces [handle] [--name (--uri U | --value JSON | --source S --external-id E [--version V])] +# +# Declare this node WROTE the dataset. With `--name` the io is declared once +# — a pointer (`--uri`, the artifact's address), an inline value (`--value`), +# or a catalog ref (`--source`/`--external-id`) — on both the node and the +# edge's binding. Bare form records lineage only. +np_trace_produces() { + np__declare_lineage out produces "$@" + return 0 +} + +# np_trace_consumes [handle] [--name (--uri U | --value JSON | --source S --external-id E [--version V])] +# +# Declare this node READ the dataset; see np_trace_produces. +np_trace_consumes() { + np__declare_lineage in consumes "$@" + return 0 +} + +# --------------------------------------------------------------------------- +# Run-to-run edges — how operations relate across the graph +# --------------------------------------------------------------------------- + +# Resolve an edge target: a handle from this process, or a PACKED CARRIER +# ("1||") — the natural address in shell, where the other +# end of an edge usually arrived via an env var. Prints the target's ref. +np__edge_target_ref() { + if np__is_handle "$1"; then + np__ref_of "$1" + return 0 + fi + _edge_target_ref_context=$(np_trace_extract "$1") || return 1 + _edge_target_ref_trace=${_edge_target_ref_context%% *} + _edge_target_ref_run=${_edge_target_ref_context#* } + np__json_obj type run trace_id "$_edge_target_ref_trace" run_id "$_edge_target_ref_run" + return 0 +} + +# Emit one relationship edge from a node this process holds. +# $1 handle, $2 edge type, $3 target ref JSON, $4 verb for drop records. +np__emit_ref_edge() { + _emit_ref_edge_from=$(np__ref_of "$1") + if [ "$_emit_ref_edge_from" = "$3" ]; then + np__drop "$4" 'self-edge forbidden' + return 0 + fi + # An edge must not point FROM a node the read model has never seen. + np_trace_start "$1" + _emit_ref_edge_data=$(np__json_obj_raw from "$_emit_ref_edge_from" to "$3") + np__spool "$2" "$(np__node_get "$1" nrn)" "$_emit_ref_edge_data" >/dev/null + np__flush_foreign "$1" + return 0 +} + +# The shared argv handling of the run-to-run edge verbs. +# $1 edge type, $2 verb, then the caller's argv: [handle] . +np__declare_relation() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _declare_relation_type=$1 + _declare_relation_verb=$2 + shift 2 + _declare_relation_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_declare_relation_handle" || { np__drop "$_declare_relation_verb" 'no node in scope'; return 0; } + if [ -z "${1:-}" ]; then + np__drop "$_declare_relation_verb" 'a target (handle or packed carrier) is required' + return 0 + fi + _declare_relation_target=$(np__edge_target_ref "$1") || { + np__drop "$_declare_relation_verb" 'target is not a handle or a valid carrier' + return 0 + } + np__emit_ref_edge "$_declare_relation_handle" "$_declare_relation_type" "$_declare_relation_target" "$_declare_relation_verb" + return 0 +} + +# np_trace_triggered_by [handle] +# +# The operation that CAUSED this one — a cross-trace fact (the target is +# usually another trace's run, addressed by its packed carrier). +np_trace_triggered_by() { + np__declare_relation "$NP_TYPE_EDGE_TRIGGERED_BY" triggered_by "$@" + return 0 +} + +# np_trace_retry_of [handle] — this run retries that one. +np_trace_retry_of() { + np__declare_relation "$NP_TYPE_EDGE_RETRY_OF" retry_of "$@" + return 0 +} + +# np_trace_continues [handle] — this run resumes that one's work. +np_trace_continues() { + np__declare_relation "$NP_TYPE_EDGE_CONTINUES" continues "$@" + return 0 +} + +# np_trace_correlates [handle] — related, with no causal claim. +np_trace_correlates() { + np__declare_relation "$NP_TYPE_EDGE_CORRELATES" correlates "$@" + return 0 +} + +# np_trace_compensates [handle] — this run undoes that one's effect. +np_trace_compensates() { + np__declare_relation "$NP_TYPE_EDGE_COMPENSATES" compensates "$@" + return 0 +} + +# np_trace_link [handle] +# +# Escape hatch over the named verbs — emit any known edge type. Prefer the +# named functions when one fits. +np_trace_link() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _link_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_link_handle" || { np__drop 'link' 'no node in scope'; return 0; } + _link_type=${1:-} + case "$_link_type" in + "$NP_TYPE_EDGE_TRIGGERED_BY" | "$NP_TYPE_EDGE_RETRY_OF" | "$NP_TYPE_EDGE_CONTINUES" \ + | "$NP_TYPE_EDGE_CORRELATES" | "$NP_TYPE_EDGE_COMPENSATES" | "$NP_TYPE_EDGE_PARENT") ;; + *) np__drop 'link' "unknown edge type '${_link_type}'"; return 0 ;; + esac + if [ -z "${2:-}" ]; then + np__drop 'link' 'a target (handle or packed carrier) is required' + return 0 + fi + _link_target=$(np__edge_target_ref "$2") || { + np__drop 'link' 'target is not a handle or a valid carrier' + return 0 + } + np__emit_ref_edge "$_link_handle" "$_link_type" "$_link_target" link + return 0 +} + +# np_trace_instance_of [handle] [--nrn N] +# +# This run instantiates a reusable JOB definition — the read model resolves +# the run's plan from the definition. Emit the definition itself with +# np_trace_job. +np_trace_instance_of() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _instance_of_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_instance_of_handle" || { np__drop 'instance_of' 'no node in scope'; return 0; } + _instance_of_namespace=${1:-} + _instance_of_name=${2:-} + _instance_of_version=${3:-} + if [ "$#" -ge 3 ]; then + shift 3 + fi + _instance_of_nrn='' + while [ "$#" -gt 0 ]; do + case "$1" in + --nrn) _instance_of_nrn=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_instance_of_namespace" ] || [ -z "$_instance_of_name" ] || [ -z "$_instance_of_version" ]; then + np__drop 'instance_of' 'namespace, name and version are required' + return 0 + fi + _instance_of_target=$(np__json_obj type job namespace "$_instance_of_namespace" \ + name "$_instance_of_name" version "$_instance_of_version" nrn "$_instance_of_nrn") + np__emit_ref_edge "$_instance_of_handle" "$NP_TYPE_EDGE_INSTANCE_OF" "$_instance_of_target" instance_of + return 0 +} + +# --------------------------------------------------------------------------- +# Definition nodes — identities, not executions +# --------------------------------------------------------------------------- + +# np_trace_dataset [--nrn N] +# +# Emit a dataset node — an identity a lineage edge can point at. The id is +# the CANONICAL address (see np_trace_produces); edges to an unemitted +# dataset still resolve, so this is only needed to carry the node itself. +np_trace_dataset() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _dataset_id=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + _dataset_nrn='' + while [ "$#" -gt 0 ]; do + case "$1" in + --nrn) _dataset_nrn=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_dataset_id" ]; then + np__drop 'dataset' 'an id is required' + return 0 + fi + np__spool "$NP_TYPE_NODE_DATASET" "$_dataset_nrn" "$(np__json_obj id "$_dataset_id")" >/dev/null + return 0 +} + +# np_trace_job [--nrn N] [--plan JSON] +# +# Emit a job definition node — the reusable spec runs link instance_of, with +# its expected step plan (previewable before any run exists). +np_trace_job() { + [ "${NP_TRACE_ENABLED:-1}" = '1' ] || return 0 + _job_namespace=${1:-} + _job_name=${2:-} + _job_version=${3:-} + if [ "$#" -ge 3 ]; then + shift 3 + fi + _job_nrn='' + _job_plan='' + while [ "$#" -gt 0 ]; do + case "$1" in + --nrn) _job_nrn=${2:-}; shift 2 ;; + --plan) _job_plan=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_job_namespace" ] || [ -z "$_job_name" ] || [ -z "$_job_version" ]; then + np__drop 'job' 'namespace, name and version are required' + return 0 + fi + case "$_job_plan" in + '' | \[*) ;; + *) np__drop 'job' 'the plan must be a JSON array of steps'; return 0 ;; + esac + if [ -n "$_job_plan" ]; then + _job_data=$(np__json_obj_raw \ + namespace "$(np__json_str "$_job_namespace")" \ + name "$(np__json_str "$_job_name")" \ + version "$(np__json_str "$_job_version")" \ + facets "{$(np__json_str "$NP_FACET_PLAN"):$_job_plan}") + else + _job_data=$(np__json_obj namespace "$_job_namespace" name "$_job_name" version "$_job_version") + fi + np__spool "$NP_TYPE_NODE_JOB" "$_job_nrn" "$_job_data" >/dev/null + return 0 +} + +# --------------------------------------------------------------------------- +# The remaining core-facet setters +# --------------------------------------------------------------------------- + +# np_trace_actor [handle] [--source S] +# +# WHO acted. The sibling SDKs also accept a bearer JWT and decode it; that +# sugar needs base64, which this SDK's runtime toolset excludes — pass the +# identity explicitly (the np CLI stamps the actor on workflow runs already). +np_trace_actor() { + _actor_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_actor_handle" || return 0 + _actor_kind=${1:-} + _actor_id=${2:-} + if [ "$#" -ge 2 ]; then + shift 2 + fi + _actor_source='' + while [ "$#" -gt 0 ]; do + case "$1" in + --source) _actor_source=${2:-}; shift 2 ;; + *) shift ;; + esac + done + case "$_actor_kind" in + user | service) ;; + *) np__drop 'actor' "kind must be user or service, got '${_actor_kind}'"; return 0 ;; + esac + if [ -z "$_actor_id" ]; then + np__drop 'actor' 'an id is required' + return 0 + fi + np__stage_facet "$_actor_handle" "$NP_FACET_ACTOR" \ + "$(np__json_obj kind "$_actor_kind" id "$_actor_id" source "$_actor_source")" + np__flush_foreign "$_actor_handle" + return 0 +} + +# np_trace_decision [handle] [--available a,b,c] [--expression E] +# +# The branch(es) this node chose, with the option set and the human-readable +# expression when known. +np_trace_decision() { + _decision_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_decision_handle" || return 0 + _decision_chosen=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + _decision_available='' + _decision_expression='' + while [ "$#" -gt 0 ]; do + case "$1" in + --available) _decision_available=${2:-}; shift 2 ;; + --expression) _decision_expression=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_decision_chosen" ]; then + np__drop 'decision' 'at least one chosen branch is required' + return 0 + fi + np__stage_facet "$_decision_handle" "$NP_FACET_DECISION" \ + "$(np__json_obj_raw \ + chosen "$(np__json_str_array_csv "$_decision_chosen")" \ + available "$(if [ -n "$_decision_available" ]; then np__json_str_array_csv "$_decision_available"; fi)" \ + expression "$(if [ -n "$_decision_expression" ]; then np__json_str "$_decision_expression"; fi)")" + np__flush_foreign "$_decision_handle" + return 0 +} + +# np_trace_retry [handle] [--next-attempt N] [--delay-ms MS] +np_trace_retry() { + _retry_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_retry_handle" || return 0 + _retry_attempt=${1:-} + if [ "$#" -gt 0 ]; then + shift + fi + _retry_next='' + _retry_delay='' + while [ "$#" -gt 0 ]; do + case "$1" in + --next-attempt) _retry_next=${2:-}; shift 2 ;; + --delay-ms) _retry_delay=${2:-}; shift 2 ;; + *) shift ;; + esac + done + case "$_retry_attempt$_retry_next$_retry_delay" in + '' | *[!0-9]*) np__drop 'retry' 'attempt, next-attempt and delay-ms must be non-negative integers'; return 0 ;; + esac + np__stage_facet "$_retry_handle" "$NP_FACET_RETRY" \ + "$(np__json_obj_raw attempt "$_retry_attempt" next_attempt "$_retry_next" delay_ms "$_retry_delay")" + np__flush_foreign "$_retry_handle" + return 0 +} + +# np_trace_signal [handle] [--timeout-ms MS] +np_trace_signal() { + _signal_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_signal_handle" || return 0 + _signal_name=${1:-} + _signal_direction=${2:-} + if [ "$#" -ge 2 ]; then + shift 2 + fi + _signal_timeout='' + while [ "$#" -gt 0 ]; do + case "$1" in + --timeout-ms) _signal_timeout=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_signal_name" ]; then + np__drop 'signal' 'a name is required' + return 0 + fi + case "$_signal_direction" in + wait | received) ;; + *) np__drop 'signal' "direction must be wait or received, got '${_signal_direction}'"; return 0 ;; + esac + case "$_signal_timeout" in + '' | *[!0-9]*) + if [ -n "$_signal_timeout" ]; then + np__drop 'signal' 'timeout-ms must be a non-negative integer' + return 0 + fi + ;; + esac + np__stage_facet "$_signal_handle" "$NP_FACET_SIGNAL" \ + "$(np__json_obj_raw \ + name "$(np__json_str "$_signal_name")" \ + direction "$(np__json_str "$_signal_direction")" \ + timeout_ms "$_signal_timeout")" + np__flush_foreign "$_signal_handle" + return 0 +} + +# np_trace_external_links [handle] [--label L] +# +# One off-platform link (a CI run, a dashboard). Accumulates: call once per +# link, the facet is the array of everything declared so far. +np_trace_external_links() { + _external_links_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_external_links_handle" || return 0 + _external_links_rel=${1:-} + _external_links_uri=${2:-} + if [ "$#" -ge 2 ]; then + shift 2 + fi + _external_links_label='' + while [ "$#" -gt 0 ]; do + case "$1" in + --label) _external_links_label=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_external_links_rel" ] || [ -z "$_external_links_uri" ]; then + np__drop 'external_links' 'rel and uri are required' + return 0 + fi + _external_links_link=$(np__json_obj rel "$_external_links_rel" uri "$_external_links_uri" label "$_external_links_label") + _external_links_links=$(np__node_get "$_external_links_handle" external_links) + if [ -n "$_external_links_links" ]; then + _external_links_links="$_external_links_links,$_external_links_link" + else + _external_links_links=$_external_links_link + fi + np__node_set "$_external_links_handle" external_links "$_external_links_links" + np__stage_facet "$_external_links_handle" "$NP_FACET_EXTERNAL_LINKS" "[$_external_links_links]" + np__flush_foreign "$_external_links_handle" + return 0 +} + +# np_trace_engine_status [handle] [--raw JSON] +# +# The underlying engine's own view of this node (a k8s rollout's status, a +# queue's verdict), verbatim. +np_trace_engine_status() { + _engine_status_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_engine_status_handle" || return 0 + _engine_status_engine=${1:-} + _engine_status_state=${2:-} + if [ "$#" -ge 2 ]; then + shift 2 + fi + _engine_status_raw='' + while [ "$#" -gt 0 ]; do + case "$1" in + --raw) _engine_status_raw=${2:-}; shift 2 ;; + *) shift ;; + esac + done + if [ -z "$_engine_status_engine" ] || [ -z "$_engine_status_state" ]; then + np__drop 'engine_status' 'engine and state are required' + return 0 + fi + case "$_engine_status_raw" in + '' | \{*) ;; + *) np__drop 'engine_status' 'raw must be a JSON object'; return 0 ;; + esac + np__stage_facet "$_engine_status_handle" "$NP_FACET_ENGINE_STATUS" \ + "$(np__json_obj_raw \ + engine "$(np__json_str "$_engine_status_engine")" \ + state "$(np__json_str "$_engine_status_state")" \ + raw "$_engine_status_raw")" + np__flush_foreign "$_engine_status_handle" + return 0 +} + +# np_trace_dropped [handle] +# +# A record of data intentionally dropped — pair with np_trace_skip. +np_trace_dropped() { + _dropped_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_dropped_handle" || return 0 + if [ -z "${1:-}" ]; then + np__drop 'dropped' 'a reason is required' + return 0 + fi + np__stage_facet "$_dropped_handle" "$NP_FACET_DROPPED" "$(np__json_obj reason "$1")" + np__flush_foreign "$_dropped_handle" + return 0 +} + +# np_trace_plan [handle] +# +# Declare the node's EXPECTED step plan ([{"key":...,"title":...}, ...]) so +# the read model reports expected-vs-observed progress. On a reusable +# definition, prefer np_trace_job --plan. +np_trace_plan() { + _plan_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_plan_handle" || return 0 + case "${1:-}" in + \[*) ;; + *) np__drop 'plan' 'the plan must be a JSON array of steps'; return 0 ;; + esac + np__stage_facet "$_plan_handle" "$NP_FACET_PLAN" "$1" + np__flush_foreign "$_plan_handle" + return 0 +} + +# np_trace_affordances [handle] +# +# What this node OFFERS a human to do — a declared fact the UI renders as a +# control (view live logs, switch traffic). One affordance object +# ('{"kind":"deploy-log",...}') or a bare array of them; the wire form is +# always the array. +np_trace_affordances() { + _affordances_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_affordances_handle" || return 0 + _affordances_body=${1:-} + case "$_affordances_body" in + \[*) ;; + \{*) _affordances_body="[$_affordances_body]" ;; + *) np__drop 'affordances' 'body must be a JSON object or array'; return 0 ;; + esac + np__stage_facet "$_affordances_handle" "$NP_FACET_AFFORDANCES" "$_affordances_body" + np__flush_foreign "$_affordances_handle" + return 0 +} + +# np_trace_progress [handle] [unit] +# +# How far a CONVERGING phase has advanced toward its declared target — +# instances 3 of 10, traffic 40 of 100. Non-negative integers; the optional +# unit names what is counted ("percent", "instances"). +np_trace_progress() { + _progress_handle=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_progress_handle" || return 0 + _progress_current=${1:-} + _progress_target=${2:-} + _progress_unit=${3:-} + if [ -z "$_progress_current" ] || [ -z "$_progress_target" ]; then + np__drop 'progress' 'current and target must be non-negative integers' + return 0 + fi + case "$_progress_current$_progress_target" in + *[!0-9]*) np__drop 'progress' 'current and target must be non-negative integers'; return 0 ;; + esac + np__stage_facet "$_progress_handle" "$NP_FACET_PROGRESS" \ + "$(np__json_obj_raw current "$_progress_current" target "$_progress_target" \ + unit "$(if [ -n "$_progress_unit" ]; then np__json_str "$_progress_unit"; fi)")" + np__flush_foreign "$_progress_handle" + return 0 +} + +# --------------------------------------------------------------------------- +# Lifecycle terminals +# --------------------------------------------------------------------------- + +# The shared terminal path. $1 = handle, $2 = status. +np__terminalize() { + np__is_handle "$1" || return 0 + if [ "$(np__node_get "$1" closed)" = '1' ]; then + return 0 + fi + # An adopted node belongs to the process that created it. Its owner decides + # its outcome; emitting a terminal here would assert a state we did not + # observe, and would race the owner's own terminal event. + if np__is_foreign "$1"; then + np__drop 'terminal' 'refusing to close an adopted node' + return 0 + fi + np_trace_start "$1" + np__stage_timing "$1" "$(np__iso8601)" + np__node_set "$1" closed 1 + np__emit_node "$1" "$2" + np__ambient_clear "$1" + # Restore the parent as ambient so a sibling opened next lands correctly. + _terminalize_parent=$(np__node_get "$1" parent) + if [ -n "$_terminalize_parent" ] && np__is_handle "$_terminalize_parent"; then + if [ "$(np__node_get "$_terminalize_parent" closed)" != '1' ]; then + np__ambient_set "$_terminalize_parent" + fi + fi + return 0 +} + +np_trace_complete() { + np__terminalize "$(np__resolve_handle "${1:-}")" "$NP_STATUS_COMPLETED" + return 0 +} + +# An idempotent completing close. +np_trace_end() { + np_trace_complete "$@" + return 0 +} + +np_trace_fail() { + _fail_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + # Refuse a foreign fail WHOLE, before the message stages: half-applying it + # (error facet emitted via the foreign flush, close refused) would smear an + # unowned outcome onto the node. Recording an observed fact on a foreign + # node is np_trace_error, deliberately. + if np__is_foreign "$_fail_h"; then + np__drop 'terminal' 'refusing to close an adopted node' + return 0 + fi + if [ -n "${1:-}" ]; then + np_trace_error "$_fail_h" --message "$1" + fi + # fail cascades to still-open child steps; complete deliberately does not — + # auto-completing an open child would assert a success the SDK cannot vouch + # for, and back-date its duration. + np__cascade_fail "$_fail_h" "${1:-}" + np__terminalize "$_fail_h" "$NP_STATUS_FAILED" + return 0 +} + +# True when $1 is a descendant of $2, by walking the parent chain upward. +# Deliberately NOT recursive: POSIX sh has no `local`, so a recursive walk +# clobbers its caller's loop variables — which silently skipped intermediate +# nodes in the cascade. +np__is_descendant_of() { + _is_descendant_of_cur=$(np__node_get "$1" parent) + _is_descendant_of_guard=0 + while [ -n "$_is_descendant_of_cur" ] && [ "$_is_descendant_of_guard" -lt 64 ]; do + if [ "$_is_descendant_of_cur" = "$2" ]; then + return 0 + fi + _is_descendant_of_cur=$(np__node_get "$_is_descendant_of_cur" parent) + _is_descendant_of_guard=$((_is_descendant_of_guard + 1)) + done + return 1 +} + +# Fail every still-open descendant. One flat pass over the registry, deepest +# first, so a node is closed before anything reads it as a parent. +np__cascade_fail() { + _cascade_fail_depth=64 + while [ "$_cascade_fail_depth" -ge 0 ]; do + for _cascade_fail_file in "$NP_TRACE_DIR/nodes"/*; do + [ -f "$_cascade_fail_file" ] || continue + _cascade_fail_h=${_cascade_fail_file##*/} + [ "$_cascade_fail_h" = "$1" ] && continue + [ "$(np__node_get "$_cascade_fail_h" closed)" = '1' ] && continue + np__is_descendant_of "$_cascade_fail_h" "$1" || continue + [ "$(np__depth_of "$_cascade_fail_h")" -eq "$_cascade_fail_depth" ] || continue + if [ -n "$2" ]; then + np_trace_error "$_cascade_fail_h" --message "$2" + fi + np__terminalize "$_cascade_fail_h" "$NP_STATUS_FAILED" + done + _cascade_fail_depth=$((_cascade_fail_depth - 1)) + done + return 0 +} + +# How many parent links sit above this node. +np__depth_of() { + _depth_of_cur=$(np__node_get "$1" parent) + _depth_of_n=0 + while [ -n "$_depth_of_cur" ] && [ "$_depth_of_n" -lt 64 ]; do + _depth_of_n=$((_depth_of_n + 1)) + _depth_of_cur=$(np__node_get "$_depth_of_cur" parent) + done + printf '%s' "$_depth_of_n" +} + +np_trace_skip() { + _skip_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__is_handle "$_skip_h" || return 0 + if [ -n "${1:-}" ]; then + np__stage_facet "$_skip_h" "$NP_FACET_DROPPED" "$(np__json_obj reason "$1")" + fi + np__terminalize "$_skip_h" "$NP_STATUS_SKIPPED" + return 0 +} + +np_trace_cancel() { + _cancel_h=$(np__resolve_handle "${1:-}") + if np__is_handle "${1:-}"; then + shift + fi + np__terminalize "$_cancel_h" "$NP_STATUS_CANCELLED" + return 0 +} + +np_trace_timeout() { + np__terminalize "$(np__resolve_handle "${1:-}")" "$NP_STATUS_TIMED_OUT" + return 0 +} + +# Non-terminal: the node stays open. +np_trace_waiting() { + _waiting_h=$(np__resolve_handle "${1:-}") + np__is_handle "$_waiting_h" || return 0 + np_trace_start "$_waiting_h" + np__emit_node "$_waiting_h" "$NP_STATUS_WAITING" + return 0 +} + +# ---- src/cli.sh ---- +# cli.sh — argv to function shim (Phase 2). From beeb855fa51527e75d437d7aa553de20c278d1dd Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 14:51:26 -0300 Subject: [PATCH 23/52] fix(k8s): a step's diagnostics attach to the row the user clicks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - the rollout wait IS the planned step (Instance health check): its heartbeats, instance counts and narrative attach to that step directly instead of a same-span sub-step — one human moment, one node, and the row's detail shows the diagnosis. - apply lineage records after the per-manifest sub-step closes, so the workloads/services/ingress produced land on the planned Apply manifests row (technical sub-steps never enter the human timeline; their evidence must not vanish with them). --- k8s/apply_templates | 5 ++++- k8s/deployment/wait_deployment_active | 13 ++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/k8s/apply_templates b/k8s/apply_templates index 4191070a..8589b729 100644 --- a/k8s/apply_templates +++ b/k8s/apply_templates @@ -58,10 +58,13 @@ while IFS= read -r TEMPLATE_FILE; do # onto the trace, not a bare "failed to apply". if KUBECTL_OUT=$(kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND 2>&1); then [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" + [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 0 + # Lineage AFTER the sub-step closes: what was applied belongs to the PLANNED + # step the user clicks ("Apply manifests"), not to an invisible technical + # sub-step — the row's detail must show the workloads it produced. if [[ "$ACTION" == "apply" ]] && command -v np_scope_k8s_applied >/dev/null 2>&1; then np_scope_k8s_applied "${K8S_NAMESPACE:-}" "$KUBECTL_OUT" fi - [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 0 else [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" >&2 log error " ❌ Failed to apply $FILENAME${KUBECTL_OUT:+: $KUBECTL_OUT}" diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index c6d9d2eb..32ba9439 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -216,11 +216,11 @@ log debug "📋 Namespace: $K8S_NAMESPACE" log debug "📋 Timeout: ${TIMEOUT}s (max $MAX_ITERATIONS iterations)" log debug "" -# The rollout wait is its own SUB-STEP in the trace; heartbeats carry the live -# replica counts. (Guarded: overrides may reuse this script without k8s/logging -# loaded.) -if command -v np_scope_step_begin >/dev/null 2>&1; then - np_scope_step_begin wait-deployment-active --title "Wait for deployment rollout ($K8S_DEPLOYMENT_NAME)" +# The rollout wait IS the planned step ("Instance health check") — heartbeats, +# counts and the narrative attach to that step directly, so clicking its row +# shows the diagnosis. A sub-step would split one human moment across two +# nodes. (Guarded: overrides may reuse this script without k8s/logging loaded.) +if command -v np_scope_wait_heartbeat >/dev/null 2>&1; then np_scope_wait_heartbeat "deployment-active" 0 "$TIMEOUT" "starting" fi @@ -297,7 +297,7 @@ while true; do if [ "$desired" = "$current" ] && [ "$desired" = "$updated" ] && [ "$desired" = "$ready" ] && [ "$desired" -gt 0 ]; then log debug "" log info "✅ All pods in deployment '$K8S_DEPLOYMENT_NAME' are available and ready!" - if command -v np_scope_step_end >/dev/null 2>&1; then + if command -v np_scope_progress >/dev/null 2>&1; then np_scope_progress "$ready" "$desired" instances # Every pod is ready: any earlier probe warning has recovered, and # the terminal narrative must say so (a crash-loop on the way here @@ -330,7 +330,6 @@ while true; do np_scope_affordance "{\"kind\":\"deploy-log\",\"application_id\":\"$_wda_app_id\",\"scope_id\":\"$_wda_scope_id\",\"type\":\"application\"${_wda_start_ms:+,\"start_time\":$_wda_start_ms}}" fi - np_scope_step_end 0 fi break fi From 050af7c8aabe8c95a256e93121fc5de53daa7327 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 15:22:44 -0300 Subject: [PATCH 24/52] fix(tracing): vendor SDK with adopted-step coordinate triple MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enrichments on the CLI's planned step (io, explain, wait labels) were rejected by ingest — the adopted derived-path node emitted no key/attempt/iteration. Vendors catalog-tracing-sh#10. --- nptrace.sh | 12 +++++++++++- vendor/catalog-tracing-sh | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/nptrace.sh b/nptrace.sh index b26096db..80e53d3c 100755 --- a/nptrace.sh +++ b/nptrace.sh @@ -845,7 +845,8 @@ np_trace_adopt() { # rather than a named id — the np CLI hands us the step it is running. Accept # either: parse it as a node path first, and only fall back to the named-id # rules when it has no delimiter. - if ! np__parse_node_id "$_adopt_run" >/dev/null 2>&1; then + if ! _adopt_coords=$(np__parse_node_id "$_adopt_run" 2>/dev/null); then + _adopt_coords='' if ! _adopt_why=$(np__named_id_violation "$_adopt_run"); then np__drop 'adopt' "run_id $_adopt_why" return 1 @@ -856,6 +857,15 @@ np_trace_adopt() { np__node_set "$_adopt_h" kind run np__node_set "$_adopt_h" trace_id "$_adopt_trace" np__node_set "$_adopt_h" run_id "$_adopt_run" + # A keyed (derived-path) node event must carry its coordinate triple — the + # API rejects a derived run_id whose key/attempt/iteration are absent. The + # foreign re-emit (np__flush_foreign) therefore needs the coordinates on the + # handle, even though the node itself stays the upstream owner's to close. + if [ -n "$_adopt_coords" ]; then + np__node_set "$_adopt_h" key "$(printf '%s' "$_adopt_coords" | cut -d' ' -f2)" + np__node_set "$_adopt_h" attempt "$(printf '%s' "$_adopt_coords" | cut -d' ' -f3)" + np__node_set "$_adopt_h" iteration "$(printf '%s' "$_adopt_coords" | cut -d' ' -f4)" + fi np__node_set "$_adopt_h" nrn "${NP_TRACE_NRN:-}" np__node_set "$_adopt_h" foreign 1 # started=1 suppresses the lazy `started` emit; closed=0 keeps it usable as a diff --git a/vendor/catalog-tracing-sh b/vendor/catalog-tracing-sh index 882ba1f4..4ffabfe9 160000 --- a/vendor/catalog-tracing-sh +++ b/vendor/catalog-tracing-sh @@ -1 +1 @@ -Subproject commit 882ba1f42cd17387554be648a940d56be795ff85 +Subproject commit 4ffabfe94b3134b0b54db6a85d0f727adef76e6c From f591ef662de24099483d05b29945af128011e6a0 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 15:49:25 -0300 Subject: [PATCH 25/52] fix(tracing): adopted-step observations accumulate in one bag Each np_scope_* call minted a fresh adopt handle, so every foreign re-emit carried a single fact and the API's last-writer-wins facet fold kept only the final one (apply showed one pointer instead of three; a lost tail event took its fact with it). The adopted handle is now cached per NP_TRACE value, so the bag is cumulative and every re-emit carries the full picture. --- k8s/logging | 19 +++++++++++++++---- scheduled_task/logging | 19 +++++++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/k8s/logging b/k8s/logging index 21812f2b..6d66d6b9 100644 --- a/k8s/logging +++ b/k8s/logging @@ -69,10 +69,16 @@ log() { # ============================================================================= # Resolve the node observations attach to: the open sub-step when one belongs -# to the CURRENT platform step, else a fresh adoption of NP_TRACE. Adoption is -# per-call on purpose: NP_TRACE changes as the CLI moves between steps, so an -# observation must attach to the step current at the moment it happened — and -# a sub-step opened under a PREVIOUS platform step must never swallow it. +# to the CURRENT platform step, else the adoption of NP_TRACE. The adopted +# handle is CACHED per NP_TRACE value: successive observations on the same +# platform step must accumulate in ONE bag, because every foreign re-emit +# carries the handle's full current bag and the API folds each facet +# last-writer-wins — fresh handles per call would leave only the last fact +# standing (and a lost tail event would take its fact with it; a cumulative +# bag re-carries earlier facts on every later emit). NP_TRACE changes as the +# CLI moves between steps, so the cache keys on it — an observation always +# attaches to the step current at the moment it happened, and a sub-step or +# adoption from a PREVIOUS platform step must never swallow it. _np_scopes_node() { command -v np_trace_adopt >/dev/null 2>&1 || return 1 [ -n "${NP_TRACE:-}" ] || return 1 @@ -84,9 +90,14 @@ _np_scopes_node() { # The platform moved on with a sub-step still open — stale; forget it. _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" fi + if [ -n "${_NP_SCOPES_ADOPTED:-}" ] && [ "${_NP_SCOPES_ADOPTED_UNDER:-}" = "$NP_TRACE" ]; then + printf '%s' "$_NP_SCOPES_ADOPTED" + return 0 + fi local _nd_node _nd_node=$(np_trace_adopt 2>/dev/null) || return 1 [ -n "$_nd_node" ] || return 1 + _NP_SCOPES_ADOPTED="$_nd_node" _NP_SCOPES_ADOPTED_UNDER="$NP_TRACE" printf '%s' "$_nd_node" return 0 } diff --git a/scheduled_task/logging b/scheduled_task/logging index 21812f2b..6d66d6b9 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -69,10 +69,16 @@ log() { # ============================================================================= # Resolve the node observations attach to: the open sub-step when one belongs -# to the CURRENT platform step, else a fresh adoption of NP_TRACE. Adoption is -# per-call on purpose: NP_TRACE changes as the CLI moves between steps, so an -# observation must attach to the step current at the moment it happened — and -# a sub-step opened under a PREVIOUS platform step must never swallow it. +# to the CURRENT platform step, else the adoption of NP_TRACE. The adopted +# handle is CACHED per NP_TRACE value: successive observations on the same +# platform step must accumulate in ONE bag, because every foreign re-emit +# carries the handle's full current bag and the API folds each facet +# last-writer-wins — fresh handles per call would leave only the last fact +# standing (and a lost tail event would take its fact with it; a cumulative +# bag re-carries earlier facts on every later emit). NP_TRACE changes as the +# CLI moves between steps, so the cache keys on it — an observation always +# attaches to the step current at the moment it happened, and a sub-step or +# adoption from a PREVIOUS platform step must never swallow it. _np_scopes_node() { command -v np_trace_adopt >/dev/null 2>&1 || return 1 [ -n "${NP_TRACE:-}" ] || return 1 @@ -84,9 +90,14 @@ _np_scopes_node() { # The platform moved on with a sub-step still open — stale; forget it. _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" fi + if [ -n "${_NP_SCOPES_ADOPTED:-}" ] && [ "${_NP_SCOPES_ADOPTED_UNDER:-}" = "$NP_TRACE" ]; then + printf '%s' "$_NP_SCOPES_ADOPTED" + return 0 + fi local _nd_node _nd_node=$(np_trace_adopt 2>/dev/null) || return 1 [ -n "$_nd_node" ] || return 1 + _NP_SCOPES_ADOPTED="$_nd_node" _NP_SCOPES_ADOPTED_UNDER="$NP_TRACE" printf '%s' "$_nd_node" return 0 } From dc25cc3e13fbef42aea7f49ec983d1af91ccb556 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 17:29:43 -0300 Subject: [PATCH 26/52] feat(tracing): phase-titled checks, human narratives, curated diagnose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The instance wait speaks each phase's words (WAIT_TITLE): provision keeps 'Instance health check'; the switch phase verifies scaled instances; finalize verifies final capacity. - Narratives translate provider codes to plain words (the native scopes' reason table), and a calm boot narrates itself instead of going silent. - Diagnose bookkeeping (notify before/after each check, context build) goes untraced — the checks are the story. - Evidence gaps closed: replicas io on rolling scales (an honest skip on blue-green), removed-* pointers on deletes, verification summaries on the networking verifiers, quota numbers on the capacity pre-flight. - Scope workflows curated to the same grammar: titles on the observable acts, trace: false on template generation and context plumbing. --- k8s/apply_templates | 5 ++ k8s/deployment/scale_deployments | 13 ++++ .../validate_alb_target_group_capacity | 6 ++ .../verify_http_route_reconciliation | 1 + k8s/deployment/verify_ingress_reconciliation | 6 ++ k8s/deployment/wait_deployment_active | 66 +++++++++++++++++-- k8s/deployment/workflows/diagnose.yaml | 5 ++ k8s/deployment/workflows/finalize.yaml | 5 +- k8s/deployment/workflows/switch_traffic.yaml | 5 +- k8s/logging | 34 ++++++++++ k8s/scope/workflows/create.yaml | 14 ++++ k8s/scope/workflows/delete.yaml | 12 +++- k8s/scope/workflows/diagnose.yaml | 4 ++ scheduled_task/logging | 34 ++++++++++ 14 files changed, 200 insertions(+), 10 deletions(-) diff --git a/k8s/apply_templates b/k8s/apply_templates index 8589b729..0cfe0c81 100644 --- a/k8s/apply_templates +++ b/k8s/apply_templates @@ -65,6 +65,11 @@ while IFS= read -r TEMPLATE_FILE; do if [[ "$ACTION" == "apply" ]] && command -v np_scope_k8s_applied >/dev/null 2>&1; then np_scope_k8s_applied "${K8S_NAMESPACE:-}" "$KUBECTL_OUT" fi + # A removal is evidence too: the finalize/rollback row that deleted the + # previous (or aborted) deployment names what it removed. + if [[ "$ACTION" == "delete" ]] && command -v np_scope_k8s_deleted >/dev/null 2>&1; then + np_scope_k8s_deleted "${K8S_NAMESPACE:-}" "$KUBECTL_OUT" + fi else [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" >&2 log error " ❌ Failed to apply $FILENAME${KUBECTL_OUT:+: $KUBECTL_OUT}" diff --git a/k8s/deployment/scale_deployments b/k8s/deployment/scale_deployments index 9e703eed..a20a9ed7 100755 --- a/k8s/deployment/scale_deployments +++ b/k8s/deployment/scale_deployments @@ -7,6 +7,14 @@ GREEN_DEPLOYMENT_ID=$DEPLOYMENT_ID BLUE_REPLICAS=$(echo "$CONTEXT" | jq -r .blue_replicas) BLUE_DEPLOYMENT_ID=$(echo "$CONTEXT" | jq .scope.current_active_deployment -r) +if [ "$DEPLOY_STRATEGY" != "rolling" ]; then + # Blue-green carries its replica counts in the manifests themselves — this step + # has nothing to do, and the trace says so instead of showing a hollow success. + if command -v np_step_skip >/dev/null 2>&1; then + np_step_skip "scaling rides the manifests for the $DEPLOY_STRATEGY strategy" + fi +fi + if [ "$DEPLOY_STRATEGY" = "rolling" ]; then GREEN_DEPLOYMENT_NAME="d-$SCOPE_ID-$GREEN_DEPLOYMENT_ID" BLUE_DEPLOYMENT_NAME="d-$SCOPE_ID-$BLUE_DEPLOYMENT_ID" @@ -41,6 +49,11 @@ if [ "$DEPLOY_STRATEGY" = "rolling" ]; then unset TIMEOUT unset SKIP_DEPLOYMENT_STATUS_CHECK + # What this step DID, on the row a user clicks: the resulting replica split. + if command -v np_scope_output >/dev/null 2>&1; then + np_scope_output replicas "{\"green\": $GREEN_REPLICAS, \"previous\": $BLUE_REPLICAS}" + fi + log debug "" log info "✨ Deployments scaled successfully" fi diff --git a/k8s/deployment/validate_alb_target_group_capacity b/k8s/deployment/validate_alb_target_group_capacity index 71d01d9e..94a0bad8 100755 --- a/k8s/deployment/validate_alb_target_group_capacity +++ b/k8s/deployment/validate_alb_target_group_capacity @@ -184,3 +184,9 @@ if [[ "$LISTENER_COUNT" -ge "$ALB_MAX_LISTENERS" ]]; then fi log info "✅ ALB listener capacity validated: $LISTENER_COUNT/$ALB_MAX_LISTENERS" + +# The row's summary: the quota headroom this pre-flight verified, in numbers. +if command -v np_scope_explain >/dev/null 2>&1; then + np_scope_explain --title "Validate load balancer capacity" --what "Load balancer has room: $TARGET_GROUP_COUNT/$ALB_MAX_TARGET_GROUPS target groups, $LISTENER_COUNT/$ALB_MAX_LISTENERS listeners in use" + np_scope_output capacity "{\"target_groups\": {\"used\": $TARGET_GROUP_COUNT, \"max\": $ALB_MAX_TARGET_GROUPS}, \"listeners\": {\"used\": $LISTENER_COUNT, \"max\": $ALB_MAX_LISTENERS}}" +fi diff --git a/k8s/deployment/verify_http_route_reconciliation b/k8s/deployment/verify_http_route_reconciliation index 6962efc6..c4b914fb 100644 --- a/k8s/deployment/verify_http_route_reconciliation +++ b/k8s/deployment/verify_http_route_reconciliation @@ -52,6 +52,7 @@ while [ $elapsed -lt $MAX_WAIT_SECONDS ]; do log info "✅ HTTPRoute successfully reconciled (Accepted: True, ResolvedRefs: True)" if command -v np_scope_step_end >/dev/null 2>&1; then np_scope_step_end 0 + np_scope_explain --title "Verify networking" --what "HTTPRoute $HTTPROUTE_NAME reconciled — DNS routing verified" fi return 0 fi diff --git a/k8s/deployment/verify_ingress_reconciliation b/k8s/deployment/verify_ingress_reconciliation index a593dd31..b013759d 100644 --- a/k8s/deployment/verify_ingress_reconciliation +++ b/k8s/deployment/verify_ingress_reconciliation @@ -263,6 +263,9 @@ while [ $elapsed -lt $MAX_WAIT_SECONDS ]; do log info "✅ ALB configuration validated successfully" if command -v np_scope_step_end >/dev/null 2>&1; then np_scope_step_end 0 + # The planned row's plain-language summary — what was verified, on the + # step the user clicks (after the sub-step closes). + np_scope_explain --title "Verify networking" --what "Load-balancer routing verified for $INGRESS_NAME — rules and weights match the deployment" fi return 0 fi @@ -297,6 +300,9 @@ while [ $elapsed -lt $MAX_WAIT_SECONDS ]; do if command -v np_scope_step_end >/dev/null 2>&1; then np_scope_step_end 0 fi + if command -v np_scope_explain >/dev/null 2>&1; then + np_scope_explain --title "Verify networking" --what "Ingress $INGRESS_NAME reconciled — the controller reported the routing applied" + fi return 0 fi fi diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index 32ba9439..f4b3abcf 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -60,7 +60,47 @@ LAST_REPORTED_COUNTS="" # a crash-loop that heals mid-wait must not read as an uneventful wait. UNHEALTHY_POD_COUNT=0 UNHEALTHY_POD_REASONS="" -WAIT_TITLE="Instance health check" +# The row's phase-specific words (the workflow step sets both the trace title and +# WAIT_TITLE): the provision wait is "Instance health check"; the switch phase +# verifies scaled instances; finalize verifies final capacity. +WAIT_TITLE="${WAIT_TITLE:-Instance health check}" + +# Provider reason CODES -> plain words for the narrative surfaces (same table the +# native scopes use). The raw code is machine detail — it stays on the io facets +# for the dialog; the narrative speaks human. Unknown codes fall through verbatim. +humanize_k8s_reason() { + case "$1" in + OOMKilled) echo "out of memory" ;; + CrashLoopBackOff) echo "crashing repeatedly" ;; + ImagePullBackOff|ErrImagePull|InvalidImageName) echo "can't pull the container image" ;; + CreateContainerConfigError|CreateContainerError|RunContainerError|ContainerCannotRun) echo "container failed to start" ;; + Error) echo "exited with an error" ;; + DeadlineExceeded) echo "timed out" ;; + Evicted) echo "evicted from its node" ;; + Unschedulable|FailedScheduling) echo "can't be scheduled onto a node" ;; + BackOff) echo "restarting after failures" ;; + FailedMount|FailedAttachVolume) echo "can't attach its storage" ;; + FailedCreate) echo "couldn't be created" ;; + Unhealthy) echo "failing health checks" ;; + *) echo "$1" ;; + esac +} + +# A comma-separated reason list, translated and deduped AFTER translation — two +# codes that mean the same thing read once. +humanize_k8s_reasons() { + local _hr_out="" _hr_word + IFS=', ' read -ra _hr_parts <<< "$1" + for _hr_part in "${_hr_parts[@]}"; do + [ -n "$_hr_part" ] || continue + _hr_word=$(humanize_k8s_reason "$_hr_part") + case ", $_hr_out," in + *", $_hr_word,"*) ;; + *) _hr_out="${_hr_out:+$_hr_out, }$_hr_word" ;; + esac + done + echo "$_hr_out" +} # Report the wait's live narrative onto the trace — the counted io, the # instances-health meter a host renders as pips, and the plain-language @@ -87,7 +127,8 @@ report_wait_narrative() { [ -n "$restarted" ] || restarted="[]" restart_total=$(echo "$restarted" | jq 'map(.restarts) | add // 0' 2>/dev/null) || restart_total=0 restart_reasons=$(echo "$restarted" | jq -r '[.[].reason // empty] | unique | join(", ")' 2>/dev/null) || restart_reasons="" - restart_clause="${restart_reasons:+ ($restart_reasons)}" + restart_words=$(humanize_k8s_reasons "${restart_reasons:-}") + restart_clause="${restart_words:+ ($restart_words)}" problems=$(echo "$pods_json" \ | jq -c '[.items[] | .metadata.name as $pod | .status.containerStatuses[]? | select(.state.waiting.reason != null @@ -143,9 +184,11 @@ report_wait_narrative() { + (if $crash != "" then {last_crash_log: $crash} else {} end)') np_scope_output instances "$instances" + # The meter is user-facing: its reason captions speak human words; the io + # above keeps the raw codes for the dialog. meter=$(jq -nc --argjson h "$ready_now" --argjson l "$launched_now" --argjson d "$desired_now" \ - --argjson u "$problem_count" --arg reasons "$problem_reasons" --argjson t "$restart_total" \ - --arg rreasons "$restart_reasons" \ + --argjson u "$problem_count" --arg reasons "$(humanize_k8s_reasons "$problem_reasons")" --argjson t "$restart_total" \ + --arg rreasons "$(humanize_k8s_reasons "$restart_reasons")" \ '{kind: "instances-health", healthy: $h, launched: $l, desired: $d} + (if $u > 0 then {unhealthy: $u, reasons: ($reasons | split(", ") | map(select(. != "")))} else {} end) + (if $t > 0 then {restarts: $t} else {} end) @@ -156,8 +199,10 @@ report_wait_narrative() { [ "$restart_total" -eq 1 ] && restarts_label="restart" [ -n "$WAIT_REAL_DETAIL" ] && detail_clause=": $WAIT_REAL_DETAIL" if [ "$problem_count" -gt 0 ]; then + local problem_words + problem_words=$(humanize_k8s_reasons "${problem_reasons:-}") np_scope_explain --title "$WAIT_TITLE" --severity warn \ - --what "Waiting for $ready_now/$desired_now instances to be healthy — $problem_count with ${problem_reasons:-failing health checks}$detail_clause" \ + --what "Waiting for $ready_now/$desired_now instances to be healthy — $problem_count with ${problem_words:-failing health checks}$detail_clause" \ --impact "The deployment fails if the instances don't become healthy before the health-check timeout." elif [ "$restart_total" -gt 0 ]; then if [ "$all_healthy" = "true" ]; then @@ -172,7 +217,13 @@ report_wait_narrative() { elif [ "$all_healthy" = "true" ]; then np_scope_explain --title "$WAIT_TITLE" --what "All $desired_now instances healthy" else - np_scope_explain --title "$WAIT_TITLE" --what "Waiting for $ready_now/$desired_now instances to be healthy" + # Calm but not silent: instances exist and are booting — a heavy boot reads + # as a heavy boot, never as a black box (and never as a problem). + local graced_clause="" + if [ "$launched_now" -gt "$ready_now" ] 2>/dev/null; then + graced_clause=" (instances starting — normal while the app boots)" + fi + np_scope_explain --title "$WAIT_TITLE" --what "Waiting for $ready_now/$desired_now instances to be healthy$graced_clause" fi return 0 } @@ -245,8 +296,9 @@ while true; do if [ -n "$timeout_cause" ]; then timeout_message="$timeout_cause ($timeout_message)" fi + timeout_reason_words=$(humanize_k8s_reasons "${timeout_reasons:-}") np_scope_explain --title "$WAIT_TITLE" --severity error \ - --what "Gave up with ${ready:-0}/${desired:-0} instances healthy${timeout_reasons:+ — $timeout_reasons}${timeout_cause:+: $timeout_cause}" + --what "Gave up with ${ready:-0}/${desired:-0} instances healthy${timeout_reason_words:+ — $timeout_reason_words}${timeout_cause:+: $timeout_cause}" np_scope_error "$timeout_message" \ "$(jq -nc --argjson h "${ready:-0}" --argjson l "${launched:-0}" --argjson d "${desired:-0}" --arg reasons "$timeout_reasons" \ '{instances: {healthy: $h, launched: $l, desired: $d}} + (if $reasons != "" then {reasons: ($reasons | split(", "))} else {} end)')" diff --git a/k8s/deployment/workflows/diagnose.yaml b/k8s/deployment/workflows/diagnose.yaml index faf96a44..0765ad4b 100644 --- a/k8s/deployment/workflows/diagnose.yaml +++ b/k8s/deployment/workflows/diagnose.yaml @@ -17,6 +17,7 @@ steps: - name: build context type: script file: "$SERVICE_PATH/diagnose/build_context" + trace: false output: - name: CONTEXT type: environment @@ -28,10 +29,14 @@ steps: name: notify_check_running type: script file: "$SERVICE_PATH/diagnose/notify_check_running" + # Bookkeeping around each check — the CHECKS are the diagnosis's story; + # two notify steps per check would triple every disclosure. + trace: false after_each: name: notify_check_results type: script file: "$SERVICE_PATH/diagnose/notify_diagnose_results" + trace: false folders: - "$SERVICE_PATH/diagnose/service" - "$SERVICE_PATH/diagnose/scope" diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index 7a65c0fb..75440a7b 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -57,11 +57,14 @@ steps: file: "$SERVICE_PATH/deployment/wait_deployment_active" trace: key: finalize-instances-check - title: Instance health check + # The same wait, in THIS phase's words: full capacity must be healthy + # before the previous version is removed. + title: Verify final capacity group: finalize configuration: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS SKIP_DEPLOYMENT_STATUS_CHECK: true + WAIT_TITLE: Verify final capacity - name: route traffic trace: title: Configure ingress diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index 17ca9aa0..3abc6fbf 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -60,10 +60,13 @@ steps: file: "$SERVICE_PATH/deployment/wait_deployment_active" trace: key: switch-instances-check - title: Instance health check + # The same wait, in THIS phase's words: green must hold its scaled + # replicas healthy before any weight shifts. + title: Verify scaled instances group: switching-traffic configuration: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS + WAIT_TITLE: Verify scaled instances - name: route traffic type: script file: "$SERVICE_PATH/deployment/networking/gateway/route_traffic" diff --git a/k8s/logging b/k8s/logging index 6d66d6b9..4e32f1c2 100644 --- a/k8s/logging +++ b/k8s/logging @@ -383,6 +383,40 @@ np_scope_k8s_applied() { return 0 } + +# np_scope_k8s_deleted +# +# The removal counterpart of np_scope_k8s_applied: record what a delete removed +# ("deployment.apps \"name\" deleted" lines) as pointers on the planned step, so a +# finalize's "Remove previous deployment" (or a rollback's "Remove new deployment") +# names what it removed. +np_scope_k8s_deleted() { + command -v np_trace_output >/dev/null 2>&1 || return 0 + local _kd_ns="$1" _kd_node _kd_line _kd_kind _kd_name + [ -n "$_kd_ns" ] || return 0 + _kd_node=$(_np_scopes_node) || return 0 + while IFS= read -r _kd_line; do + [ -n "$_kd_line" ] || continue + case "$_kd_line" in + *" deleted") ;; + *) continue ;; + esac + _kd_kind="${_kd_line%%[ /]*}" + _kd_name="${_kd_line#*\"}" + _kd_name="${_kd_name%%\"*}" + [ -n "$_kd_name" ] || continue + case "$_kd_kind" in + deployment.apps|deployment) + np_trace_output "$_kd_node" removed-workload --uri "$_kd_name" ;; + service) + np_trace_output "$_kd_node" removed-service --uri "$_kd_name" ;; + ingress.networking.k8s.io|ingress) + np_trace_output "$_kd_node" removed-ingress --uri "$_kd_name" ;; + esac + done <<< "$2" + return 0 +} + # A silent failure (no `log error` on the way down) must still be clear: the # ERR trap remembers the last failing top-level command, and the EXIT trap # reports it against the step that was current when the shell died. diff --git a/k8s/scope/workflows/create.yaml b/k8s/scope/workflows/create.yaml index 422e6034..75b6e946 100644 --- a/k8s/scope/workflows/create.yaml +++ b/k8s/scope/workflows/create.yaml @@ -70,6 +70,7 @@ steps: type: script file: "$SERVICE_PATH/scope/validate_alb_capacity" trace: + title: Validate load balancer capacity flavors: [route53] - name: iam type: workflow @@ -77,9 +78,13 @@ steps: - name: create role type: script file: "$SERVICE_PATH/scope/iam/create_role" + trace: + title: Create IAM role - name: build service account type: script file: "$SERVICE_PATH/scope/iam/build_service_account" + # Template generation — the apply below is the observable act. + trace: false configuration: ACTION: create output: @@ -89,6 +94,9 @@ steps: - name: apply type: script file: "$SERVICE_PATH/apply_templates" + trace: + key: apply-service-account + title: Apply service account configuration: ACTION: apply DRY_RUN: false @@ -98,6 +106,8 @@ steps: - name: generate domain type: script file: "$SERVICE_PATH/scope/networking/dns/domain/generate_domain" + # Pure computation — the DNS create below is the observable act. + trace: false output: - name: SCOPE_DOMAIN type: environment @@ -126,6 +136,9 @@ steps: - name: apply dns templates type: script file: "$SERVICE_PATH/apply_templates" + trace: + key: apply-dns + title: Apply DNS records configuration: ACTION: apply DRY_RUN: false @@ -133,4 +146,5 @@ steps: type: script file: "$SERVICE_PATH/scope/wait_on_balancer" trace: + title: Wait for load balancer flavors: [external_dns] diff --git a/k8s/scope/workflows/delete.yaml b/k8s/scope/workflows/delete.yaml index e22b313a..16daa485 100644 --- a/k8s/scope/workflows/delete.yaml +++ b/k8s/scope/workflows/delete.yaml @@ -43,12 +43,15 @@ steps: - name: delete dns type: script file: "$SERVICE_PATH/scope/networking/dns/manage_dns" + trace: + title: Delete DNS records configuration: ACTION: DELETE pre: name: build dns context type: script file: "$SERVICE_PATH/scope/networking/dns/build_dns_context" + trace: false output: - name: HOSTED_PUBLIC_ZONE_ID type: environment @@ -60,6 +63,8 @@ steps: - name: build service account type: script file: "$SERVICE_PATH/scope/iam/build_service_account" + # Template generation — the delete below is the observable act. + trace: false configuration: ACTION: delete output: @@ -69,9 +74,14 @@ steps: - name: apply type: script file: "$SERVICE_PATH/apply_templates" + trace: + key: remove-service-account + title: Remove service account configuration: ACTION: delete DRY_RUN: false - name: delete role type: script - file: "$SERVICE_PATH/scope/iam/delete_role" \ No newline at end of file + file: "$SERVICE_PATH/scope/iam/delete_role" + trace: + title: Delete IAM role \ No newline at end of file diff --git a/k8s/scope/workflows/diagnose.yaml b/k8s/scope/workflows/diagnose.yaml index faf96a44..5ec03834 100644 --- a/k8s/scope/workflows/diagnose.yaml +++ b/k8s/scope/workflows/diagnose.yaml @@ -17,6 +17,7 @@ steps: - name: build context type: script file: "$SERVICE_PATH/diagnose/build_context" + trace: false output: - name: CONTEXT type: environment @@ -28,10 +29,13 @@ steps: name: notify_check_running type: script file: "$SERVICE_PATH/diagnose/notify_check_running" + # Bookkeeping around each check — the CHECKS are the diagnosis's story. + trace: false after_each: name: notify_check_results type: script file: "$SERVICE_PATH/diagnose/notify_diagnose_results" + trace: false folders: - "$SERVICE_PATH/diagnose/service" - "$SERVICE_PATH/diagnose/scope" diff --git a/scheduled_task/logging b/scheduled_task/logging index 6d66d6b9..4e32f1c2 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -383,6 +383,40 @@ np_scope_k8s_applied() { return 0 } + +# np_scope_k8s_deleted +# +# The removal counterpart of np_scope_k8s_applied: record what a delete removed +# ("deployment.apps \"name\" deleted" lines) as pointers on the planned step, so a +# finalize's "Remove previous deployment" (or a rollback's "Remove new deployment") +# names what it removed. +np_scope_k8s_deleted() { + command -v np_trace_output >/dev/null 2>&1 || return 0 + local _kd_ns="$1" _kd_node _kd_line _kd_kind _kd_name + [ -n "$_kd_ns" ] || return 0 + _kd_node=$(_np_scopes_node) || return 0 + while IFS= read -r _kd_line; do + [ -n "$_kd_line" ] || continue + case "$_kd_line" in + *" deleted") ;; + *) continue ;; + esac + _kd_kind="${_kd_line%%[ /]*}" + _kd_name="${_kd_line#*\"}" + _kd_name="${_kd_name%%\"*}" + [ -n "$_kd_name" ] || continue + case "$_kd_kind" in + deployment.apps|deployment) + np_trace_output "$_kd_node" removed-workload --uri "$_kd_name" ;; + service) + np_trace_output "$_kd_node" removed-service --uri "$_kd_name" ;; + ingress.networking.k8s.io|ingress) + np_trace_output "$_kd_node" removed-ingress --uri "$_kd_name" ;; + esac + done <<< "$2" + return 0 +} + # A silent failure (no `log error` on the way down) must still be clear: the # ERR trap remembers the last failing top-level command, and the EXIT trap # reports it against the step that was current when the shell died. From 98d6a7ec00af2df9d03c6d5cb29af2d790fda0b0 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 17:34:35 -0300 Subject: [PATCH 27/52] fix(tracing): the adopted-node cache survives subshells MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every np_scope_* helper resolves its node inside a command substitution, so the shell-variable cache died with each subshell and every emission still carried a one-fact bag (apply showed only its last pointer). The cache now lives in the SDK state dir as a file — the one memory all the subshells share. --- k8s/logging | 18 +++++++++++++----- scheduled_task/logging | 18 +++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/k8s/logging b/k8s/logging index 4e32f1c2..608e71e1 100644 --- a/k8s/logging +++ b/k8s/logging @@ -90,14 +90,22 @@ _np_scopes_node() { # The platform moved on with a sub-step still open — stale; forget it. _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" fi - if [ -n "${_NP_SCOPES_ADOPTED:-}" ] && [ "${_NP_SCOPES_ADOPTED_UNDER:-}" = "$NP_TRACE" ]; then - printf '%s' "$_NP_SCOPES_ADOPTED" - return 0 + # The cache lives in the SDK's state dir, not a shell variable: every np_scope_* + # helper resolves the node inside a command substitution, where a variable write + # dies with the subshell — a FILE is the only memory all those subshells share. + local _nd_node _nd_under _nd_cached + if [ -n "${NP_TRACE_DIR:-}" ] && [ -f "$NP_TRACE_DIR/scopes_adopted" ]; then + IFS=' ' read -r _nd_under _nd_cached < "$NP_TRACE_DIR/scopes_adopted" 2>/dev/null || true + if [ "$_nd_under" = "$NP_TRACE" ] && [ -n "$_nd_cached" ]; then + printf '%s' "$_nd_cached" + return 0 + fi fi - local _nd_node _nd_node=$(np_trace_adopt 2>/dev/null) || return 1 [ -n "$_nd_node" ] || return 1 - _NP_SCOPES_ADOPTED="$_nd_node" _NP_SCOPES_ADOPTED_UNDER="$NP_TRACE" + if [ -n "${NP_TRACE_DIR:-}" ]; then + printf '%s %s' "$NP_TRACE" "$_nd_node" > "$NP_TRACE_DIR/scopes_adopted" 2>/dev/null || true + fi printf '%s' "$_nd_node" return 0 } diff --git a/scheduled_task/logging b/scheduled_task/logging index 4e32f1c2..608e71e1 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -90,14 +90,22 @@ _np_scopes_node() { # The platform moved on with a sub-step still open — stale; forget it. _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" fi - if [ -n "${_NP_SCOPES_ADOPTED:-}" ] && [ "${_NP_SCOPES_ADOPTED_UNDER:-}" = "$NP_TRACE" ]; then - printf '%s' "$_NP_SCOPES_ADOPTED" - return 0 + # The cache lives in the SDK's state dir, not a shell variable: every np_scope_* + # helper resolves the node inside a command substitution, where a variable write + # dies with the subshell — a FILE is the only memory all those subshells share. + local _nd_node _nd_under _nd_cached + if [ -n "${NP_TRACE_DIR:-}" ] && [ -f "$NP_TRACE_DIR/scopes_adopted" ]; then + IFS=' ' read -r _nd_under _nd_cached < "$NP_TRACE_DIR/scopes_adopted" 2>/dev/null || true + if [ "$_nd_under" = "$NP_TRACE" ] && [ -n "$_nd_cached" ]; then + printf '%s' "$_nd_cached" + return 0 + fi fi - local _nd_node _nd_node=$(np_trace_adopt 2>/dev/null) || return 1 [ -n "$_nd_node" ] || return 1 - _NP_SCOPES_ADOPTED="$_nd_node" _NP_SCOPES_ADOPTED_UNDER="$NP_TRACE" + if [ -n "${NP_TRACE_DIR:-}" ]; then + printf '%s %s' "$NP_TRACE" "$_nd_node" > "$NP_TRACE_DIR/scopes_adopted" 2>/dev/null || true + fi printf '%s' "$_nd_node" return 0 } From cb79dd3fd16a935ac0398eddf4d5e7cd04ae2b98 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 17:57:19 -0300 Subject: [PATCH 28/52] fix(tracing): land the wait's final truth and the apply lineage now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The success-path emissions (all-healthy narrative, consumed image, produced log pointer, the deploy-log offer) rode only the exit flush, whose budget drains oldest-first — the tail is exactly what it loses. One bounded flush at the success block (and after the manifests' lineage) delivers the diagnosis while the step is still the story. --- k8s/apply_templates | 6 ++++++ k8s/deployment/wait_deployment_active | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/k8s/apply_templates b/k8s/apply_templates index 0cfe0c81..7b4b8500 100644 --- a/k8s/apply_templates +++ b/k8s/apply_templates @@ -114,4 +114,10 @@ if [[ "${TRACE_TRAFFIC_SWITCH:-false}" == "true" ]] && [[ -n "${CONTEXT:-}" ]] \ fi fi +# The manifests' lineage (produced / removed pointers) is this step's evidence — +# land it now instead of betting the exit flush's budget on it. +if command -v np_trace_flush >/dev/null 2>&1; then + NP_TRACE_FLUSH_TIMEOUT=5 np_trace_flush +fi + source "$SERVICE_PATH/backup/backup_templates" --action="$ACTION" --files "${APPLIED_FILES[@]}" diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index f4b3abcf..ddc8526d 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -382,6 +382,11 @@ while true; do np_scope_affordance "{\"kind\":\"deploy-log\",\"application_id\":\"$_wda_app_id\",\"scope_id\":\"$_wda_scope_id\",\"type\":\"application\"${_wda_start_ms:+,\"start_time\":$_wda_start_ms}}" fi + # The wait's FINAL truth (all healthy, the lineage, the log offer) must not + # ride only the exit flush — a spent budget there loses exactly the tail. + # One bounded flush here: the wait already took its time; 10s to land the + # diagnosis is the cheapest part of it. + NP_TRACE_FLUSH_TIMEOUT=10 np_trace_flush fi break fi From 2e5d16f873c8b068f0a559d2cc2dcf16b5679c2b Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 18:18:08 -0300 Subject: [PATCH 29/52] fix(tracing): derive the wait's phase title from its step identity Passing WAIT_TITLE through step configuration broke the switch and finalize workflows: the engine splices configuration values unquoted into the shell fragment, so the spaced title parsed 'scaled' as a command and every traffic increment failed. The wait now derives its phase words from the step run path NP_TRACE already carries. --- k8s/deployment/wait_deployment_active | 14 ++++++++++---- k8s/deployment/workflows/finalize.yaml | 1 - k8s/deployment/workflows/switch_traffic.yaml | 1 - 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index ddc8526d..f8047491 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -60,10 +60,16 @@ LAST_REPORTED_COUNTS="" # a crash-loop that heals mid-wait must not read as an uneventful wait. UNHEALTHY_POD_COUNT=0 UNHEALTHY_POD_REASONS="" -# The row's phase-specific words (the workflow step sets both the trace title and -# WAIT_TITLE): the provision wait is "Instance health check"; the switch phase -# verifies scaled instances; finalize verifies final capacity. -WAIT_TITLE="${WAIT_TITLE:-Instance health check}" +# The row's phase-specific words, derived from the step this wait is running AS +# (NP_TRACE ends with the step's own run path, `...~@attempt.iteration`) — +# they must match the trace titles the workflows declare. Never passed through +# step configuration: the engine splices configuration values into the shell +# fragment, where a spaced value breaks the command. +case "${NP_TRACE:-}" in + *~switch-instances-check@*) WAIT_TITLE="Verify scaled instances" ;; + *~finalize-instances-check@*) WAIT_TITLE="Verify final capacity" ;; + *) WAIT_TITLE="Instance health check" ;; +esac # Provider reason CODES -> plain words for the narrative surfaces (same table the # native scopes use). The raw code is machine detail — it stays on the io facets diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index 75440a7b..3bb5c2d0 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -64,7 +64,6 @@ steps: configuration: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS SKIP_DEPLOYMENT_STATUS_CHECK: true - WAIT_TITLE: Verify final capacity - name: route traffic trace: title: Configure ingress diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index 3abc6fbf..f4b1ff6a 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -66,7 +66,6 @@ steps: group: switching-traffic configuration: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS - WAIT_TITLE: Verify scaled instances - name: route traffic type: script file: "$SERVICE_PATH/deployment/networking/gateway/route_traffic" From b6858b8e09134f715cc562781e738d53de5d6895 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 18:48:08 -0300 Subject: [PATCH 30/52] fix(tracing): progress speaks the wire's unit vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'instances' is not a tracing.progress unit (the closed set is percent, count, bytes, milliseconds) — the API rejected the whole event, and the cumulative bag re-carried the poisoned facet on every later emission, dead-lettering the wait's final narrative, its lineage and the log offer. The wait now counts in 'count', and the vendored SDK drops any out-of-vocabulary unit client-side so no caller can poison a node again. --- k8s/deployment/wait_deployment_active | 4 ++-- k8s/utils/tests/trace_logging.bats | 2 +- nptrace.sh | 15 +++++++++++++-- vendor/catalog-tracing-sh | 2 +- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index f8047491..c197c064 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -356,7 +356,7 @@ while true; do log debug "" log info "✅ All pods in deployment '$K8S_DEPLOYMENT_NAME' are available and ready!" if command -v np_scope_progress >/dev/null 2>&1; then - np_scope_progress "$ready" "$desired" instances + np_scope_progress "$ready" "$desired" count # Every pod is ready: any earlier probe warning has recovered, and # the terminal narrative must say so (a crash-loop on the way here # stays visible as "healthy — after N restarts"). @@ -404,7 +404,7 @@ while true; do np_scope_wait_heartbeat "deployment-active" "$elapsed_s" "$TIMEOUT" "progressing" \ "wait.desired=$desired" "wait.launched=$launched" \ "wait.ready=$ready" "wait.available=$current" "wait.updated=$updated" - np_scope_progress "$ready" "$desired" instances + np_scope_progress "$ready" "$desired" count fi fi diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index d986c009..067dd6fa 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -290,7 +290,7 @@ secret/sec-1 created" @test "affordance and progress land as their core facets" { run_logged ' np_scope_affordance "{\"kind\":\"deploy-log\",\"application_id\":\"7\"}" - np_scope_progress 3 10 instances + np_scope_progress 3 10 count ' [ "$status" -eq 0 ] echo "$output" | grep -q '"tracing.affordances":\[{"kind":"deploy-log","application_id":"7"}\]' diff --git a/nptrace.sh b/nptrace.sh index 80e53d3c..b5000d9c 100755 --- a/nptrace.sh +++ b/nptrace.sh @@ -2093,8 +2093,12 @@ np_trace_affordances() { # np_trace_progress [handle] [unit] # # How far a CONVERGING phase has advanced toward its declared target — -# instances 3 of 10, traffic 40 of 100. Non-negative integers; the optional -# unit names what is counted ("percent", "instances"). +# instances 3 of 10, traffic 40 of 100. Non-negative integers. The unit is a +# number-FORMAT hint from the wire's CLOSED vocabulary (percent, count, bytes, +# milliseconds) — the API rejects the whole EVENT over an unknown unit, and an +# enriched node re-emits its full facet bag, so one bad unit would poison every +# later emission. A word outside the vocabulary is therefore dropped here (the +# noun belongs in the step's title, not the unit). np_trace_progress() { _progress_handle=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then @@ -2111,6 +2115,13 @@ np_trace_progress() { case "$_progress_current$_progress_target" in *[!0-9]*) np__drop 'progress' 'current and target must be non-negative integers'; return 0 ;; esac + case "$_progress_unit" in + '' | percent | count | bytes | milliseconds) ;; + *) + np__drop 'progress' "unit '$_progress_unit' is not in the wire vocabulary (percent, count, bytes, milliseconds); omitted" + _progress_unit='' + ;; + esac np__stage_facet "$_progress_handle" "$NP_FACET_PROGRESS" \ "$(np__json_obj_raw current "$_progress_current" target "$_progress_target" \ unit "$(if [ -n "$_progress_unit" ]; then np__json_str "$_progress_unit"; fi)")" diff --git a/vendor/catalog-tracing-sh b/vendor/catalog-tracing-sh index 4ffabfe9..2c1e2a93 160000 --- a/vendor/catalog-tracing-sh +++ b/vendor/catalog-tracing-sh @@ -1 +1 @@ -Subproject commit 4ffabfe94b3134b0b54db6a85d0f727adef76e6c +Subproject commit 2c1e2a93970d855bed3af1cc6342532139db9d44 From 994f1357c044e93012e3f78ae84ea60db8b07a2c Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 18:54:14 -0300 Subject: [PATCH 31/52] fix(tracing): finalize milestones carry their station group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Titled steps without a group fold as technical work — the finalize row set (Promote new deployment, Configure ingress, Apply final routing, Verify networking, Remove previous deployment) and blue-green's Prepare previous deployment now declare the station they belong to. --- k8s/deployment/workflows/blue_green.yaml | 1 + k8s/deployment/workflows/finalize.yaml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/k8s/deployment/workflows/blue_green.yaml b/k8s/deployment/workflows/blue_green.yaml index d66193ec..4b8e4df0 100644 --- a/k8s/deployment/workflows/blue_green.yaml +++ b/k8s/deployment/workflows/blue_green.yaml @@ -19,4 +19,5 @@ steps: file: "$SERVICE_PATH/deployment/scale_deployments" trace: title: Prepare previous deployment + group: setting-up after: create deployment \ No newline at end of file diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index 3bb5c2d0..2c2bd24a 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -49,6 +49,7 @@ steps: - name: build green deployment trace: title: Promote new deployment + group: finalize type: script file: "$SERVICE_PATH/deployment/scale_deployments" post: @@ -67,6 +68,7 @@ steps: - name: route traffic trace: title: Configure ingress + group: finalize type: script file: "$SERVICE_PATH/deployment/networking/gateway/route_traffic" configuration: @@ -74,6 +76,7 @@ steps: - name: apply traffic trace: title: Apply final routing + group: finalize type: script file: "$SERVICE_PATH/apply_templates" configuration: @@ -90,6 +93,7 @@ steps: - name: verify_networking_reconciliation trace: title: Verify networking + group: finalize type: script file: "$SERVICE_PATH/deployment/verify_networking_reconciliation" configuration: @@ -120,6 +124,7 @@ steps: - name: delete deployment trace: title: Remove previous deployment + group: finalize type: script file: "$SERVICE_PATH/apply_templates" configuration: From 895a4bae23df93a59ab8190c1f94748ac61c2d90 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 19:56:00 -0300 Subject: [PATCH 32/52] fix(tracing): honest milestones, declared job identity, per-increment quiet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - blue_green/initial declare the job's identity labels (entity, strategy, scope.provider) so the deployment page previews the full pending checklist before the first event arrives - the scale step loses its milestone title: a no-op for blue-green (counts ride the manifests) — native declares no scaling milestone - switch_traffic opts into declared-steps-only (trace.default: false): one increment is 2-3 honest nodes, not 12 (override plumbing and permanently-skipped scale steps no longer multiply per increment) - publish_alb_metrics untraced (machine bookkeeping) - skip reasons written as sentences; vendored SDK picks up upsert-by-identity (io by name, affordances by kind, facets by namespace): the instances meter survives the deploy-log declaration and re-emitted io no longer duplicates --- k8s/deployment/scale_deployments | 2 +- k8s/deployment/verify_ingress_reconciliation | 2 +- k8s/deployment/workflows/blue_green.yaml | 17 ++- k8s/deployment/workflows/initial.yaml | 9 +- k8s/deployment/workflows/switch_traffic.yaml | 18 +-- nptrace.sh | 136 +++++++++++++------ vendor/catalog-tracing-sh | 2 +- 7 files changed, 131 insertions(+), 55 deletions(-) diff --git a/k8s/deployment/scale_deployments b/k8s/deployment/scale_deployments index a20a9ed7..34a503f9 100755 --- a/k8s/deployment/scale_deployments +++ b/k8s/deployment/scale_deployments @@ -11,7 +11,7 @@ if [ "$DEPLOY_STRATEGY" != "rolling" ]; then # Blue-green carries its replica counts in the manifests themselves — this step # has nothing to do, and the trace says so instead of showing a hollow success. if command -v np_step_skip >/dev/null 2>&1; then - np_step_skip "scaling rides the manifests for the $DEPLOY_STRATEGY strategy" + np_step_skip "instance counts are set by the manifests for the $DEPLOY_STRATEGY strategy — nothing to scale" fi fi diff --git a/k8s/deployment/verify_ingress_reconciliation b/k8s/deployment/verify_ingress_reconciliation index b013759d..942a9994 100644 --- a/k8s/deployment/verify_ingress_reconciliation +++ b/k8s/deployment/verify_ingress_reconciliation @@ -19,7 +19,7 @@ DEPLOYMENT_STRATEGY=$(echo "$CONTEXT" | jq -r ".deployment.strategy") if [ "$ALB_RECONCILIATION_ENABLED" = "false" ] && [ "$DEPLOYMENT_STRATEGY" = "blue_green" ]; then log warn "⚠️ Skipping ALB verification (ALB access needed for blue-green traffic validation)" if command -v np_step_skip >/dev/null 2>&1; then - np_step_skip "ALB verification disabled for blue-green" + np_step_skip "load balancer verification is disabled on this scope (ALB_RECONCILIATION_ENABLED=false)" fi return 0 fi diff --git a/k8s/deployment/workflows/blue_green.yaml b/k8s/deployment/workflows/blue_green.yaml index 4b8e4df0..44c37636 100644 --- a/k8s/deployment/workflows/blue_green.yaml +++ b/k8s/deployment/workflows/blue_green.yaml @@ -4,7 +4,16 @@ configuration: INGRESS_TEMPLATE: "$BLUE_GREEN_INGRESS_PATH" trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - job: k8s-deployment-blue-green + # The job's identity labels are how the deployment page FINDS this plan + # before any run exists (the pending checklist on a fresh deployment): + # the page queries entity + strategy + the scope's provider (the service + # specification id — stable across every scope this agent serves). + job: + name: k8s-deployment-blue-green + labels: + entity: deployment + strategy: blue_green + scope.provider: "@context:scope.provider" groups: - {key: setting-up, title: Setting up} - {key: waiting-instances, title: Waiting for instances to be healthy} @@ -14,10 +23,10 @@ trace: - {key: switching-traffic, title: Switching traffic} - {key: finalize, title: Finalize} steps: + # No milestone title: scaling is a no-op for blue-green (counts ride the + # manifests; the script skips itself) and technical detail for rolling — + # native's Setting up declares no scaling milestone either. - name: update blue deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" - trace: - title: Prepare previous deployment - group: setting-up after: create deployment \ No newline at end of file diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index a706455b..7f893ec6 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -7,7 +7,14 @@ configuration: # the apply step). Step NAMES never change — overrides anchor on them. trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - job: k8s-deployment-initial + # Identity labels: how the deployment page finds this plan before any run + # exists (see blue_green.yaml). + job: + name: k8s-deployment-initial + labels: + entity: deployment + strategy: initial + scope.provider: "@context:scope.provider" groups: - {key: setting-up, title: Setting up} - {key: waiting-instances, title: Waiting for instances to be healthy} diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index f4b1ff6a..12c43d84 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -6,6 +6,10 @@ configuration: # never change — overrides anchor on them. trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + # This workflow runs ONCE PER TRAFFIC INCREMENT (ten times in a 10%-step + # switch): declared steps only, so injected/override plumbing can never + # multiply into noise on the wire. + default: false job: k8s-deployment-switch-traffic groups: - {key: switching-traffic, title: Switching traffic} @@ -48,12 +52,11 @@ steps: type: environment - name: BLUE_DEPLOYMENT_ID type: environment + # Scaling is a no-op for blue-green (counts ride the manifests) and runs + # once per increment — no milestone; the post's health check is the story. - name: create deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" - trace: - title: Scale green deployment - group: switching-traffic post: name: wait deployment active type: script @@ -77,12 +80,10 @@ steps: - name: INGRESS_FILE type: file file: "$OUTPUT_DIR/ingress-$SCOPE_ID-$DEPLOYMENT_ID.yaml" + # Same no-op-for-blue-green scaling — untraced under default: false. - name: update blue deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" - trace: - title: Scale previous deployment - group: switching-traffic - name: apply traffic type: script file: "$SERVICE_PATH/apply_templates" @@ -107,9 +108,8 @@ steps: flavors: [route53, external_dns] configuration: VERIFY_WEIGHTS: true + # Metrics publishing is machine bookkeeping, once per increment. - name: publish_alb_metrics type: script file: "$SERVICE_PATH/deployment/publish_alb_metrics" - trace: - title: Publish load balancer metrics - flavors: [route53] + trace: false diff --git a/nptrace.sh b/nptrace.sh index b5000d9c..f590ec75 100755 --- a/nptrace.sh +++ b/nptrace.sh @@ -1127,26 +1127,44 @@ np_trace_child() { # Staging context # --------------------------------------------------------------------------- +# Upsert one pre-formed `"key":value` entry into a node's staged OBJECT store +# (labels / facets) under its key: re-staging a key REPLACES its entry, so an +# event never ships duplicate keys the parser has to break ties on — and a +# step that re-stages its narrative per heartbeat never grows its payload. +# $1 handle, $2 store key, $3 the `"key":value` entry, $4 the entry's key. +np__stage_entry() { + _stage_entry_handle=$1 + _stage_entry_store=$2 + _stage_entry_slot=$(np__descriptor_slot "$4") + _stage_entry_slots=$(np__node_get "$_stage_entry_handle" "${_stage_entry_store}_slots") + case " $_stage_entry_slots " in + *" $_stage_entry_slot "*) ;; + *) + _stage_entry_slots="${_stage_entry_slots:+$_stage_entry_slots }$_stage_entry_slot" + np__node_set "$_stage_entry_handle" "${_stage_entry_store}_slots" "$_stage_entry_slots" + ;; + esac + np__node_set "$_stage_entry_handle" "${_stage_entry_store}.$_stage_entry_slot" "$3" + _stage_entry_joined='' + for _stage_entry_each in $_stage_entry_slots; do + _stage_entry_value=$(np__node_get "$_stage_entry_handle" "${_stage_entry_store}.$_stage_entry_each") + [ -n "$_stage_entry_value" ] || continue + _stage_entry_joined="${_stage_entry_joined:+$_stage_entry_joined,}$_stage_entry_value" + done + np__node_set "$_stage_entry_handle" "$_stage_entry_store" "{$_stage_entry_joined}" + return 0 +} + # Merge a pre-formed `"key":value` fragment into the node's staged labels. np__stage_label() { - _stage_label_cur=$(np__node_get "$1" labels) - if [ -z "$_stage_label_cur" ] || [ "$_stage_label_cur" = '{}' ]; then - np__node_set "$1" labels "{$2}" - else - np__node_set "$1" labels "${_stage_label_cur%\}},$2}" - fi + _stage_label_key=$(printf '%s' "$2" | sed -n 's/^"\([^"]*\)".*/\1/p') + np__stage_entry "$1" labels "$2" "${_stage_label_key:-$2}" return 0 } +# Last write wins per namespace: re-staging a facet replaces its entry. np__stage_facet() { - _stage_facet_cur=$(np__node_get "$1" facets) - _stage_facet_entry="$(np__json_str "$2"):$3" - if [ -z "$_stage_facet_cur" ] || [ "$_stage_facet_cur" = '{}' ]; then - np__node_set "$1" facets "{$_stage_facet_entry}" - else - # Last write wins per namespace: drop any prior entry for this facet. - np__node_set "$1" facets "${_stage_facet_cur%\}},$_stage_facet_entry}" - fi + np__stage_entry "$1" facets "$(np__json_str "$2"):$3" "$2" return 0 } @@ -1324,19 +1342,42 @@ np__dataset_ref() { np__json_obj type dataset id "$1" } -# Append one io descriptor to a direction's list; the facet is re-staged -# whole each time (last write wins per namespace), so the array only ever -# grows. $1 handle, $2 facet namespace, $3 descriptor store key, $4 the -# already-formed descriptor JSON. -np__append_io_descriptor() { - _append_io_descriptor_descriptors=$(np__node_get "$1" "$3") - if [ -n "$_append_io_descriptor_descriptors" ]; then - _append_io_descriptor_descriptors="$_append_io_descriptor_descriptors,$4" - else - _append_io_descriptor_descriptors=$4 - fi - np__node_set "$1" "$3" "$_append_io_descriptor_descriptors" - np__stage_facet "$1" "$2" "[$_append_io_descriptor_descriptors]" +# A store-safe slot id for a descriptor's identity string (an io NAME, an +# affordance KIND): cksum is POSIX everywhere and collision-resistant enough +# for a node's handful of descriptors. +np__descriptor_slot() { + printf '%s' "$1" | cksum | tr ' \t' '__' +} + +# Upsert one descriptor into a node's list by IDENTITY; the facet is re-staged +# whole each time (last write wins per namespace). A re-declared identity +# REPLACES its previous descriptor in place — a step that reports the same +# name as its state evolves ("instances" per heartbeat) owns ONE entry +# carrying the latest telling, at its first telling's position — while a new +# identity appends. $1 handle, $2 facet namespace, $3 descriptor store key, +# $4 the already-formed descriptor JSON, $5 the identity string. +np__upsert_descriptor() { + _upsert_descriptor_handle=$1 + _upsert_descriptor_facet=$2 + _upsert_descriptor_store=$3 + _upsert_descriptor_slot=$(np__descriptor_slot "$5") + _upsert_descriptor_slots=$(np__node_get "$_upsert_descriptor_handle" "${_upsert_descriptor_store}_slots") + case " $_upsert_descriptor_slots " in + *" $_upsert_descriptor_slot "*) ;; + *) + _upsert_descriptor_slots="${_upsert_descriptor_slots:+$_upsert_descriptor_slots }$_upsert_descriptor_slot" + np__node_set "$_upsert_descriptor_handle" "${_upsert_descriptor_store}_slots" "$_upsert_descriptor_slots" + ;; + esac + np__node_set "$_upsert_descriptor_handle" "${_upsert_descriptor_store}.$_upsert_descriptor_slot" "$4" + _upsert_descriptor_list='' + for _upsert_descriptor_each in $_upsert_descriptor_slots; do + _upsert_descriptor_value=$(np__node_get "$_upsert_descriptor_handle" "${_upsert_descriptor_store}.$_upsert_descriptor_each") + [ -n "$_upsert_descriptor_value" ] || continue + _upsert_descriptor_list="${_upsert_descriptor_list:+$_upsert_descriptor_list,}$_upsert_descriptor_value" + done + np__node_set "$_upsert_descriptor_handle" "$_upsert_descriptor_store" "$_upsert_descriptor_list" + np__stage_facet "$_upsert_descriptor_handle" "$_upsert_descriptor_facet" "[$_upsert_descriptor_list]" return 0 } @@ -1420,9 +1461,9 @@ np__declare_io() { _declare_io_descriptor=$(np__build_io_descriptor "$_declare_io_verb" "$_declare_io_name" "$_declare_io_inline" \ "$_declare_io_uri" "$_declare_io_ref_source" "$_declare_io_ref_id" "$_declare_io_ref_version") || return 0 if [ "$_declare_io_direction" = 'out' ]; then - np__append_io_descriptor "$_declare_io_handle" "$NP_FACET_OUTPUT" io_output "$_declare_io_descriptor" + np__upsert_descriptor "$_declare_io_handle" "$NP_FACET_OUTPUT" io_output "$_declare_io_descriptor" "$_declare_io_name" else - np__append_io_descriptor "$_declare_io_handle" "$NP_FACET_INPUT" io_input "$_declare_io_descriptor" + np__upsert_descriptor "$_declare_io_handle" "$NP_FACET_INPUT" io_input "$_declare_io_descriptor" "$_declare_io_name" fi np__flush_foreign "$_declare_io_handle" return 0 @@ -1450,14 +1491,15 @@ np_trace_input() { return 0 } -# np__emit_io_edge [descriptor-json] +# np__emit_io_edge [descriptor-json] [descriptor-name] # # Emit one lineage edge. The direction decides everything else: `out` is # edge.produces + tracing.output, `in` is edge.consumes + tracing.input. # -# With a descriptor the io is declared ONCE: it accumulates into the node's -# io facet AND becomes the edge's tracing.binding — the same single-source -# rule as the sibling SDKs. Without one, the edge records lineage only. +# With a descriptor the io is declared ONCE: it upserts into the node's io +# facet BY NAME (a re-declared name replaces its entry) AND becomes the +# edge's tracing.binding — the same single-source rule as the sibling SDKs. +# Without one, the edge records lineage only. # # On a FOREIGN (adopted) node this is an observed fact, exactly like # np_trace_error: the edge is ours to say, and the staged io facet reaches the @@ -1478,9 +1520,11 @@ np__emit_io_edge() { fi _emit_io_edge_binding=$4 + _emit_io_edge_binding_name=${5:-$_emit_io_edge_binding} if [ -n "$_emit_io_edge_binding" ]; then - np__append_io_descriptor "$_emit_io_edge_handle" "$_emit_io_edge_facet_namespace" "$_emit_io_edge_descriptor_store" "$_emit_io_edge_binding" + np__upsert_descriptor "$_emit_io_edge_handle" "$_emit_io_edge_facet_namespace" "$_emit_io_edge_descriptor_store" \ + "$_emit_io_edge_binding" "$_emit_io_edge_binding_name" fi # An edge must not point FROM a node the read model has never seen. @@ -1550,7 +1594,8 @@ np__declare_lineage() { "$_declare_lineage_uri" "$_declare_lineage_ref_source" "$_declare_lineage_ref_id" "$_declare_lineage_ref_version") || return 0 fi - np__emit_io_edge "$_declare_lineage_handle" "$_declare_lineage_direction" "$_declare_lineage_dataset_id" "$_declare_lineage_binding" + np__emit_io_edge "$_declare_lineage_handle" "$_declare_lineage_direction" "$_declare_lineage_dataset_id" \ + "$_declare_lineage_binding" "$_declare_lineage_name" return 0 } @@ -2073,6 +2118,11 @@ np_trace_plan() { # control (view live logs, switch traffic). One affordance object # ('{"kind":"deploy-log",...}') or a bare array of them; the wire form is # always the array. +# +# A single object UPSERTS by its `kind`: re-declaring a kind replaces that +# entry (a live meter re-emitted per heartbeat), while a NEW kind joins the +# list — a later "deploy-log" never erases the "instances-health" meter. +# An array is a FULL declaration and replaces the whole list. np_trace_affordances() { _affordances_handle=$(np__resolve_handle "${1:-}") if np__is_handle "${1:-}"; then @@ -2081,11 +2131,21 @@ np_trace_affordances() { np__is_handle "$_affordances_handle" || return 0 _affordances_body=${1:-} case "$_affordances_body" in - \[*) ;; - \{*) _affordances_body="[$_affordances_body]" ;; + \[*) + np__node_set "$_affordances_handle" affordances_slots '' + np__node_set "$_affordances_handle" affordances '' + np__stage_facet "$_affordances_handle" "$NP_FACET_AFFORDANCES" "$_affordances_body" + ;; + \{*) + _affordances_kind=$(printf '%s' "$_affordances_body" \ + | sed -n 's/.*"kind"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p') + # No kind: the object itself is its identity (append-once semantics). + [ -n "$_affordances_kind" ] || _affordances_kind=$_affordances_body + np__upsert_descriptor "$_affordances_handle" "$NP_FACET_AFFORDANCES" affordances \ + "$_affordances_body" "$_affordances_kind" + ;; *) np__drop 'affordances' 'body must be a JSON object or array'; return 0 ;; esac - np__stage_facet "$_affordances_handle" "$NP_FACET_AFFORDANCES" "$_affordances_body" np__flush_foreign "$_affordances_handle" return 0 } diff --git a/vendor/catalog-tracing-sh b/vendor/catalog-tracing-sh index 2c1e2a93..3be6ad61 160000 --- a/vendor/catalog-tracing-sh +++ b/vendor/catalog-tracing-sh @@ -1 +1 @@ -Subproject commit 2c1e2a93970d855bed3af1cc6342532139db9d44 +Subproject commit 3be6ad61cdf231e3c50c2829718663a4829d6e10 From 21cd41eaedb647255f776908ff3c388b45f5a12e Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 21:23:21 -0300 Subject: [PATCH 33/52] fix(tracing): deployment workflows trace declared steps only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The finalize station folded eight 0ms override-plumbing fragments as 'technical steps' — native shows none of that. Every deployment workflow (initial, blue_green, finalize, rollback, delete) now opts into declared-steps-only: engine fragments without a trace block stay off the wire, while script-emitted child steps (the per-manifest create-secret/deployment/service acts) keep filling the fold with real observable work. publish_alb_metrics untraced everywhere (bookkeeping). --- k8s/deployment/workflows/blue_green.yaml | 5 +++++ k8s/deployment/workflows/delete.yaml | 5 +++++ k8s/deployment/workflows/finalize.yaml | 5 +++++ k8s/deployment/workflows/initial.yaml | 11 ++++++++--- k8s/deployment/workflows/rollback.yaml | 5 +++++ 5 files changed, 28 insertions(+), 3 deletions(-) diff --git a/k8s/deployment/workflows/blue_green.yaml b/k8s/deployment/workflows/blue_green.yaml index 44c37636..9252d240 100644 --- a/k8s/deployment/workflows/blue_green.yaml +++ b/k8s/deployment/workflows/blue_green.yaml @@ -4,6 +4,11 @@ configuration: INGRESS_TEMPLATE: "$BLUE_GREEN_INGRESS_PATH" trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + # Declared steps only: engine fragments without a trace block (override + # plumbing) stay off the wire — the fold shows real observable acts, not + # 0ms no-ops. Script-emitted child steps (per-manifest applies) are + # unaffected. + default: false # The job's identity labels are how the deployment page FINDS this plan # before any run exists (the pending checklist on a fresh deployment): # the page queries entity + strategy + the scope's provider (the service diff --git a/k8s/deployment/workflows/delete.yaml b/k8s/deployment/workflows/delete.yaml index 0c99a14e..99e6f12e 100644 --- a/k8s/deployment/workflows/delete.yaml +++ b/k8s/deployment/workflows/delete.yaml @@ -2,6 +2,11 @@ include: - "$SERVICE_PATH/values.yaml" trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + # Declared steps only: engine fragments without a trace block (override + # plumbing) stay off the wire — the fold shows real observable acts, not + # 0ms no-ops. Script-emitted child steps (per-manifest applies) are + # unaffected. + default: false job: k8s-deployment-delete steps: - name: load logging diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index 2c2bd24a..ac79350c 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -4,6 +4,11 @@ configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + # Declared steps only: engine fragments without a trace block (override + # plumbing) stay off the wire — the fold shows real observable acts, not + # 0ms no-ops. Script-emitted child steps (per-manifest applies) are + # unaffected. + default: false job: k8s-deployment-finalize groups: - {key: finalize, title: Finalize} diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index 7f893ec6..ea61111e 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -7,6 +7,11 @@ configuration: # the apply step). Step NAMES never change — overrides anchor on them. trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + # Declared steps only: engine fragments without a trace block (override + # plumbing) stay off the wire — the fold shows real observable acts, not + # 0ms no-ops. Script-emitted child steps (per-manifest applies) are + # unaffected. + default: false # Identity labels: how the deployment page finds this plan before any run # exists (see blue_green.yaml). job: @@ -119,12 +124,12 @@ steps: flavors: [route53, external_dns] configuration: VERIFY_WEIGHTS: false + # Metrics publishing is machine bookkeeping — untraced, like the + # per-increment copy in switch_traffic. - name: publish_alb_metrics type: script file: "$SERVICE_PATH/deployment/publish_alb_metrics" - trace: - title: Publish load balancer metrics - flavors: [route53] + trace: false - name: wait deployment active type: script file: "$SERVICE_PATH/deployment/wait_deployment_active" diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index ec25a33f..304f9a9e 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -4,6 +4,11 @@ configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + # Declared steps only: engine fragments without a trace block (override + # plumbing) stay off the wire — the fold shows real observable acts, not + # 0ms no-ops. Script-emitted child steps (per-manifest applies) are + # unaffected. + default: false job: k8s-deployment-rollback groups: - {key: finalize, title: Finalize} From 78d6c9dc363707c1c5a89a4854b82b2eb964323f Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 21:48:20 -0300 Subject: [PATCH 34/52] fix(tracing): vendored SDK sends wire-shaped edge bindings io lineage edges (image consumed, logs/ingress/workload produced) were silently dead-lettering: the binding carried the io descriptor's kind/uri, which the API's tracing.binding schema rejects. The bumped SDK sends {name} on the edge and keeps the full descriptor on the node. --- nptrace.sh | 19 +++++++++++-------- vendor/catalog-tracing-sh | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/nptrace.sh b/nptrace.sh index f590ec75..1d971f6e 100755 --- a/nptrace.sh +++ b/nptrace.sh @@ -1497,9 +1497,11 @@ np_trace_input() { # edge.produces + tracing.output, `in` is edge.consumes + tracing.input. # # With a descriptor the io is declared ONCE: it upserts into the node's io -# facet BY NAME (a re-declared name replaces its entry) AND becomes the -# edge's tracing.binding — the same single-source rule as the sibling SDKs. -# Without one, the edge records lineage only. +# facet BY NAME (a re-declared name replaces its entry), and the edge carries +# a tracing.binding built from it. The binding's WIRE shape is `{name, +# content_type?, size_bytes?}` — never the io descriptor itself (whose +# kind/uri/value the binding schema rejects, dead-lettering the edge). +# Without a descriptor, the edge records lineage only. # # On a FOREIGN (adopted) node this is an observed fact, exactly like # np_trace_error: the edge is ours to say, and the staged io facet reaches the @@ -1519,18 +1521,19 @@ np__emit_io_edge() { _emit_io_edge_descriptor_store=io_input fi - _emit_io_edge_binding=$4 - _emit_io_edge_binding_name=${5:-$_emit_io_edge_binding} + _emit_io_edge_descriptor=$4 + _emit_io_edge_name=${5:-} - if [ -n "$_emit_io_edge_binding" ]; then + if [ -n "$_emit_io_edge_descriptor" ]; then np__upsert_descriptor "$_emit_io_edge_handle" "$_emit_io_edge_facet_namespace" "$_emit_io_edge_descriptor_store" \ - "$_emit_io_edge_binding" "$_emit_io_edge_binding_name" + "$_emit_io_edge_descriptor" "${_emit_io_edge_name:-$_emit_io_edge_descriptor}" fi # An edge must not point FROM a node the read model has never seen. np_trace_start "$_emit_io_edge_handle" - if [ -n "$_emit_io_edge_binding" ]; then + if [ -n "$_emit_io_edge_descriptor" ] && [ -n "$_emit_io_edge_name" ]; then + _emit_io_edge_binding=$(np__json_obj name "$_emit_io_edge_name") _emit_io_edge_edge_data=$(np__json_obj_raw \ from "$(np__ref_of "$_emit_io_edge_handle")" \ to "$(np__dataset_ref "$_emit_io_edge_dataset_id")" \ diff --git a/vendor/catalog-tracing-sh b/vendor/catalog-tracing-sh index 3be6ad61..b85abf11 160000 --- a/vendor/catalog-tracing-sh +++ b/vendor/catalog-tracing-sh @@ -1 +1 @@ -Subproject commit 3be6ad61cdf231e3c50c2829718663a4829d6e10 +Subproject commit b85abf11547cfe01a4e151f7be9b477e7fe545f4 From dd1772afbb57142d74de85715dcf48e9a8f13747 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 22:07:49 -0300 Subject: [PATCH 35/52] fix(tracing): waits declare their signal, not label chips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit np_scope_wait_heartbeat filed per-poll diagnostics (wait.what/ready/ state/desired/..., including an unresolved-config literal) as LABELS, which the dialog renders as tag chips — machine noise on a human surface. What a phase waits on is the tracing.signal facet: name + direction + deadline; the counts already ride the meter and progress facets, elapsed rides the node's own clock. --- k8s/logging | 28 +++++++++++++++------------- scheduled_task/logging | 28 +++++++++++++++------------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/k8s/logging b/k8s/logging index 608e71e1..c933caa9 100644 --- a/k8s/logging +++ b/k8s/logging @@ -236,22 +236,24 @@ np_scope_step_timeout() { # np_scope_wait_heartbeat [state] [k=v ...] # -# Mark the current step (or open sub-step) `waiting` and record progress, -# flushed immediately so the wait is visible LIVE, not after the workflow -# ends. Extra k=v pairs carry structured progress — replica counts, ALB state -# — as labels on the node. The flush is bounded to 2s per beat: with the API -# down, a 30s-cadence wait loses at most ~6% of its poll budget — the deadline -# is wall-clock, so the timeout is never extended. Always defined; a no-op -# when the workflow is untraced. +# Mark the current step (or open sub-step) `waiting` and declare WHAT it +# waits on as the tracing.signal facet — never as labels: tags are queryable +# filing, and per-poll diagnostics rendered as label chips are noise (counts +# ride the meter/progress facets; elapsed rides the node's own clock). The +# state word and extra k=v pairs are accepted for call-site compatibility and +# deliberately not recorded. The flush is bounded to 2s per beat: with the +# API down, a 30s-cadence wait loses at most ~6% of its poll budget — the +# deadline is wall-clock, so the timeout is never extended. Always defined; +# a no-op when the workflow is untraced. np_scope_wait_heartbeat() { local _hb_node _hb_node=$(_np_scopes_node) || return 0 - local _hb_what="${1:-}" _hb_elapsed="${2:-0}" _hb_timeout="${3:-0}" _hb_state="${4:-}" - shift 4 2>/dev/null || shift $# - np_trace_labels "$_hb_node" \ - "wait.what=$_hb_what" "wait.elapsed_s=$_hb_elapsed" "wait.timeout_s=$_hb_timeout" \ - ${_hb_state:+"wait.state=$_hb_state"} \ - ${1+"$@"} + local _hb_what="${1:-}" _hb_timeout="${3:-0}" + # A non-numeric timeout (an unresolved configuration) just drops the deadline. + case "$_hb_timeout" in + '' | *[!0-9]*) np_trace_signal "$_hb_node" "$_hb_what" wait ;; + *) np_trace_signal "$_hb_node" "$_hb_what" wait --timeout-ms $(( _hb_timeout * 1000 )) ;; + esac np_trace_waiting "$_hb_node" NP_TRACE_FLUSH_TIMEOUT=2 np_trace_flush return 0 diff --git a/scheduled_task/logging b/scheduled_task/logging index 608e71e1..c933caa9 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -236,22 +236,24 @@ np_scope_step_timeout() { # np_scope_wait_heartbeat [state] [k=v ...] # -# Mark the current step (or open sub-step) `waiting` and record progress, -# flushed immediately so the wait is visible LIVE, not after the workflow -# ends. Extra k=v pairs carry structured progress — replica counts, ALB state -# — as labels on the node. The flush is bounded to 2s per beat: with the API -# down, a 30s-cadence wait loses at most ~6% of its poll budget — the deadline -# is wall-clock, so the timeout is never extended. Always defined; a no-op -# when the workflow is untraced. +# Mark the current step (or open sub-step) `waiting` and declare WHAT it +# waits on as the tracing.signal facet — never as labels: tags are queryable +# filing, and per-poll diagnostics rendered as label chips are noise (counts +# ride the meter/progress facets; elapsed rides the node's own clock). The +# state word and extra k=v pairs are accepted for call-site compatibility and +# deliberately not recorded. The flush is bounded to 2s per beat: with the +# API down, a 30s-cadence wait loses at most ~6% of its poll budget — the +# deadline is wall-clock, so the timeout is never extended. Always defined; +# a no-op when the workflow is untraced. np_scope_wait_heartbeat() { local _hb_node _hb_node=$(_np_scopes_node) || return 0 - local _hb_what="${1:-}" _hb_elapsed="${2:-0}" _hb_timeout="${3:-0}" _hb_state="${4:-}" - shift 4 2>/dev/null || shift $# - np_trace_labels "$_hb_node" \ - "wait.what=$_hb_what" "wait.elapsed_s=$_hb_elapsed" "wait.timeout_s=$_hb_timeout" \ - ${_hb_state:+"wait.state=$_hb_state"} \ - ${1+"$@"} + local _hb_what="${1:-}" _hb_timeout="${3:-0}" + # A non-numeric timeout (an unresolved configuration) just drops the deadline. + case "$_hb_timeout" in + '' | *[!0-9]*) np_trace_signal "$_hb_node" "$_hb_what" wait ;; + *) np_trace_signal "$_hb_node" "$_hb_what" wait --timeout-ms $(( _hb_timeout * 1000 )) ;; + esac np_trace_waiting "$_hb_node" NP_TRACE_FLUSH_TIMEOUT=2 np_trace_flush return 0 From 14096fbb7ac7261d435107433e5743cdae95801e Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 22:30:15 -0300 Subject: [PATCH 36/52] fix(tracing): applied/removed manifests named by their literal k8s kind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The io names said 'workload' — a k8s category, not a manifest kind. The dialog now names what kubectl actually reported: deployment (and removed-deployment on deletes); the dataset ids were always exact (k8s-deployment:/). --- k8s/logging | 4 ++-- scheduled_task/logging | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/logging b/k8s/logging index c933caa9..ddd32c5e 100644 --- a/k8s/logging +++ b/k8s/logging @@ -383,7 +383,7 @@ np_scope_k8s_applied() { _ka_name="${_ka_name%% *}" case "$_ka_kind" in deployment.apps|deployment) - np_scope_produces "k8s-deployment:$_ka_ns/$_ka_name" workload "$_ka_name" ;; + np_scope_produces "k8s-deployment:$_ka_ns/$_ka_name" deployment "$_ka_name" ;; service) np_scope_produces "k8s-service:$_ka_ns/$_ka_name" service "$_ka_name" ;; ingress.networking.k8s.io|ingress) @@ -417,7 +417,7 @@ np_scope_k8s_deleted() { [ -n "$_kd_name" ] || continue case "$_kd_kind" in deployment.apps|deployment) - np_trace_output "$_kd_node" removed-workload --uri "$_kd_name" ;; + np_trace_output "$_kd_node" removed-deployment --uri "$_kd_name" ;; service) np_trace_output "$_kd_node" removed-service --uri "$_kd_name" ;; ingress.networking.k8s.io|ingress) diff --git a/scheduled_task/logging b/scheduled_task/logging index c933caa9..ddd32c5e 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -383,7 +383,7 @@ np_scope_k8s_applied() { _ka_name="${_ka_name%% *}" case "$_ka_kind" in deployment.apps|deployment) - np_scope_produces "k8s-deployment:$_ka_ns/$_ka_name" workload "$_ka_name" ;; + np_scope_produces "k8s-deployment:$_ka_ns/$_ka_name" deployment "$_ka_name" ;; service) np_scope_produces "k8s-service:$_ka_ns/$_ka_name" service "$_ka_name" ;; ingress.networking.k8s.io|ingress) @@ -417,7 +417,7 @@ np_scope_k8s_deleted() { [ -n "$_kd_name" ] || continue case "$_kd_kind" in deployment.apps|deployment) - np_trace_output "$_kd_node" removed-workload --uri "$_kd_name" ;; + np_trace_output "$_kd_node" removed-deployment --uri "$_kd_name" ;; service) np_trace_output "$_kd_node" removed-service --uri "$_kd_name" ;; ingress.networking.k8s.io|ingress) From 537730f793296034d89014d2a99049d4c10a69ac Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sat, 29 Aug 2026 23:18:20 -0300 Subject: [PATCH 37/52] feat(tracing): job definitions scoped per provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every workflow's trace.job carries namespace: @context:scope.provider — the scope's service specification. Two providers shipping the same workflow name with an identical plan hash previously converged on ONE job definition (namespace 'workflows') and would fight over its identity labels; per-provider namespaces make each scope type own its jobs by construction. --- k8s/deployment/workflows/blue_green.yaml | 1 + k8s/deployment/workflows/delete.yaml | 7 ++++++- k8s/deployment/workflows/finalize.yaml | 7 ++++++- k8s/deployment/workflows/initial.yaml | 1 + k8s/deployment/workflows/rollback.yaml | 7 ++++++- k8s/deployment/workflows/switch_traffic.yaml | 7 ++++++- k8s/scope/workflows/create.yaml | 7 ++++++- k8s/scope/workflows/delete.yaml | 7 ++++++- k8s/scope/workflows/update.yaml | 7 ++++++- 9 files changed, 44 insertions(+), 7 deletions(-) diff --git a/k8s/deployment/workflows/blue_green.yaml b/k8s/deployment/workflows/blue_green.yaml index 9252d240..33f5319c 100644 --- a/k8s/deployment/workflows/blue_green.yaml +++ b/k8s/deployment/workflows/blue_green.yaml @@ -15,6 +15,7 @@ trace: # specification id — stable across every scope this agent serves). job: name: k8s-deployment-blue-green + namespace: "@context:scope.provider" labels: entity: deployment strategy: blue_green diff --git a/k8s/deployment/workflows/delete.yaml b/k8s/deployment/workflows/delete.yaml index 99e6f12e..08de893e 100644 --- a/k8s/deployment/workflows/delete.yaml +++ b/k8s/deployment/workflows/delete.yaml @@ -7,7 +7,12 @@ trace: # 0ms no-ops. Script-emitted child steps (per-manifest applies) are # unaffected. default: false - job: k8s-deployment-delete + # Per-provider identity: jobs are scoped to the scope's service + # specification, so two providers sharing a workflow name never + # converge on one definition. + job: + name: k8s-deployment-delete + namespace: "@context:scope.provider" steps: - name: load logging type: script diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index ac79350c..cc3a1308 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -9,7 +9,12 @@ trace: # 0ms no-ops. Script-emitted child steps (per-manifest applies) are # unaffected. default: false - job: k8s-deployment-finalize + # Per-provider identity: jobs are scoped to the scope's service + # specification, so two providers sharing a workflow name never + # converge on one definition. + job: + name: k8s-deployment-finalize + namespace: "@context:scope.provider" groups: - {key: finalize, title: Finalize} steps: diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index ea61111e..c7ab9d2b 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -16,6 +16,7 @@ trace: # exists (see blue_green.yaml). job: name: k8s-deployment-initial + namespace: "@context:scope.provider" labels: entity: deployment strategy: initial diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index 304f9a9e..04e32c34 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -9,7 +9,12 @@ trace: # 0ms no-ops. Script-emitted child steps (per-manifest applies) are # unaffected. default: false - job: k8s-deployment-rollback + # Per-provider identity: jobs are scoped to the scope's service + # specification, so two providers sharing a workflow name never + # converge on one definition. + job: + name: k8s-deployment-rollback + namespace: "@context:scope.provider" groups: - {key: finalize, title: Finalize} steps: diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index 12c43d84..e45d34a7 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -10,7 +10,12 @@ trace: # switch): declared steps only, so injected/override plumbing can never # multiply into noise on the wire. default: false - job: k8s-deployment-switch-traffic + # Per-provider identity: jobs are scoped to the scope's service + # specification, so two providers sharing a workflow name never + # converge on one definition. + job: + name: k8s-deployment-switch-traffic + namespace: "@context:scope.provider" groups: - {key: switching-traffic, title: Switching traffic} steps: diff --git a/k8s/scope/workflows/create.yaml b/k8s/scope/workflows/create.yaml index 75b6e946..4e7c8366 100644 --- a/k8s/scope/workflows/create.yaml +++ b/k8s/scope/workflows/create.yaml @@ -6,7 +6,12 @@ include: # anchor on them. trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - job: k8s-scope-create + # Per-provider identity: jobs are scoped to the scope's service + # specification, so two providers sharing a workflow name never + # converge on one definition. + job: + name: k8s-scope-create + namespace: "@context:scope.provider" steps: - name: load logging type: script diff --git a/k8s/scope/workflows/delete.yaml b/k8s/scope/workflows/delete.yaml index 16daa485..c686203b 100644 --- a/k8s/scope/workflows/delete.yaml +++ b/k8s/scope/workflows/delete.yaml @@ -2,7 +2,12 @@ include: - "$SERVICE_PATH/values.yaml" trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - job: k8s-scope-delete + # Per-provider identity: jobs are scoped to the scope's service + # specification, so two providers sharing a workflow name never + # converge on one definition. + job: + name: k8s-scope-delete + namespace: "@context:scope.provider" steps: - name: load logging type: script diff --git a/k8s/scope/workflows/update.yaml b/k8s/scope/workflows/update.yaml index 2feb3e4d..a784b1e2 100644 --- a/k8s/scope/workflows/update.yaml +++ b/k8s/scope/workflows/update.yaml @@ -2,7 +2,12 @@ include: - "$SERVICE_PATH/scope/workflows/create.yaml" trace: flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - job: k8s-scope-update + # Per-provider identity: jobs are scoped to the scope's service + # specification, so two providers sharing a workflow name never + # converge on one definition. + job: + name: k8s-scope-update + namespace: "@context:scope.provider" steps: - name: networking type: workflow From ed9dc05a2556061f816f0e94a02b85948142dbf5 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sun, 30 Aug 2026 08:25:27 -0300 Subject: [PATCH 38/52] feat(workflows): every traced workflow declares what a human calls it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page titled these runs by humanizing `labels.workflow` — this file's basename — which meant the name on screen was whatever the file happened to be called. `trace.title` says it outright, landing on the run's explain.title where every consumer looks first. --- k8s/deployment/workflows/blue_green.yaml | 4 ++++ k8s/deployment/workflows/delete.yaml | 1 + k8s/deployment/workflows/finalize.yaml | 1 + k8s/deployment/workflows/initial.yaml | 1 + k8s/deployment/workflows/rollback.yaml | 1 + k8s/deployment/workflows/switch_traffic.yaml | 1 + k8s/scope/workflows/create.yaml | 1 + k8s/scope/workflows/delete.yaml | 1 + k8s/scope/workflows/update.yaml | 1 + scheduled_task/scope/workflows/trigger-job.yaml | 1 + 10 files changed, 13 insertions(+) diff --git a/k8s/deployment/workflows/blue_green.yaml b/k8s/deployment/workflows/blue_green.yaml index 33f5319c..89517c12 100644 --- a/k8s/deployment/workflows/blue_green.yaml +++ b/k8s/deployment/workflows/blue_green.yaml @@ -3,6 +3,10 @@ include: configuration: INGRESS_TEMPLATE: "$BLUE_GREEN_INGRESS_PATH" trace: + # What a HUMAN calls this execution. It lands on the run's explain.title — + # the first rung of every consumer's naming contract — so the page shows the + # name we chose, never a wording derived from this file's basename. + title: Blue/green deployment flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] # Declared steps only: engine fragments without a trace block (override # plumbing) stay off the wire — the fold shows real observable acts, not diff --git a/k8s/deployment/workflows/delete.yaml b/k8s/deployment/workflows/delete.yaml index 08de893e..512a41d3 100644 --- a/k8s/deployment/workflows/delete.yaml +++ b/k8s/deployment/workflows/delete.yaml @@ -1,6 +1,7 @@ include: - "$SERVICE_PATH/values.yaml" trace: + title: Remove deployment flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] # Declared steps only: engine fragments without a trace block (override # plumbing) stay off the wire — the fold shows real observable acts, not diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index cc3a1308..c79702d0 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -3,6 +3,7 @@ include: configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" trace: + title: Finalize deployment flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] # Declared steps only: engine fragments without a trace block (override # plumbing) stay off the wire — the fold shows real observable acts, not diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index c7ab9d2b..b4464c4a 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -6,6 +6,7 @@ configuration: # create-* steps are emitted per applied manifest by apply_templates, under # the apply step). Step NAMES never change — overrides anchor on them. trace: + title: Initial deployment flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] # Declared steps only: engine fragments without a trace block (override # plumbing) stay off the wire — the fold shows real observable acts, not diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index 04e32c34..1d76438f 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -3,6 +3,7 @@ include: configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" trace: + title: Roll back deployment flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] # Declared steps only: engine fragments without a trace block (override # plumbing) stay off the wire — the fold shows real observable acts, not diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index e45d34a7..015cc737 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -5,6 +5,7 @@ configuration: # Trace identities speak the platform's deploy step vocabulary. Step NAMES # never change — overrides anchor on them. trace: + title: Switch traffic flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] # This workflow runs ONCE PER TRAFFIC INCREMENT (ten times in a 10%-step # switch): declared steps only, so injected/override plumbing can never diff --git a/k8s/scope/workflows/create.yaml b/k8s/scope/workflows/create.yaml index 4e7c8366..da4f78a6 100644 --- a/k8s/scope/workflows/create.yaml +++ b/k8s/scope/workflows/create.yaml @@ -5,6 +5,7 @@ include: # reads identically in the timeline. Step NAMES never change — overrides # anchor on them. trace: + title: Create scope flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] # Per-provider identity: jobs are scoped to the scope's service # specification, so two providers sharing a workflow name never diff --git a/k8s/scope/workflows/delete.yaml b/k8s/scope/workflows/delete.yaml index c686203b..cf1abfee 100644 --- a/k8s/scope/workflows/delete.yaml +++ b/k8s/scope/workflows/delete.yaml @@ -1,6 +1,7 @@ include: - "$SERVICE_PATH/values.yaml" trace: + title: Delete scope flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] # Per-provider identity: jobs are scoped to the scope's service # specification, so two providers sharing a workflow name never diff --git a/k8s/scope/workflows/update.yaml b/k8s/scope/workflows/update.yaml index a784b1e2..3eedc336 100644 --- a/k8s/scope/workflows/update.yaml +++ b/k8s/scope/workflows/update.yaml @@ -1,6 +1,7 @@ include: - "$SERVICE_PATH/scope/workflows/create.yaml" trace: + title: Update scope flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] # Per-provider identity: jobs are scoped to the scope's service # specification, so two providers sharing a workflow name never diff --git a/scheduled_task/scope/workflows/trigger-job.yaml b/scheduled_task/scope/workflows/trigger-job.yaml index b07ade7e..5b05a728 100644 --- a/scheduled_task/scope/workflows/trigger-job.yaml +++ b/scheduled_task/scope/workflows/trigger-job.yaml @@ -4,6 +4,7 @@ provider_categories: - container-orchestration - cloud-providers trace: + title: Run scheduled task job: scheduled-task-trigger steps: - name: load logging From 83791cc81400c79f6c19d5a5cdd5b145bbc1f4a8 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sun, 30 Aug 2026 08:38:31 -0300 Subject: [PATCH 39/52] test(tracing): pin the wait, binding and progress contracts as they now stand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four assertions still described the behaviour these changes replaced, so the branch was red on work that is correct: - a wait states `tracing.signal` (name, direction, deadline in ms), not `wait.*` labels — the labels were rendering as code-y chips on the phase; - an edge's binding names WHICH io the edge is about and nothing more (sending the whole descriptor is what dead-lettered every lineage edge); - progress units are the closed set, so `count` stays `count`. The extra-k=v-labels case is replaced by its inverse — a guard that no wait bookkeeping reaches the labels — and a wait whose timeout never resolved is now covered too. --- k8s/utils/tests/trace_logging.bats | 35 +++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index 067dd6fa..b4647d48 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -103,13 +103,22 @@ run_logged() { [ "$(echo "$output" | grep 'apply-manifests@0.0"' | grep -c 'tracing.error')" -eq 1 ] } -@test "wait heartbeat marks the step waiting with progress labels" { +@test "wait heartbeat marks the step waiting and states the signal it waits on" { run_logged 'np_scope_wait_heartbeat "alb-active" 90 300 "pending"' [ "$status" -eq 0 ] echo "$output" | grep -q '"status":"waiting"' - echo "$output" | grep -q '"wait.what":"alb-active"' - echo "$output" | grep -q '"wait.elapsed_s":"90"' - echo "$output" | grep -q '"wait.timeout_s":"300"' + # What is being waited for is a FACET, not labels: a wait is a first-class + # thing every consumer can render, and the deadline rides it in milliseconds. + echo "$output" | grep -q '"tracing.signal":{"name":"alb-active","direction":"wait","timeout_ms":300000}' +} + +@test "a wait with an unresolved timeout still states the wait, without a deadline" { + # An unresolved configuration ($DEPLOYMENT_MAX_WAIT_IN_SECONDS arriving + # literal) must not put a nonsense deadline on the wire. + run_logged 'np_scope_wait_heartbeat "alb-active" 90 "MAX_WAIT" "pending"' + [ "$status" -eq 0 ] + echo "$output" | grep -q '"tracing.signal":{"name":"alb-active","direction":"wait"}' + ! echo "$output" | grep -q 'timeout_ms' } @test "without NP_TRACE, logging is byte-identical to plain logging" { @@ -191,11 +200,13 @@ hello" ] echo "$output" | grep '"status":"waiting"' | grep -q 'wait-alb-active@0.0' } -@test "heartbeat extra k=v pairs land as labels" { - run_logged 'np_scope_wait_heartbeat "deployment-active" 20 600 "progressing" "wait.ready=2" "wait.desired=5"' +@test "a heartbeat puts NO wait bookkeeping in the labels" { + # Labels are how a node is FILED, and a consumer renders them as tags. The + # wait's own bookkeeping is not filing metadata — it read as a row of code-y + # chips on the phase, which is what the signal facet replaced. + run_logged 'np_scope_wait_heartbeat "deployment-active" 20 600 "progressing"' [ "$status" -eq 0 ] - echo "$output" | grep -q '"wait.ready":"2"' - echo "$output" | grep -q '"wait.desired":"5"' + ! echo "$output" | grep -q '"wait\.' } @test "a sub-step left open when the platform moves on is forgotten, not reused" { @@ -251,7 +262,9 @@ hello" ] run_logged 'np_scope_produces "dns-record:api.example.com" dns_record "api.example.com"' [ "$status" -eq 0 ] echo "$output" | grep '"edge.produces"' | grep -q '"id":"dns-record:api.example.com"' - echo "$output" | grep -q '"tracing.binding":{"kind":"pointer","name":"dns_record","uri":"api.example.com"}' + # The edge binding says WHICH io of the node the edge is about — a name, and + # nothing else. The descriptor itself stays whole on the node's io facet. + echo "$output" | grep -q '"tracing.binding":{"name":"dns_record"}' # foreign re-emit carries the io facet on the adopted step echo "$output" | grep '"tracing.output"' | grep -q 'apply-manifests@0.0' } @@ -294,7 +307,9 @@ secret/sec-1 created" ' [ "$status" -eq 0 ] echo "$output" | grep -q '"tracing.affordances":\[{"kind":"deploy-log","application_id":"7"}\]' - echo "$output" | grep -q '"tracing.progress":{"current":3,"target":10,"unit":"instances"}' + # The unit is a CLOSED set every consumer can render (percent/count/bytes/ + # milliseconds) — never a free-text noun the reader has to interpret. + echo "$output" | grep -q '"tracing.progress":{"current":3,"target":10,"unit":"count"}' } @test "lineage helpers are defined no-ops when untraced" { From 0817dc5ebb00a3bc4849b63609d948a026c24b98 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sun, 30 Aug 2026 09:08:06 -0300 Subject: [PATCH 40/52] chore(vendor): track the SDK at its merged commit on main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit catalog-tracing-sh#10 is merged, and its branch is gone. The submodule pointed at a commit that now only exists inside main's history — the vendored content is identical, but .gitmodules declares `branch = main`, so the pointer should be main's tip. --- vendor/catalog-tracing-sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/catalog-tracing-sh b/vendor/catalog-tracing-sh index b85abf11..31bb0721 160000 --- a/vendor/catalog-tracing-sh +++ b/vendor/catalog-tracing-sh @@ -1 +1 @@ -Subproject commit b85abf11547cfe01a4e151f7be9b477e7fe545f4 +Subproject commit 31bb0721c8d9fa102084470cebff56ca60ccf93f From 01de7d4e027b8255e7be698c8e4f1bc595679f3d Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sun, 30 Aug 2026 09:42:38 -0300 Subject: [PATCH 41/52] feat(workflows): finalize and rollback say what each step is for, and stop advertising work they cannot do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'Promote new deployment' (and rollback's 'Restore previous deployment') run scale_deployments, which is a no-op for every strategy but rolling — so on a blue-green scope they were permanently grey rows nobody could explain. The scope's deploy strategy becomes a flavor DIMENSION, read from the same context path the scripts read, and those steps gate on `rolling`. One job still, two plan versions — which is exactly what hashing the effective plan is for. Every remaining finalize/rollback step now carries a one-line description, so the checklist reads without prior knowledge of the workflow. --- k8s/deployment/workflows/finalize.yaml | 15 ++++++++++++++- k8s/deployment/workflows/rollback.yaml | 11 ++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index c79702d0..ba4a0f24 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -4,7 +4,11 @@ configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" trace: title: Finalize deployment - flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + # The scope's deploy strategy is a flavor DIMENSION: steps that only one + # strategy can ever run gate on it, so a checklist never advertises work + # this execution cannot do. Absent config reads as not-rolling, which is + # the same default the scripts take. + flavors: ["$K8S_FLAVOR", "$DNS_TYPE", "@context:providers.scope-configurations.deployment.deployment_strategy"] # Declared steps only: engine fragments without a trace block (override # plumbing) stay off the wire — the fold shows real observable acts, not # 0ms no-ops. Script-emitted child steps (per-manifest applies) are @@ -60,6 +64,10 @@ steps: - name: build green deployment trace: title: Promote new deployment + description: Scales the new version up to full capacity before the old one is removed. + # ROLLING ONLY: every other strategy takes its instance counts from the + # manifests, so this step has nothing to do and never enters their plan. + flavors: [rolling] group: finalize type: script file: "$SERVICE_PATH/deployment/scale_deployments" @@ -72,6 +80,7 @@ steps: # The same wait, in THIS phase's words: full capacity must be healthy # before the previous version is removed. title: Verify final capacity + description: Confirms every instance of the new version is healthy before the old one goes away. group: finalize configuration: TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS @@ -79,6 +88,7 @@ steps: - name: route traffic trace: title: Configure ingress + description: Rewrites the routing rules so the new version owns the scope's address. group: finalize type: script file: "$SERVICE_PATH/deployment/networking/gateway/route_traffic" @@ -87,6 +97,7 @@ steps: - name: apply traffic trace: title: Apply final routing + description: Puts the rewritten routing live. group: finalize type: script file: "$SERVICE_PATH/apply_templates" @@ -104,6 +115,7 @@ steps: - name: verify_networking_reconciliation trace: title: Verify networking + description: Waits for the load balancer to report the new routing as settled. group: finalize type: script file: "$SERVICE_PATH/deployment/verify_networking_reconciliation" @@ -135,6 +147,7 @@ steps: - name: delete deployment trace: title: Remove previous deployment + description: Deletes the old version's workload, now that nothing routes to it. group: finalize type: script file: "$SERVICE_PATH/apply_templates" diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index 1d76438f..0aa1e0aa 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -4,7 +4,9 @@ configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" trace: title: Roll back deployment - flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] + # The deploy strategy is a flavor dimension: a step only one strategy can + # run never enters another's checklist (see finalize.yaml). + flavors: ["$K8S_FLAVOR", "$DNS_TYPE", "@context:providers.scope-configurations.deployment.deployment_strategy"] # Declared steps only: engine fragments without a trace block (override # plumbing) stay off the wire — the fold shows real observable acts, not # 0ms no-ops. Script-emitted child steps (per-manifest applies) are @@ -62,10 +64,15 @@ steps: file: "$SERVICE_PATH/deployment/scale_deployments" trace: title: Restore previous deployment + description: Scales the previous version back up to full capacity. + # ROLLING ONLY, like finalize's promote step: other strategies never + # scaled anything down, so there is nothing to restore. + flavors: [rolling] group: finalize - name: rollback traffic trace: title: Restore traffic routing + description: Rewrites the routing rules to send traffic back to the previous version. group: finalize type: script file: "$SERVICE_PATH/deployment/networking/gateway/rollback_traffic" @@ -78,6 +85,7 @@ steps: - name: apply traffic trace: title: Apply routing + description: Puts the restored routing live. group: finalize type: script file: "$SERVICE_PATH/apply_templates" @@ -112,6 +120,7 @@ steps: - name: delete deployment trace: title: Remove new deployment + description: Deletes the version being rolled back, now that nothing routes to it. group: finalize type: script file: "$SERVICE_PATH/apply_templates" From c071460ebce3d888fb936d53b8480878df29df95 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sun, 30 Aug 2026 10:30:17 -0300 Subject: [PATCH 42/52] feat(tracing): a stuck wait says why, and what to do about it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The console hints already classify a failing wait — 'The application did not pass its health check at /health-bad. Detected: Startup probe — app responded with HTTP 404 (expected 2xx).' plus a targeted fix — but that never reached the trace, so a reader who opened the phase saw counts and a timeout and had to go to the logs for the reason. The same classifier now feeds the step's explain: WHY on the reason, NEXT on the fix. It rides the LIVE narrative too, not just the give-up — the reader asks 'why?' the moment the meter goes amber, not ten minutes later. --- .../tests/wait_deployment_active.bats | 51 +++++++++++++++++++ k8s/deployment/wait_deployment_active | 32 +++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/k8s/deployment/tests/wait_deployment_active.bats b/k8s/deployment/tests/wait_deployment_active.bats index 52c83d39..b3f6972a 100644 --- a/k8s/deployment/tests/wait_deployment_active.bats +++ b/k8s/deployment/tests/wait_deployment_active.bats @@ -656,6 +656,57 @@ teardown() { assert_contains "$output" "HTTP 502" } +@test "wait_deployment_active: the trace carries WHY it is stuck and what to do, not just the counts" { + # The console hints already classified this failure; the step's explain must say the + # same thing, so a reader who opens the phase gets the reason and the fix without + # going to the logs. + run bash -c " + sleep() { :; } + export -f sleep + + kubectl() { + case \"\$*\" in + \"get deployment\"*\"-o json\"*) + echo '{\"spec\":{\"replicas\":1},\"status\":{\"availableReplicas\":0,\"updatedReplicas\":0,\"readyReplicas\":0}}' + ;; + \"get pods -n test-namespace -l deployment_id=deploy-456 -o jsonpath\"*) + echo 'd-scope-123-deploy-456-abc' + ;; + \"get events\"*\"Pod\"*) + echo '{\"items\":[{\"lastTimestamp\":\"9999-12-31T23:59:59Z\",\"type\":\"Warning\",\"involvedObject\":{\"kind\":\"Pod\",\"name\":\"d-scope-123-deploy-456-abc\"},\"reason\":\"Unhealthy\",\"message\":\"Startup probe failed: HTTP probe failed with statuscode: 404\"}]}' + ;; + \"get events\"*) echo '{\"items\":[]}' ;; + esac + } + export -f kubectl + + np() { echo 'running'; } + export -f np + + # Capture what the step's facets are told, without a tracing backend. The + # timeout's trace block is guarded on np_scope_step_timeout existing, so the + # stubs must cover the whole terminal trio. + np_scope_explain() { echo \"EXPLAIN \$*\"; } + np_scope_error() { echo \"ERROR \$*\"; } + np_scope_step_timeout() { echo \"TIMEOUT \$*\"; } + export -f np_scope_explain np_scope_error np_scope_step_timeout + + export CONTEXT='{\"scope\":{\"name\":\"Stage\",\"capabilities\":{\"health_check\":{\"path\":\"/health-bad\"}}}}' + export SERVICE_PATH='$SERVICE_PATH' K8S_NAMESPACE='$K8S_NAMESPACE' + export SCOPE_ID='$SCOPE_ID' DEPLOYMENT_ID='$DEPLOYMENT_ID' + export TIMEOUT=10 NP_API_KEY='$NP_API_KEY' SKIP_DEPLOYMENT_STATUS_CHECK='false' + bash '$BATS_TEST_DIRNAME/../wait_deployment_active' + " + + [ "$status" -eq 1 ] + # WHY, in the reader's words, naming the configured path and what was detected. + assert_contains "$output" "did not pass its health check at /health-bad" + assert_contains "$output" "Detected: Startup probe" + assert_contains "$output" "HTTP 404" + # …and the actionable next step. + assert_contains "$output" "--next" +} + # ============================================================================= # Latest Timestamp Initialization # ============================================================================= diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index c197c064..2a762f42 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -108,6 +108,24 @@ humanize_k8s_reasons() { echo "$_hr_out" } +# The classified diagnosis, for the TRACE — the same words the console hints print, +# from the same classifier (diagnose_failure in print_failed_deployment_hints, which +# reads ALL_EVENTS): WHY it is stuck ("The application did not pass its health check at +# /health — Detected: Startup probe — app responded with HTTP 404 (expected 2xx)") and +# what to do about it. Sets WAIT_WHY / WAIT_NEXT, both empty when nothing classifies. +# Cheap enough per heartbeat because it only runs once something is actually wrong. +WAIT_WHY="" +WAIT_NEXT="" +classify_wait_failure() { + WAIT_WHY="" + WAIT_NEXT="" + command -v diagnose_failure >/dev/null 2>&1 || return 0 + diagnose_failure >/dev/null 2>&1 || true + WAIT_WHY="${HUMAN_MESSAGE:-}" + WAIT_NEXT="${SUGGESTED_FIX:-}" + return 0 +} + # Report the wait's live narrative onto the trace — the counted io, the # instances-health meter a host renders as pips, and the plain-language # explain with the severity an operator should read it at. Re-emitted per @@ -207,9 +225,14 @@ report_wait_narrative() { if [ "$problem_count" -gt 0 ]; then local problem_words problem_words=$(humanize_k8s_reasons "${problem_reasons:-}") + # The reader asks "why?" the moment the meter goes amber, not only when the + # wait finally gives up — so the diagnosis rides the LIVE narrative too. + classify_wait_failure np_scope_explain --title "$WAIT_TITLE" --severity warn \ --what "Waiting for $ready_now/$desired_now instances to be healthy — $problem_count with ${problem_words:-failing health checks}$detail_clause" \ - --impact "The deployment fails if the instances don't become healthy before the health-check timeout." + ${WAIT_WHY:+--why "$WAIT_WHY"} \ + --impact "The deployment fails if the instances don't become healthy before the health-check timeout." \ + ${WAIT_NEXT:+--next "$WAIT_NEXT"} elif [ "$restart_total" -gt 0 ]; then if [ "$all_healthy" = "true" ]; then np_scope_explain --title "$WAIT_TITLE" --severity warn \ @@ -303,8 +326,13 @@ while true; do timeout_message="$timeout_cause ($timeout_message)" fi timeout_reason_words=$(humanize_k8s_reasons "${timeout_reasons:-}") + # print_failed_deployment_hints (sourced above) has just classified the + # failure; the trace says the same thing the console said. np_scope_explain --title "$WAIT_TITLE" --severity error \ - --what "Gave up with ${ready:-0}/${desired:-0} instances healthy${timeout_reason_words:+ — $timeout_reason_words}${timeout_cause:+: $timeout_cause}" + --what "Gave up with ${ready:-0}/${desired:-0} instances healthy${timeout_reason_words:+ — $timeout_reason_words}${timeout_cause:+: $timeout_cause}" \ + ${HUMAN_MESSAGE:+--why "$HUMAN_MESSAGE"} \ + --impact "The previous version keeps serving traffic; this deployment does not go live." \ + ${SUGGESTED_FIX:+--next "$SUGGESTED_FIX"} np_scope_error "$timeout_message" \ "$(jq -nc --argjson h "${ready:-0}" --argjson l "${launched:-0}" --argjson d "${desired:-0}" --arg reasons "$timeout_reasons" \ '{instances: {healthy: $h, launched: $l, desired: $d}} + (if $reasons != "" then {reasons: ($reasons | split(", "))} else {} end)')" From c419557b5a3d8677751f160137b46904b26296e5 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Sun, 30 Aug 2026 11:32:46 -0300 Subject: [PATCH 43/52] fix(tracing): a multi-word reason survives the reason list whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The phase read '1 with Startup, probe, failing'. The reason list splits on the comma — but IFS was ', ', so it split on whitespace too, and the event sweep's already-human reasons ('Startup probe failing') were shredded into three codes that matched nothing and were echoed verbatim. Split on the comma alone, trimming the space that follows it. Codes still translate and still dedupe after translation. --- .../tests/wait_deployment_active.bats | 20 +++++++++++++++++++ k8s/deployment/wait_deployment_active | 12 +++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/k8s/deployment/tests/wait_deployment_active.bats b/k8s/deployment/tests/wait_deployment_active.bats index b3f6972a..ade0ebb0 100644 --- a/k8s/deployment/tests/wait_deployment_active.bats +++ b/k8s/deployment/tests/wait_deployment_active.bats @@ -656,6 +656,26 @@ teardown() { assert_contains "$output" "HTTP 502" } +@test "wait_deployment_active: a multi-word reason survives the reason list whole" { + # The event sweep contributes reasons already in human words ("Startup probe + # failing"). Splitting the list on whitespace as well as the comma shredded those + # into three, and the phase read "1 with Startup, probe, failing". + source "$BATS_TEST_DIRNAME/../wait_deployment_active" 2>/dev/null || true + run bash -c " + source '$BATS_TEST_DIRNAME/../print_failed_deployment_hints' 2>/dev/null || true + \$(declare -f humanize_k8s_reason humanize_k8s_reasons 2>/dev/null) + true + " + # Exercise the helpers directly out of the script's own source. + eval "$(sed -n '/^humanize_k8s_reason()/,/^}/p;/^humanize_k8s_reasons()/,/^}/p' "$BATS_TEST_DIRNAME/../wait_deployment_active")" + + [ "$(humanize_k8s_reasons 'Startup probe failing')" = "Startup probe failing" ] + [ "$(humanize_k8s_reasons 'OOMKilled, CrashLoopBackOff')" = "out of memory, crashing repeatedly" ] + [ "$(humanize_k8s_reasons 'Startup probe failing, OOMKilled')" = "Startup probe failing, out of memory" ] + # Two codes meaning the same thing still read once. + [ "$(humanize_k8s_reasons 'ImagePullBackOff, ErrImagePull')" = "can't pull the container image" ] +} + @test "wait_deployment_active: the trace carries WHY it is stuck and what to do, not just the counts" { # The console hints already classified this failure; the step's explain must say the # same thing, so a reader who opens the phase gets the reason and the fix without diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index 2a762f42..a26d2213 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -92,12 +92,20 @@ humanize_k8s_reason() { esac } -# A comma-separated reason list, translated and deduped AFTER translation — two +# A COMMA-separated reason list, translated and deduped AFTER translation — two # codes that mean the same thing read once. +# +# Split on the comma ALONE. Not every entry is a one-word kubernetes code: the event +# sweep contributes phrases it has already put into human words ("Startup probe +# failing"), and splitting on whitespace too shredded those into their own "reasons" — +# the phase read "1 with Startup, probe, failing". humanize_k8s_reasons() { local _hr_out="" _hr_word - IFS=', ' read -ra _hr_parts <<< "$1" + IFS=',' read -ra _hr_parts <<< "$1" for _hr_part in "${_hr_parts[@]}"; do + # Trim the space that follows the comma in the joined form. + _hr_part="${_hr_part#"${_hr_part%%[![:space:]]*}"}" + _hr_part="${_hr_part%"${_hr_part##*[![:space:]]}"}" [ -n "$_hr_part" ] || continue _hr_word=$(humanize_k8s_reason "$_hr_part") case ", $_hr_out," in From b2d612ccff514dab997ca3e661110cef13a7405a Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Mon, 31 Aug 2026 09:08:39 -0300 Subject: [PATCH 44/52] fix(k8s): name the CAUSE of a crash loop, not the loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A container that is OOM-killed reports two things: `state.waiting.reason` CrashLoopBackOff — the mechanism that keeps restarting it — and `lastState.terminated.reason` OOMKilled, why it actually died. Both the classifier and the wait narrative read the waiting reason first, so every surface named the symptom: Waiting for 0/1 instances to be healthy — 1 with crashing repeatedly: back-off 20s restarting failed container=application pod=d-253247585-630235960-dd685c5-trn9s_nullplatform(498dc0ab-…) Reason: The container started and crashed repeatedly. Next: Review application logs for startup errors (…, panics). That sends the reader hunting for a bug in their startup path when the answer is the memory limit — and the OOM branch the classifier already carries could never fire. The OOMKilled fact was on the step all along, three panels down under "Restarted → Reason". Now the backoff wrappers (CrashLoopBackOff, BackOff) defer to the termination reason; every other waiting reason IS a cause and still wins, so ImagePullBackOff is unaffected. The line becomes: Waiting for 0/1 instances to be healthy — 1 with out of memory Reason: The container exceeded its memory limit (256Mi) and was terminated. Next: Increase ram_memory for scope 'Stage' or reduce memory usage. The back-off message is dropped from the narrative — it restates the reason and then buries the line in two ids nobody reads — and what remains is capped to a line. Neither is lost: the verbatim message, the CrashLoopBackOff wrapper (as `reason`) and the new `cause` all stay on the io, so an agent still sees the loop, the ids and the untruncated text. --- k8s/deployment/print_failed_deployment_hints | 14 ++- .../tests/wait_deployment_active.bats | 88 +++++++++++++++++++ k8s/deployment/wait_deployment_active | 25 +++++- 3 files changed, 124 insertions(+), 3 deletions(-) diff --git a/k8s/deployment/print_failed_deployment_hints b/k8s/deployment/print_failed_deployment_hints index 33b08ff8..3754ab9b 100644 --- a/k8s/deployment/print_failed_deployment_hints +++ b/k8s/deployment/print_failed_deployment_hints @@ -122,9 +122,21 @@ diagnose_failure() { fi if [[ -n "$pods_json" ]] && echo "$pods_json" | jq -e . >/dev/null 2>&1; then + # CrashLoopBackOff/BackOff name the MECHANISM (it keeps restarting), never + # the cause: the cause is how the container died the last time round. An + # OOM-killed container in a restart loop reports BOTH, and leading with the + # wrapper sends the reader to "review your startup logs" for what is really + # a memory limit. So a backoff wrapper defers to the termination reason. + # Any other waiting reason (ImagePullBackOff, CreateContainerError) IS the + # cause and still wins. FAILURE_REASON=$(echo "$pods_json" | jq -r ' [.items[].status.containerStatuses[]? - | (.state.waiting.reason // .lastState.terminated.reason // empty) + | (.state.waiting.reason // "") as $w + | (.lastState.terminated.reason // "") as $t0 + | (if $t0 == "Completed" then "" else $t0 end) as $t + | (if ($w == "CrashLoopBackOff" or $w == "BackOff") and $t != "" then $t + elif $w != "" then $w + else $t end) ] | map(select(. != "" and . != "Completed")) | group_by(.) | max_by(length) | .[0] // empty' 2>/dev/null) diff --git a/k8s/deployment/tests/wait_deployment_active.bats b/k8s/deployment/tests/wait_deployment_active.bats index ade0ebb0..44304d51 100644 --- a/k8s/deployment/tests/wait_deployment_active.bats +++ b/k8s/deployment/tests/wait_deployment_active.bats @@ -928,3 +928,91 @@ teardown() { assert_contains "$output" "Could not report instance counts" assert_contains "$output" "✅ All pods in deployment 'd-scope-123-deploy-456' are available and ready!" } + +@test "wait_deployment_active: an OOM kill in a restart loop is reported as out of memory, not as the loop" { + # CrashLoopBackOff is the MECHANISM; OOMKilled is why the container died. Reading + # the wrapper sent the operator to "review your startup logs" for what was really + # a memory limit, so the backoff wrappers defer to the termination reason. + cat > "$BATS_TMPDIR/oom-pods.json" <<'JSON' +{"items":[{"metadata":{"name":"d-1-2-abc"}, +"status":{"containerStatuses":[{"name":"application", +"state":{"waiting":{"reason":"CrashLoopBackOff","message":"back-off 20s restarting failed container=application pod=d-1-2-abc_nullplatform(498dc0ab)"}}, +"lastState":{"terminated":{"reason":"OOMKilled","exitCode":137}}, +"restartCount":3}]}}]} +JSON + + # The narrative's reason list: the cause wins. + run jq -r '[.items[] | .status.containerStatuses[]? + | .state.waiting.reason as $w + | (.lastState.terminated.reason // "") as $t0 + | (if $t0 == "Completed" then "" else $t0 end) as $t + | (if ($w == "CrashLoopBackOff" or $w == "BackOff") then $t else $w end)] + | unique | join(", ")' "$BATS_TMPDIR/oom-pods.json" + [ "$status" -eq 0 ] + [ "$output" = "OOMKilled" ] + + eval "$(sed -n '/^humanize_k8s_reason()/,/^}/p;/^humanize_k8s_reasons()/,/^}/p' "$BATS_TEST_DIRNAME/../wait_deployment_active")" + [ "$(humanize_k8s_reasons "$output")" = "out of memory" ] +} + +@test "wait_deployment_active: the mechanism stays on the io even though the narrative names the cause" { + # An agent reading the step must still see that it was looping — the wrapper is + # kept as `reason`, the cause added as `cause`. Only the human line is narrowed. + cat > "$BATS_TMPDIR/oom-pods.json" <<'JSON' +{"items":[{"metadata":{"name":"d-1-2-abc"}, +"status":{"containerStatuses":[{"name":"application", +"state":{"waiting":{"reason":"CrashLoopBackOff","message":"back-off 20s restarting failed container=application"}}, +"lastState":{"terminated":{"reason":"OOMKilled"}}, +"restartCount":3}]}}]} +JSON + run jq -c '[.items[] | .metadata.name as $pod | .status.containerStatuses[]? + | .state.waiting.reason as $w + | (.lastState.terminated.reason // "") as $t0 + | (if $t0 == "Completed" then "" else $t0 end) as $t + | {pod: $pod, reason: $w, + cause: (if ($w == "CrashLoopBackOff" or $w == "BackOff") then $t else "" end)}]' \ + "$BATS_TMPDIR/oom-pods.json" + [ "$status" -eq 0 ] + echo "$output" | grep -q '"reason":"CrashLoopBackOff"' + echo "$output" | grep -q '"cause":"OOMKilled"' +} + +@test "wait_deployment_active: a real cause still wins over the termination reason" { + # Only the backoff wrappers defer. ImagePullBackOff IS the cause and must not be + # replaced by whatever the container happened to exit with last time. + cat > "$BATS_TMPDIR/pull-pods.json" <<'JSON' +{"items":[{"metadata":{"name":"d-1-2-abc"}, +"status":{"containerStatuses":[{"name":"application", +"state":{"waiting":{"reason":"ImagePullBackOff","message":"manifest unknown"}}, +"lastState":{"terminated":{"reason":"Error"}}}]}}]} +JSON + run jq -r '[.items[] | .status.containerStatuses[]? + | .state.waiting.reason as $w + | (.lastState.terminated.reason // "") as $t + | (if ($w == "CrashLoopBackOff" or $w == "BackOff") and $t != "" then $t else $w end)] + | unique | join(", ")' "$BATS_TMPDIR/pull-pods.json" + [ "$status" -eq 0 ] + [ "$output" = "ImagePullBackOff" ] +} + +@test "wait_deployment_active: the back-off boilerplate never reaches the phase line" { + # "back-off 20s restarting failed container=… pod=…(uuid)" repeats the reason and + # then runs the line off the page with two ids. It is dropped from the narrative; + # a registry's real error is information and is kept (bounded). + detail_of() { + local d="$1" + case "$d" in + "back-off "*"restarting failed container="*) d="" ;; + esac + if [ ${#d} -gt 140 ]; then d="${d:0:137}..."; fi + printf '%s' "$d" + } + + [ -z "$(detail_of 'back-off 20s restarting failed container=application pod=d-253247585-630235960_nullplatform(498dc0ab-d1f9)')" ] + [ "$(detail_of 'manifest unknown: manifest tagged v9 not found')" = 'manifest unknown: manifest tagged v9 not found' ] + + long=$(printf 'x%.0s' $(seq 1 200)) + capped=$(detail_of "$long") + [ "${#capped}" -eq 140 ] + case "$capped" in *"...") ;; *) return 1 ;; esac +} diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index a26d2213..1e7b8c40 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -166,12 +166,18 @@ report_wait_narrative() { | select(.state.waiting.reason != null and .state.waiting.reason != "ContainerCreating" and .state.waiting.reason != "PodInitializing") - | {pod: $pod, reason: .state.waiting.reason, + | .state.waiting.reason as $w + | (.lastState.terminated.reason // "") as $t0 + | (if $t0 == "Completed" then "" else $t0 end) as $t + | {pod: $pod, reason: $w, + cause: (if ($w == "CrashLoopBackOff" or $w == "BackOff") then $t else "" end), message: ((.state.waiting.message // "") | .[0:300])} | with_entries(select(.value != ""))]' 2>/dev/null) || problems="[]" [ -n "$problems" ] || problems="[]" problem_count=$(echo "$problems" | jq 'length' 2>/dev/null) || problem_count=0 - problem_reasons=$(echo "$problems" | jq -r '[.[].reason] | unique | join(", ")' 2>/dev/null) || problem_reasons="" + # The narrative names the CAUSE (out of memory), not the wrapper that kept + # restarting it; `reason` stays on the io so an agent still sees the loop. + problem_reasons=$(echo "$problems" | jq -r '[.[] | (.cause // .reason)] | unique | join(", ")' 2>/dev/null) || problem_reasons="" # Kubernetes' own words for the FIRST problem — the real error, verbatim. real_detail=$(echo "$problems" | jq -r '[.[].message // empty] | first // ""' 2>/dev/null \ | tr '\n' ' ' | sed 's/[[:space:]]*$//') || real_detail="" @@ -199,6 +205,21 @@ report_wait_narrative() { fi fi + # A back-off message is pure MECHANISM: "back-off 20s restarting failed + # container=application pod=d-253247585-…_nullplatform(498dc0ab-…)" repeats the + # reason we just named and then buries the line in two ids no reader parses. + # It is the whole reason the phase line ran off the page. Drop it from the + # narrative — the cause is on `why`, and the verbatim message is still on the + # io for an agent. Everything else (a registry's real error) is information. + case "$WAIT_REAL_DETAIL" in + "back-off "*"restarting failed container="*) WAIT_REAL_DETAIL="" ;; + esac + # What is left is read at a glance, so keep it to about one line; the io + # carries it untruncated. + if [ ${#WAIT_REAL_DETAIL} -gt 140 ]; then + WAIT_REAL_DETAIL="${WAIT_REAL_DETAIL:0:137}..." + fi + # Realtime without noise: emit only when the SITUATION changes (counts, a # new problem, a recovery) — the loop calls this every poll. local snapshot="$ready_now/$desired_now/$launched_now/$problem_count/$problem_reasons/$restart_total/$all_healthy" From 4e26364984730da8485b8cb17571698d2a54c2cf Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Mon, 31 Aug 2026 09:20:49 -0300 Subject: [PATCH 45/52] fix(k8s): every crash cause explains itself, not just the OOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auditing the other warn/failure paths after the OOM fix turned up three gaps, one of them introduced by that fix: - `Error` had no branch. Deferring the backoff wrapper to the termination reason sends the ORDINARY crash loop — a panic, a bad config, a missing dependency, all of which terminate as `Error` — to the nameless default: "Pods are failing with reason: Error", with no suggested fix at all. That is worse than the generic wording it replaced. `Error` now carries the crash-loop advice plus its exit code, the first thing anyone greps for. - The restart-only narrative never diagnosed. A container that crashed and is running again sits between waiting states, so nothing classified it, and the branch said only THAT it restarted — an amber phase with no cause and no next step. It calls the classifier like every other surface now. - Evicted, Unschedulable and DeadlineExceeded were in the humanize table but had no branch, so they reached the reader named and unexplained. The unknown-reason default keeps its EMPTY suggested fix: that emptiness is the signal that hands the reader the full generic troubleshooting checklist, which is richer than any line this branch could invent for a code it does not know. Filling it in swapped that checklist for a one-liner. --- k8s/deployment/print_failed_deployment_hints | 24 ++++++++ .../tests/print_failed_deployment_hints.bats | 61 +++++++++++++++++++ k8s/deployment/wait_deployment_active | 14 ++++- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/k8s/deployment/print_failed_deployment_hints b/k8s/deployment/print_failed_deployment_hints index 3754ab9b..7fdd36f9 100644 --- a/k8s/deployment/print_failed_deployment_hints +++ b/k8s/deployment/print_failed_deployment_hints @@ -193,6 +193,18 @@ diagnose_failure() { CrashLoopBackOff|BackOff) HUMAN_MESSAGE="The container started and crashed repeatedly." SUGGESTED_FIX="Review application logs for startup errors (failed dependencies, bad config, panics)." ;; + Error|ContainerStatusUnknown) + # The ordinary crash-loop cause: the app ran and exited non-zero. Now + # that the backoff wrapper defers to the termination reason, this is + # where a panic or a bad config lands, so it needs the crash-loop + # wording the wrapper used to give it — plus the exit code, which is + # the first thing an operator greps their logs for. + if [[ -n "$FAILURE_EXIT_CODE" ]]; then + HUMAN_MESSAGE="The container started and exited with code ${FAILURE_EXIT_CODE}." + else + HUMAN_MESSAGE="The container started and crashed repeatedly." + fi + SUGGESTED_FIX="Review application logs for startup errors (failed dependencies, bad config, panics)." ;; OOMKilled) if [[ -n "$req_memory" ]]; then HUMAN_MESSAGE="The container exceeded its memory limit (${req_memory}Mi) and was terminated." @@ -242,11 +254,23 @@ diagnose_failure() { FailedCreate|FailedCreatePodSandBox) HUMAN_MESSAGE="Kubernetes could not create the pod sandbox." SUGGESTED_FIX="Check node health, CNI configuration, and pod security policies." ;; + Evicted) + HUMAN_MESSAGE="The pod was evicted from its node." + SUGGESTED_FIX="The node ran out of memory or disk. Lower the scope's resource requests, or free capacity on the cluster." ;; + Unschedulable) + HUMAN_MESSAGE="No node can accept the pod." + SUGGESTED_FIX="Reduce requested resources, free cluster capacity, or review nodeSelector/affinity rules." ;; + DeadlineExceeded) + HUMAN_MESSAGE="The pod ran past its active deadline and was stopped." + SUGGESTED_FIX="Raise activeDeadlineSeconds, or find why the workload takes longer than its deadline allows." ;; "") HUMAN_MESSAGE="" SUGGESTED_FIX="" ;; *) HUMAN_MESSAGE="Pods are failing with reason: $FAILURE_REASON" + # Deliberately empty: an empty fix is the signal that hands the reader + # the full generic troubleshooting checklist below, which is richer than + # any one line this branch could invent for a reason it does not know. SUGGESTED_FIX="" ;; esac } diff --git a/k8s/deployment/tests/print_failed_deployment_hints.bats b/k8s/deployment/tests/print_failed_deployment_hints.bats index aae55005..9a856ea6 100644 --- a/k8s/deployment/tests/print_failed_deployment_hints.bats +++ b/k8s/deployment/tests/print_failed_deployment_hints.bats @@ -488,3 +488,64 @@ assert_not_contains() { [ "$status" -eq 0 ] assert_contains "$output" "📊 Progress at failure: 1/3 ready, 2/3 available" } + +@test "print_failed_deployment_hints: an OOM kill inside a crash loop is diagnosed as the OOM" { + # CrashLoopBackOff is the restart MECHANISM and OOMKilled is the cause; reading + # the wrapper first sent the operator to their startup logs for a memory limit. + export K8S_NAMESPACE="ns" DEPLOYMENT_ID="d1" + kubectl() { + case "$*" in + "get pods"*) + echo '{"items":[{"status":{"containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CrashLoopBackOff","message":"back-off 20s restarting failed container=app pod=d-1_ns(abc)"}},"lastState":{"terminated":{"reason":"OOMKilled","exitCode":137}}}]}}]}' + ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" + + [ "$status" -eq 0 ] + assert_contains "$output" "exceeded its memory limit" + assert_contains "$output" "Increase ram_memory" + assert_not_contains "$output" "started and crashed repeatedly" +} + +@test "print_failed_deployment_hints: a plain non-zero exit in a crash loop keeps its startup-log advice" { + # Deferring to the termination reason must NOT strand the ordinary crash loop: + # `Error` had no branch of its own, so it would have fallen through to the + # nameless default with no suggested fix at all. + export K8S_NAMESPACE="ns" DEPLOYMENT_ID="d1" + kubectl() { + case "$*" in + "get pods"*) + echo '{"items":[{"status":{"containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CrashLoopBackOff","message":"back-off 20s restarting failed container=app"}},"lastState":{"terminated":{"reason":"Error","exitCode":1}}}]}}]}' + ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" + + [ "$status" -eq 0 ] + assert_contains "$output" "exited with code 1" + assert_contains "$output" "Review application logs for startup errors" + assert_not_contains "$output" "Pods are failing with reason: Error" +} + +@test "print_failed_deployment_hints: an evicted pod says why and what to do" { + export K8S_NAMESPACE="ns" DEPLOYMENT_ID="d1" + kubectl() { + case "$*" in + "get pods"*) + echo '{"items":[{"status":{"containerStatuses":[{"name":"app","state":{"waiting":{"reason":"Evicted"}}}]}}]}' + ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" + + [ "$status" -eq 0 ] + assert_contains "$output" "evicted from its node" + assert_contains "$output" "ran out of memory or disk" +} diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index 1e7b8c40..1f6d2d0a 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -263,14 +263,24 @@ report_wait_narrative() { --impact "The deployment fails if the instances don't become healthy before the health-check timeout." \ ${WAIT_NEXT:+--next "$WAIT_NEXT"} elif [ "$restart_total" -gt 0 ]; then + # A container that crashed and is running again is between waiting states, + # so nothing above classified it — and this branch used to say only THAT it + # restarted, never why or what to do. A restart has a cause (the + # termination reason is right there in `restart_reasons`), so ask for it + # here too rather than leaving the amber phase mute. + classify_wait_failure if [ "$all_healthy" = "true" ]; then np_scope_explain --title "$WAIT_TITLE" --severity warn \ --what "All $desired_now instances healthy — after $restart_total $restarts_label$restart_clause" \ - --impact "The instances crashed on the way here; the last crash output is attached to this step." + ${WAIT_WHY:+--why "$WAIT_WHY"} \ + --impact "The instances crashed on the way here; the last crash output is attached to this step." \ + ${WAIT_NEXT:+--next "$WAIT_NEXT"} else np_scope_explain --title "$WAIT_TITLE" --severity warn \ --what "Waiting for $ready_now/$desired_now instances to be healthy — $restart_total $restarts_label so far$restart_clause$detail_clause" \ - --impact "The last crash output is attached to this step." + ${WAIT_WHY:+--why "$WAIT_WHY"} \ + --impact "The last crash output is attached to this step." \ + ${WAIT_NEXT:+--next "$WAIT_NEXT"} fi elif [ "$all_healthy" = "true" ]; then np_scope_explain --title "$WAIT_TITLE" --what "All $desired_now instances healthy" From 8f45ad88b686a3dfba481b31d8ff7bc6011b6226 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Mon, 31 Aug 2026 09:51:57 -0300 Subject: [PATCH 46/52] fix(k8s): suggest the setting the console shows, not the field key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ram_memory` is the capability KEY. The scope form calls it "RAM Memory", so "Increase ram_memory for scope 'Stage'" sent the reader looking for a field that is not on the page — the same leak the OOM diagnosis itself was: the right answer, spoken in a vocabulary the reader has no way to act on. Three strings named an internal or foreign identifier: - ram_memory -> "the RAM Memory of scope 'X'". - initialDelaySeconds/timeoutSeconds are the KUBERNETES spellings; the scope's own settings are `health_check.initial_delay_seconds` / `timeout_seconds`, shown as Initial Delay and Timeout. - activeDeadlineSeconds is not a scope setting at all, so it named a field the console does not have. Reworded to say the thing instead of the field. Genuine Kubernetes objects (PVC, configmap, nodeSelector/affinity) keep their Kubernetes names: those are what they are, and renaming them would make them unfindable in the cluster the reader has to go look at. Three tests asserted the old wording; two of them encoded a field key as the user-facing contract, which is what let this ship. --- k8s/deployment/print_failed_deployment_hints | 6 +++--- .../tests/print_failed_deployment_hints.bats | 12 ++++++++---- k8s/deployment/tests/wait_deployment_active.bats | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/k8s/deployment/print_failed_deployment_hints b/k8s/deployment/print_failed_deployment_hints index 7fdd36f9..30d594d6 100644 --- a/k8s/deployment/print_failed_deployment_hints +++ b/k8s/deployment/print_failed_deployment_hints @@ -211,7 +211,7 @@ diagnose_failure() { else HUMAN_MESSAGE="The container exceeded its memory limit and was terminated." fi - SUGGESTED_FIX="Increase ram_memory for scope '$scope_name' or reduce application memory usage." ;; + SUGGESTED_FIX="Raise the RAM Memory of scope '$scope_name', or reduce how much memory the application uses." ;; CreateContainerConfigError) HUMAN_MESSAGE="The container configuration is invalid." SUGGESTED_FIX="Check for missing secrets or configmaps referenced by the deployment." ;; @@ -247,7 +247,7 @@ diagnose_failure() { elif [[ "$UNHEALTHY_MESSAGE" =~ statuscode:[[:space:]]*([0-9]+) ]]; then SUGGESTED_FIX="The app responded with HTTP ${BASH_REMATCH[1]} on $health_check_path — inspect application logs for startup errors; the process is running but $health_check_path is not returning 2xx." elif [[ "$UNHEALTHY_MESSAGE" == *"context deadline exceeded"* || "$UNHEALTHY_MESSAGE" == *"Client.Timeout"* || "$UNHEALTHY_MESSAGE" == *"i/o timeout"* ]]; then - SUGGESTED_FIX="The probe timed out — the app may be slow to start or $health_check_path is blocking. Consider increasing startup probe initialDelaySeconds/timeoutSeconds, or making $health_check_path lighter." + SUGGESTED_FIX="The probe timed out — the app may be slow to start, or $health_check_path is blocking. Raise the health check's Initial Delay or Timeout on scope '$scope_name', or make $health_check_path lighter." else SUGGESTED_FIX="Ensure the app listens on port 8080 and returns 2xx on $health_check_path within the readiness window." fi ;; @@ -262,7 +262,7 @@ diagnose_failure() { SUGGESTED_FIX="Reduce requested resources, free cluster capacity, or review nodeSelector/affinity rules." ;; DeadlineExceeded) HUMAN_MESSAGE="The pod ran past its active deadline and was stopped." - SUGGESTED_FIX="Raise activeDeadlineSeconds, or find why the workload takes longer than its deadline allows." ;; + SUGGESTED_FIX="Find why the workload runs longer than the deadline it was given, or allow it more time." ;; "") HUMAN_MESSAGE="" SUGGESTED_FIX="" ;; diff --git a/k8s/deployment/tests/print_failed_deployment_hints.bats b/k8s/deployment/tests/print_failed_deployment_hints.bats index 9a856ea6..7d68f0ef 100644 --- a/k8s/deployment/tests/print_failed_deployment_hints.bats +++ b/k8s/deployment/tests/print_failed_deployment_hints.bats @@ -83,7 +83,10 @@ assert_not_contains() { assert_contains "$output" "📋 Reason: The container exceeded its memory limit (512Mi)" assert_contains "$output" "📋 Detected: OOMKilled on container app (exit 137)" assert_contains "$output" "📋 Details: out of memory" - assert_contains "$output" "💡 Suggested fix: Increase ram_memory for scope 'my-app'" + # The console calls this setting "RAM Memory"; `ram_memory` is the capability KEY and + # sends the reader looking for a field the scope form does not show. + assert_contains "$output" "💡 Suggested fix: Raise the RAM Memory of scope 'my-app'" + assert_not_contains "$output" "ram_memory" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -301,8 +304,9 @@ assert_not_contains() { [ "$status" -eq 0 ] assert_contains "$output" "Detected: Startup probe" assert_contains "$output" "timed out" - # SUGGESTED_FIX mentions timing knobs - assert_contains "$output" "initialDelaySeconds" + # SUGGESTED_FIX names the timing knobs the CONSOLE shows, not their kubernetes spellings. + assert_contains "$output" "Initial Delay or Timeout" + assert_not_contains "$output" "initialDelaySeconds" } @test "print_failed_deployment_hints: falls back to raw Unhealthy message when translation is impossible" { @@ -506,7 +510,7 @@ assert_not_contains() { [ "$status" -eq 0 ] assert_contains "$output" "exceeded its memory limit" - assert_contains "$output" "Increase ram_memory" + assert_contains "$output" "Raise the RAM Memory" assert_not_contains "$output" "started and crashed repeatedly" } diff --git a/k8s/deployment/tests/wait_deployment_active.bats b/k8s/deployment/tests/wait_deployment_active.bats index 44304d51..ce449930 100644 --- a/k8s/deployment/tests/wait_deployment_active.bats +++ b/k8s/deployment/tests/wait_deployment_active.bats @@ -149,7 +149,7 @@ teardown() { # The hint script must read pod state and surface the user-friendly reason assert_contains "$output" "📋 Reason: The container exceeded its memory limit" assert_contains "$output" "📋 Detected: OOMKilled on container app (exit 137)" - assert_contains "$output" "💡 Suggested fix: Increase ram_memory for scope 'my-app'" + assert_contains "$output" "💡 Suggested fix: Raise the RAM Memory of scope 'my-app'" } # ============================================================================= From 133403e9a8646c2c8a27a7715ae3421715a32154 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Mon, 31 Aug 2026 11:54:45 -0300 Subject: [PATCH 47/52] feat(k8s): every workflow says what a human calls it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven of the sixteen workflows declared no `trace:` block at all, so their runs reached a consumer with every naming rung empty: run_id : workflow-diagnose-395c6e18-5574-4b54-b39e-735541b16aa2 explain : null <- no declared title job : null key : null The title contract then lands on the run id, and a consumer that will not print a uuid substitutes its own neutral word: the deployment page showed the failure diagnostics as "Activity · 21 steps · all succeeded", which names nothing the reader can act on. The name was in `labels.workflow: "diagnose"` the whole time — and that is exactly the label a consumer must NOT title from: it is `filepath.Base` of this file, private to this engine's layout, and the next producer's `workflow` label means something else. So the fix is to say it, not to have it read. Titled: the deployment diagnose and kill_instance workflows, and the scope diagnose, pause/resume-autoscaling, restart-pods and set-desired-instance-count workflows. Every workflow in the package now names itself. --- k8s/deployment/workflows/diagnose.yaml | 6 ++++++ k8s/deployment/workflows/kill_instance.yaml | 6 ++++++ k8s/scope/workflows/diagnose.yaml | 6 ++++++ k8s/scope/workflows/pause-autoscaling.yaml | 6 ++++++ k8s/scope/workflows/restart-pods.yaml | 6 ++++++ k8s/scope/workflows/resume-autoscaling.yaml | 6 ++++++ k8s/scope/workflows/set-desired-instance-count.yaml | 6 ++++++ 7 files changed, 42 insertions(+) diff --git a/k8s/deployment/workflows/diagnose.yaml b/k8s/deployment/workflows/diagnose.yaml index 0765ad4b..de0606a4 100644 --- a/k8s/deployment/workflows/diagnose.yaml +++ b/k8s/deployment/workflows/diagnose.yaml @@ -1,3 +1,9 @@ +trace: + # What a HUMAN calls this execution. It lands on the run's explain.title — the + # first rung of every consumer's naming contract. Without it a consumer has only + # the run id, and prints its own neutral word: this run rendered as + # "Activity · N steps", which names nothing the reader can act on. + title: Diagnose the failed deployment continue_on_error: true include: - "$SERVICE_PATH/values.yaml" diff --git a/k8s/deployment/workflows/kill_instance.yaml b/k8s/deployment/workflows/kill_instance.yaml index 5a8d2676..a7321054 100644 --- a/k8s/deployment/workflows/kill_instance.yaml +++ b/k8s/deployment/workflows/kill_instance.yaml @@ -1,3 +1,9 @@ +trace: + # What a HUMAN calls this execution. It lands on the run's explain.title — the + # first rung of every consumer's naming contract. Without it a consumer has only + # the run id, and prints its own neutral word: this run rendered as + # "Activity · N steps", which names nothing the reader can act on. + title: Kill an instance include: - "$SERVICE_PATH/values.yaml" steps: diff --git a/k8s/scope/workflows/diagnose.yaml b/k8s/scope/workflows/diagnose.yaml index 5ec03834..190e9aef 100644 --- a/k8s/scope/workflows/diagnose.yaml +++ b/k8s/scope/workflows/diagnose.yaml @@ -1,3 +1,9 @@ +trace: + # What a HUMAN calls this execution. It lands on the run's explain.title — the + # first rung of every consumer's naming contract. Without it a consumer has only + # the run id, and prints its own neutral word: this run rendered as + # "Activity · N steps", which names nothing the reader can act on. + title: Diagnose the scope continue_on_error: true include: - "$SERVICE_PATH/values.yaml" diff --git a/k8s/scope/workflows/pause-autoscaling.yaml b/k8s/scope/workflows/pause-autoscaling.yaml index fce1a9da..74a5cee5 100644 --- a/k8s/scope/workflows/pause-autoscaling.yaml +++ b/k8s/scope/workflows/pause-autoscaling.yaml @@ -1,3 +1,9 @@ +trace: + # What a HUMAN calls this execution. It lands on the run's explain.title — the + # first rung of every consumer's naming contract. Without it a consumer has only + # the run id, and prints its own neutral word: this run rendered as + # "Activity · N steps", which names nothing the reader can act on. + title: Pause autoscaling include: - "$SERVICE_PATH/values.yaml" steps: diff --git a/k8s/scope/workflows/restart-pods.yaml b/k8s/scope/workflows/restart-pods.yaml index 3b024367..fb37fcb8 100644 --- a/k8s/scope/workflows/restart-pods.yaml +++ b/k8s/scope/workflows/restart-pods.yaml @@ -1,3 +1,9 @@ +trace: + # What a HUMAN calls this execution. It lands on the run's explain.title — the + # first rung of every consumer's naming contract. Without it a consumer has only + # the run id, and prints its own neutral word: this run rendered as + # "Activity · N steps", which names nothing the reader can act on. + title: Restart the instances include: - "$SERVICE_PATH/values.yaml" steps: diff --git a/k8s/scope/workflows/resume-autoscaling.yaml b/k8s/scope/workflows/resume-autoscaling.yaml index c6cb36f0..dcce1816 100644 --- a/k8s/scope/workflows/resume-autoscaling.yaml +++ b/k8s/scope/workflows/resume-autoscaling.yaml @@ -1,3 +1,9 @@ +trace: + # What a HUMAN calls this execution. It lands on the run's explain.title — the + # first rung of every consumer's naming contract. Without it a consumer has only + # the run id, and prints its own neutral word: this run rendered as + # "Activity · N steps", which names nothing the reader can act on. + title: Resume autoscaling include: - "$SERVICE_PATH/values.yaml" steps: diff --git a/k8s/scope/workflows/set-desired-instance-count.yaml b/k8s/scope/workflows/set-desired-instance-count.yaml index 12759b4b..99b734c2 100644 --- a/k8s/scope/workflows/set-desired-instance-count.yaml +++ b/k8s/scope/workflows/set-desired-instance-count.yaml @@ -1,3 +1,9 @@ +trace: + # What a HUMAN calls this execution. It lands on the run's explain.title — the + # first rung of every consumer's naming contract. Without it a consumer has only + # the run id, and prints its own neutral word: this run rendered as + # "Activity · N steps", which names nothing the reader can act on. + title: Set the instance count include: - "$SERVICE_PATH/values.yaml" steps: From fa88bcde216a8b296ae7c35e05a791a88c5d593c Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Mon, 31 Aug 2026 12:33:10 -0300 Subject: [PATCH 48/52] fix(k8s): a diagnostic check says what it FOUND on its own step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every check already decides a verdict — 10 call sites report `failed`, 3 `warning` — and builds real evidence: a summary, affected pods, details and suggested actions. All of it went into the results file for the diagnose API and none of it onto the trace. On the wire every one of the 21 check steps read: completed sev=- Memory-Limits err=- out=0 completed sev=- Container-Crash-Detection err=- out=0 So the page showed "21 steps · all succeeded" directly under a deployment that had just died of an OOM — including the two checks that found it. `update_check_result` is the one funnel every check goes through, so the verdict is mirrored there: the finding becomes the step's `explain.what`, its first suggested action becomes `next`, and the evidence rides along as an output so the dialog can show the affected pods. The step's STATUS is deliberately left alone. A check that finds a problem did its job; failing the step would conflate "this check broke" with "this check found something". The finding rides `explain.severity` — error for a failed check, warn for a warning — the same channel every other producer surface uses. A passing or skipped check stays silent: 21 lines of green is how nothing gets read. --- k8s/diagnose/tests/diagnose_utils.bats | 57 ++++++++++++++++++++++++++ k8s/diagnose/utils/diagnose_utils | 37 +++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/k8s/diagnose/tests/diagnose_utils.bats b/k8s/diagnose/tests/diagnose_utils.bats index bb218b81..9cb6e00e 100644 --- a/k8s/diagnose/tests/diagnose_utils.bats +++ b/k8s/diagnose/tests/diagnose_utils.bats @@ -419,3 +419,60 @@ strip_ansi() { local clean=$(strip_ansi "$output") assert_contains "$clean" "⚠ No JSON result files found in $NP_OUTPUT_DIR" } + +# --- the verdict reaches the TRACE, not only the results file ----------------- + +# Stand in for the logging shims so the test can see exactly what the check said. +_stub_trace() { + np_scope_explain() { echo "EXPLAIN $*" >> "$TRACE_LOG"; } + np_scope_output() { echo "OUTPUT $1" >> "$TRACE_LOG"; } + export -f np_scope_explain np_scope_output + export TRACE_LOG="$(mktemp)" +} + +@test "diagnose_utils: a failed check states its finding on the step, with what to do" { + _stub_trace + evidence=$(evidence_json "1 of 1 pod(s) had OOMKilled containers" "critical" '["pod-a"]' '{}' \ + '["Increase memory limits or optimize application memory usage"]') + + update_check_result --status "failed" --evidence "$evidence" + + grep -q -- "--severity error" "$TRACE_LOG" + grep -q -- "OOMKilled containers" "$TRACE_LOG" + grep -q -- "--next Increase memory limits" "$TRACE_LOG" + # The evidence rides along so the dialog can show the affected pods. + grep -q "^OUTPUT check_evidence" "$TRACE_LOG" + rm -f "$TRACE_LOG" +} + +@test "diagnose_utils: a warning check reads as a warning, not an error" { + _stub_trace + evidence=$(evidence_json "2 pod(s) restarted recently" "warning" '[]' '{}' '[]') + + update_check_result --status "warning" --evidence "$evidence" + + grep -q -- "--severity warn" "$TRACE_LOG" + ! grep -q -- "--severity error" "$TRACE_LOG" + rm -f "$TRACE_LOG" +} + +@test "diagnose_utils: a passing check says nothing — a clean run is not 21 lines of green" { + _stub_trace + evidence=$(evidence_json "No OOMKilled containers detected in 1 pod(s)" "info" '[]' '{}' '[]') + + update_check_result --status "success" --evidence "$evidence" + + [ ! -s "$TRACE_LOG" ] + rm -f "$TRACE_LOG" +} + +@test "diagnose_utils: the results file is still written when the workflow is untraced" { + # No np_scope_* in scope: the mirror is a no-op and the check still records. + evidence=$(evidence_json "1 of 1 pod(s) had OOMKilled containers" "critical" '["pod-a"]' '{}' '[]') + + run update_check_result --status "failed" --evidence "$evidence" + + [ "$status" -eq 0 ] + assert_equal "$(jq -r '.status' "$SCRIPT_OUTPUT_FILE")" "failed" + assert_contains "$(jq -r '.evidence.summary' "$SCRIPT_OUTPUT_FILE")" "OOMKilled" +} diff --git a/k8s/diagnose/utils/diagnose_utils b/k8s/diagnose/utils/diagnose_utils index 94bac3b8..92fbc9f8 100644 --- a/k8s/diagnose/utils/diagnose_utils +++ b/k8s/diagnose/utils/diagnose_utils @@ -321,6 +321,43 @@ update_check_result() { fi mv "$tmpfile" "$output_file" + + # ...and say the same thing on the TRACE. A check's verdict lived only in this + # file, read by the diagnose API — so a reader on the deployment page saw 21 + # steps that exited 0 and a summary reading "all succeeded", while the check + # that had just found the OOM sat there green. The evidence is already built by + # the caller (summary, severity, recommendations); it only needed saying. + # + # The step's STATUS stays whatever the script did: a check that FOUND a problem + # did its job, and marking it failed would conflate "this check broke" with + # "this check found something". The FINDING rides `explain.severity`, which is + # the same channel every other producer surface uses for attention. + _np_trace_check_result "$status_lower" "$evidence" +} + +# Mirror a check's verdict onto its traced step: the finding as the step's +# narrative, its first recommendation as what to do, and the whole evidence as an +# output so the dialog can show the affected pods and details. A no-op when the +# workflow is untraced (the logging shims return 0 without a trace). +_np_trace_check_result() { + local _cr_status="$1" _cr_evidence="$2" _cr_severity="" _cr_summary="" _cr_action="" + command -v np_scope_explain >/dev/null 2>&1 || return 0 + + case "$_cr_status" in + failed) _cr_severity="error" ;; + warning) _cr_severity="warn" ;; + # A passing or skipped check is not news: it would turn a clean diagnosis + # into 21 lines of green noise, which is how nothing gets read. + *) return 0 ;; + esac + + _cr_summary=$(printf '%s' "$_cr_evidence" | jq -r '.summary // empty' 2>/dev/null) || _cr_summary="" + _cr_action=$(printf '%s' "$_cr_evidence" | jq -r '(.suggested_actions // [])[0] // empty' 2>/dev/null) || _cr_action="" + [ -n "$_cr_summary" ] || return 0 + + np_scope_explain --severity "$_cr_severity" --what "$_cr_summary" ${_cr_action:+--next "$_cr_action"} + np_scope_output check_evidence "$_cr_evidence" 2>/dev/null || true + return 0 } notify_results() { From aa3c825a0b40eabec7230dc743fbc00a3e6ce379 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Mon, 31 Aug 2026 18:26:27 -0300 Subject: [PATCH 49/52] test(k8s): assert the full hint lines, not fragments Review feedback: these messages are the main output the deploying dev sees, so a test should fail when one of them changes. Asserting fragments ("Detected: Startup probe", "not yet listening") let a message be reworded without a single test noticing. Every pod-diagnostic branch now asserts the complete emitted lines -- the reason, the detected line, the details, the recent-warnings block and the suggested fix -- so breaking any one of them is a red test. Verified by reordering four words inside one message: test 11 fails, where the old fragment assert passed. Also drops the explanatory prose comments from the file; the repo is public and the asserts now say what the tests were explaining. --- .../tests/print_failed_deployment_hints.bats | 68 ++++++++----------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/k8s/deployment/tests/print_failed_deployment_hints.bats b/k8s/deployment/tests/print_failed_deployment_hints.bats index 7d68f0ef..aea603ce 100644 --- a/k8s/deployment/tests/print_failed_deployment_hints.bats +++ b/k8s/deployment/tests/print_failed_deployment_hints.bats @@ -83,8 +83,6 @@ assert_not_contains() { assert_contains "$output" "📋 Reason: The container exceeded its memory limit (512Mi)" assert_contains "$output" "📋 Detected: OOMKilled on container app (exit 137)" assert_contains "$output" "📋 Details: out of memory" - # The console calls this setting "RAM Memory"; `ram_memory` is the capability KEY and - # sends the reader looking for a field the scope form does not show. assert_contains "$output" "💡 Suggested fix: Raise the RAM Memory of scope 'my-app'" assert_not_contains "$output" "ram_memory" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" @@ -232,8 +230,9 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "did not pass its health check at /health" - assert_contains "$output" "💡 Suggested fix: Ensure the app listens on port 8080 and returns 2xx on /health" + assert_contains "$output" "📋 Reason: The application did not pass its health check at /health." + assert_contains "$output" "📋 Detected: Unhealthy on container api" + assert_contains "$output" "💡 Suggested fix: Ensure the app listens on port 8080 and returns 2xx on /health within the readiness window." assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -253,13 +252,11 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - # HUMAN_MESSAGE retains the base sentence and appends the translated probe failure - assert_contains "$output" "did not pass its health check at /health" - assert_contains "$output" "Detected: Startup probe" - assert_contains "$output" "not yet listening" - # SUGGESTED_FIX is targeted: tells the user the app is not binding the port - assert_contains "$output" "not listening on port 8080" - # Generic fallback fix must NOT appear + assert_contains "$output" "📋 Reason: The application did not pass its health check at /health. Detected: Startup probe — app is not yet listening on /health." + assert_contains "$output" "📋 Detected: Unhealthy on container api" + assert_contains "$output" "📋 Recent warnings:" + assert_contains "$output" " • Unhealthy (×1)" + assert_contains "$output" "💡 Suggested fix: The container is not listening on port 8080 — verify the start command runs, the process binds to 0.0.0.0:8080, and nothing is crashing before it accepts connections." assert_not_contains "$output" "returns 2xx on /health within the readiness window" } @@ -279,11 +276,9 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "Detected: Startup probe" - assert_contains "$output" "HTTP 502" - # SUGGESTED_FIX cites the status code and points to app logs - assert_contains "$output" "responded with HTTP 502" - assert_contains "$output" "inspect application logs" + assert_contains "$output" "📋 Reason: The application did not pass its health check at /health. Detected: Startup probe — app responded with HTTP 502 (expected 2xx)." + assert_contains "$output" "📋 Detected: Unhealthy on container api" + assert_contains "$output" "💡 Suggested fix: The app responded with HTTP 502 on /health — inspect application logs for startup errors; the process is running but /health is not returning 2xx." } @test "print_failed_deployment_hints: enriches Unhealthy with timeout detail and targeted fix" { @@ -302,17 +297,14 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "Detected: Startup probe" - assert_contains "$output" "timed out" - # SUGGESTED_FIX names the timing knobs the CONSOLE shows, not their kubernetes spellings. - assert_contains "$output" "Initial Delay or Timeout" + assert_contains "$output" "📋 Reason: The application did not pass its health check at /health. Detected: Startup probe — request timed out on /health." + assert_contains "$output" "📋 Detected: Unhealthy on container api" + assert_contains "$output" "💡 Suggested fix: The probe timed out — the app may be slow to start, or /health is blocking. Raise the health check's Initial Delay or Timeout on scope 'my-app', or make /health lighter." assert_not_contains "$output" "initialDelaySeconds" } @test "print_failed_deployment_hints: falls back to raw Unhealthy message when translation is impossible" { export K8S_NAMESPACE="ns" DEPLOYMENT_ID="d1" - # Message does not match any known probe pattern → translate_probe_message returns non-zero. - # The raw text must still be surfaced in the hint instead of being silently dropped. export ALL_EVENTS='{"items":[{"type":"Warning","reason":"Unhealthy","lastTimestamp":"2026-05-20T13:13:42Z","message":"completely unknown probe failure format from a future K8s"}]}' kubectl() { @@ -327,10 +319,8 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - # Raw message appears verbatim in the reason line - assert_contains "$output" "completely unknown probe failure format from a future K8s" - # Base sentence is still there - assert_contains "$output" "did not pass its health check at /health" + assert_contains "$output" "📋 Reason: The application did not pass its health check at /health. Detected: completely unknown probe failure format from a future K8s" + assert_contains "$output" "💡 Suggested fix: Ensure the app listens on port 8080 and returns 2xx on /health within the readiness window." } @test "print_failed_deployment_hints: Unhealthy picks the latest event when multiple are present" { @@ -403,8 +393,6 @@ assert_not_contains() { # health_check_path default "/" must apply when CONTEXT is unset. assert_contains "$output" "health check at /." assert_contains "$output" "returns 2xx on /" - # Guard against the previous escape bug: a literal backslash in the message - # would indicate jq received {\} instead of {} and silently failed. assert_not_contains "$output" "{\\" } @@ -494,8 +482,6 @@ assert_not_contains() { } @test "print_failed_deployment_hints: an OOM kill inside a crash loop is diagnosed as the OOM" { - # CrashLoopBackOff is the restart MECHANISM and OOMKilled is the cause; reading - # the wrapper first sent the operator to their startup logs for a memory limit. export K8S_NAMESPACE="ns" DEPLOYMENT_ID="d1" kubectl() { case "$*" in @@ -509,15 +495,14 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "exceeded its memory limit" - assert_contains "$output" "Raise the RAM Memory" - assert_not_contains "$output" "started and crashed repeatedly" + assert_contains "$output" "📋 Reason: The container exceeded its memory limit (512Mi) and was terminated." + assert_contains "$output" "📋 Detected: OOMKilled on container app (exit 137)" + assert_contains "$output" "📋 Details: back-off 20s restarting failed container=app pod=d-1_ns(abc)" + assert_contains "$output" "💡 Suggested fix: Raise the RAM Memory of scope 'my-app', or reduce how much memory the application uses." + assert_not_contains "$output" "The container started and crashed repeatedly." } @test "print_failed_deployment_hints: a plain non-zero exit in a crash loop keeps its startup-log advice" { - # Deferring to the termination reason must NOT strand the ordinary crash loop: - # `Error` had no branch of its own, so it would have fallen through to the - # nameless default with no suggested fix at all. export K8S_NAMESPACE="ns" DEPLOYMENT_ID="d1" kubectl() { case "$*" in @@ -531,8 +516,10 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "exited with code 1" - assert_contains "$output" "Review application logs for startup errors" + assert_contains "$output" "📋 Reason: The container started and exited with code 1." + assert_contains "$output" "📋 Detected: Error on container app (exit 1)" + assert_contains "$output" "📋 Details: back-off 20s restarting failed container=app" + assert_contains "$output" "💡 Suggested fix: Review application logs for startup errors (failed dependencies, bad config, panics)." assert_not_contains "$output" "Pods are failing with reason: Error" } @@ -550,6 +537,7 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "evicted from its node" - assert_contains "$output" "ran out of memory or disk" + assert_contains "$output" "📋 Reason: The pod was evicted from its node." + assert_contains "$output" "📋 Detected: Evicted on container app" + assert_contains "$output" "💡 Suggested fix: The node ran out of memory or disk. Lower the scope's resource requests, or free capacity on the cluster." } From c64249e6ec886d263f8d2ab3f13477e9b237cd8e Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Mon, 31 Aug 2026 23:33:20 -0300 Subject: [PATCH 50/52] feat(k8s): lifecycle jobs declare their identity, so the page finds their plans The deploy workflows (blue_green, initial) register their jobs with identity labels, which is how the deployment page shows the whole checklist pending the instant an operation is requested. The lifecycle workflows (rollback, finalize, delete, switch_traffic) registered theirs with no labels at all -- so a cancel's rollback plan, though registered and current, was unfindable, and the page showed an empty canvas for the ~13 seconds between the entity saying "cancelling" and the agent picking the rollback up, then popped every step in at once. Same grammar as the deploy jobs: `entity` + `scope.provider` scope the search; `operation` names the lifecycle act -- the word the deployment entity's own statuses use (rollback, finalize, delete, switch-traffic) -- where a deploy job uses `strategy`. --- k8s/deployment/workflows/delete.yaml | 10 ++++++++++ k8s/deployment/workflows/finalize.yaml | 10 ++++++++++ k8s/deployment/workflows/rollback.yaml | 10 ++++++++++ k8s/deployment/workflows/switch_traffic.yaml | 10 ++++++++++ 4 files changed, 40 insertions(+) diff --git a/k8s/deployment/workflows/delete.yaml b/k8s/deployment/workflows/delete.yaml index 512a41d3..46631f83 100644 --- a/k8s/deployment/workflows/delete.yaml +++ b/k8s/deployment/workflows/delete.yaml @@ -14,6 +14,16 @@ trace: job: name: k8s-deployment-delete namespace: "@context:scope.provider" + # Identity labels, the same grammar the deploy workflows declare: `entity` + + # `scope.provider` scope the search, and `operation` names the lifecycle act this + # job performs — the word the deployment entity's own statuses use. They are how + # the deployment page finds this plan BEFORE the run exists: a cancel shows the + # rollback checklist pending the moment the entity says the wind-down began, + # instead of an empty canvas until the agent picks the command up. + labels: + entity: deployment + operation: delete + scope.provider: "@context:scope.provider" steps: - name: load logging type: script diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index ba4a0f24..0ce7b0b9 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -20,6 +20,16 @@ trace: job: name: k8s-deployment-finalize namespace: "@context:scope.provider" + # Identity labels, the same grammar the deploy workflows declare: `entity` + + # `scope.provider` scope the search, and `operation` names the lifecycle act this + # job performs — the word the deployment entity's own statuses use. They are how + # the deployment page finds this plan BEFORE the run exists: a cancel shows the + # rollback checklist pending the moment the entity says the wind-down began, + # instead of an empty canvas until the agent picks the command up. + labels: + entity: deployment + operation: finalize + scope.provider: "@context:scope.provider" groups: - {key: finalize, title: Finalize} steps: diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index 0aa1e0aa..afff1b77 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -18,6 +18,16 @@ trace: job: name: k8s-deployment-rollback namespace: "@context:scope.provider" + # Identity labels, the same grammar the deploy workflows declare: `entity` + + # `scope.provider` scope the search, and `operation` names the lifecycle act this + # job performs — the word the deployment entity's own statuses use. They are how + # the deployment page finds this plan BEFORE the run exists: a cancel shows the + # rollback checklist pending the moment the entity says the wind-down began, + # instead of an empty canvas until the agent picks the command up. + labels: + entity: deployment + operation: rollback + scope.provider: "@context:scope.provider" groups: - {key: finalize, title: Finalize} steps: diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index 015cc737..e1affd3f 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -17,6 +17,16 @@ trace: job: name: k8s-deployment-switch-traffic namespace: "@context:scope.provider" + # Identity labels, the same grammar the deploy workflows declare: `entity` + + # `scope.provider` scope the search, and `operation` names the lifecycle act this + # job performs — the word the deployment entity's own statuses use. They are how + # the deployment page finds this plan BEFORE the run exists: a cancel shows the + # rollback checklist pending the moment the entity says the wind-down began, + # instead of an empty canvas until the agent picks the command up. + labels: + entity: deployment + operation: switch-traffic + scope.provider: "@context:scope.provider" groups: - {key: switching-traffic, title: Switching traffic} steps: From 65b9ac445201f620923b6638e6b911cc6dcdde25 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Tue, 1 Sep 2026 13:25:55 -0300 Subject: [PATCH 51/52] feat(k8s): scope lifecycle jobs declare their identity, so consumers find their plans The deployment lifecycle jobs already register with identity labels; the scope lifecycle jobs (create, update, delete) registered with none -- so no surface can find a scope operation's plan before its run exists. Same grammar: entity + operation + scope.provider. --- k8s/scope/workflows/create.yaml | 8 ++++++++ k8s/scope/workflows/delete.yaml | 8 ++++++++ k8s/scope/workflows/update.yaml | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/k8s/scope/workflows/create.yaml b/k8s/scope/workflows/create.yaml index da4f78a6..f43daef1 100644 --- a/k8s/scope/workflows/create.yaml +++ b/k8s/scope/workflows/create.yaml @@ -13,6 +13,14 @@ trace: job: name: k8s-scope-create namespace: "@context:scope.provider" + # Identity labels, the same grammar the deployment workflows declare: + # `entity` + `scope.provider` scope the search, `operation` names the + # lifecycle act. They are how a consumer finds this plan BEFORE the run + # exists, instead of an empty canvas until the agent picks the command up. + labels: + entity: scope + operation: create + scope.provider: "@context:scope.provider" steps: - name: load logging type: script diff --git a/k8s/scope/workflows/delete.yaml b/k8s/scope/workflows/delete.yaml index cf1abfee..934a2de0 100644 --- a/k8s/scope/workflows/delete.yaml +++ b/k8s/scope/workflows/delete.yaml @@ -9,6 +9,14 @@ trace: job: name: k8s-scope-delete namespace: "@context:scope.provider" + # Identity labels, the same grammar the deployment workflows declare: + # `entity` + `scope.provider` scope the search, `operation` names the + # lifecycle act. They are how a consumer finds this plan BEFORE the run + # exists, instead of an empty canvas until the agent picks the command up. + labels: + entity: scope + operation: delete + scope.provider: "@context:scope.provider" steps: - name: load logging type: script diff --git a/k8s/scope/workflows/update.yaml b/k8s/scope/workflows/update.yaml index 3eedc336..46036585 100644 --- a/k8s/scope/workflows/update.yaml +++ b/k8s/scope/workflows/update.yaml @@ -9,6 +9,14 @@ trace: job: name: k8s-scope-update namespace: "@context:scope.provider" + # Identity labels, the same grammar the deployment workflows declare: + # `entity` + `scope.provider` scope the search, `operation` names the + # lifecycle act. They are how a consumer finds this plan BEFORE the run + # exists, instead of an empty canvas until the agent picks the command up. + labels: + entity: scope + operation: update + scope.provider: "@context:scope.provider" steps: - name: networking type: workflow From 789464284e483d0fae1c50aad6cc9105384d3509 Mon Sep 17 00:00:00 2001 From: Pablo Vilas Date: Wed, 2 Sep 2026 15:02:12 -0300 Subject: [PATCH 52/52] refactor(tracing): drop explanatory comments from the traced scripts and workflows --- k8s/apply_templates | 19 --- k8s/deployment/print_failed_deployment_hints | 16 +- k8s/deployment/scale_deployments | 3 - .../tests/wait_deployment_active.bats | 24 --- .../validate_alb_target_group_capacity | 1 - .../verify_http_route_reconciliation | 2 - k8s/deployment/verify_ingress_reconciliation | 4 - .../verify_networking_reconciliation | 2 - k8s/deployment/wait_deployment_active | 92 ----------- k8s/deployment/workflows/blue_green.yaml | 17 -- k8s/deployment/workflows/delete.yaml | 13 -- k8s/deployment/workflows/diagnose.yaml | 6 - k8s/deployment/workflows/finalize.yaml | 21 --- k8s/deployment/workflows/initial.yaml | 12 -- k8s/deployment/workflows/kill_instance.yaml | 4 - k8s/deployment/workflows/rollback.yaml | 17 -- k8s/deployment/workflows/switch_traffic.yaml | 20 --- k8s/diagnose/tests/diagnose_utils.bats | 3 - k8s/diagnose/utils/diagnose_utils | 16 -- k8s/logging | 147 ------------------ k8s/scope/build_context | 2 - k8s/scope/iam/create_role | 2 - k8s/scope/networking/dns/manage_dns | 2 - k8s/scope/networking/wait_for_alb | 10 -- k8s/scope/wait_on_balancer | 2 - k8s/scope/workflows/create.yaml | 13 -- k8s/scope/workflows/delete.yaml | 8 - k8s/scope/workflows/diagnose.yaml | 5 - k8s/scope/workflows/pause-autoscaling.yaml | 4 - k8s/scope/workflows/restart-pods.yaml | 4 - k8s/scope/workflows/resume-autoscaling.yaml | 4 - .../workflows/set-desired-instance-count.yaml | 4 - k8s/scope/workflows/update.yaml | 7 - k8s/utils/tests/trace_logging.bats | 42 ----- scheduled_task/logging | 147 ------------------ 35 files changed, 1 insertion(+), 694 deletions(-) diff --git a/k8s/apply_templates b/k8s/apply_templates index 7b4b8500..3508b9da 100644 --- a/k8s/apply_templates +++ b/k8s/apply_templates @@ -31,10 +31,6 @@ while IFS= read -r TEMPLATE_FILE; do IGNORE_NOT_FOUND="--ignore-not-found=true" fi - # Each manifest apply is its own SUB-STEP, keyed by the platform's step - # vocabulary (create-deployment, create-service, ...) so a custom - # deploy's trace reads like a native one — the manifest filename prefix - # names what is being created. TRACE_STEP_KEY="" if [[ "$ACTION" == "apply" ]] && command -v np_scope_step_begin >/dev/null 2>&1; then case "$FILENAME" in @@ -53,20 +49,12 @@ while IFS= read -r TEMPLATE_FILE; do [[ -n "$TRACE_STEP_KEY" ]] && np_scope_step_begin "$TRACE_STEP_KEY" fi - # Captured with stderr (and re-echoed) so applied resources can be - # recorded as lineage — and so a FAILURE carries kubectl's actual reason - # onto the trace, not a bare "failed to apply". if KUBECTL_OUT=$(kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND 2>&1); then [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 0 - # Lineage AFTER the sub-step closes: what was applied belongs to the PLANNED - # step the user clicks ("Apply manifests"), not to an invisible technical - # sub-step — the row's detail must show the workloads it produced. if [[ "$ACTION" == "apply" ]] && command -v np_scope_k8s_applied >/dev/null 2>&1; then np_scope_k8s_applied "${K8S_NAMESPACE:-}" "$KUBECTL_OUT" fi - # A removal is evidence too: the finalize/rollback row that deleted the - # previous (or aborted) deployment names what it removed. if [[ "$ACTION" == "delete" ]] && command -v np_scope_k8s_deleted >/dev/null 2>&1; then np_scope_k8s_deleted "${K8S_NAMESPACE:-}" "$KUBECTL_OUT" fi @@ -94,11 +82,6 @@ if [[ "$DRY_RUN" == "true" ]]; then exit 1 fi -# When this apply IS the blue/green traffic switch (switch_traffic.yaml sets -# TRACE_TRAFFIC_SWITCH on the step), report the switch the way the platform's -# own scopes do: the request as input, what landed as output, the resulting -# traffic split as the affordance a host renders, and the phase's convergence -# toward 100%. if [[ "${TRACE_TRAFFIC_SWITCH:-false}" == "true" ]] && [[ -n "${CONTEXT:-}" ]] \ && command -v np_scope_explain >/dev/null 2>&1; then TRAFFIC_TO=$(echo "$CONTEXT" | jq -r '.deployment.strategy_data.desired_switched_traffic // 100') @@ -114,8 +97,6 @@ if [[ "${TRACE_TRAFFIC_SWITCH:-false}" == "true" ]] && [[ -n "${CONTEXT:-}" ]] \ fi fi -# The manifests' lineage (produced / removed pointers) is this step's evidence — -# land it now instead of betting the exit flush's budget on it. if command -v np_trace_flush >/dev/null 2>&1; then NP_TRACE_FLUSH_TIMEOUT=5 np_trace_flush fi diff --git a/k8s/deployment/print_failed_deployment_hints b/k8s/deployment/print_failed_deployment_hints index 30d594d6..9527f37d 100644 --- a/k8s/deployment/print_failed_deployment_hints +++ b/k8s/deployment/print_failed_deployment_hints @@ -122,13 +122,6 @@ diagnose_failure() { fi if [[ -n "$pods_json" ]] && echo "$pods_json" | jq -e . >/dev/null 2>&1; then - # CrashLoopBackOff/BackOff name the MECHANISM (it keeps restarting), never - # the cause: the cause is how the container died the last time round. An - # OOM-killed container in a restart loop reports BOTH, and leading with the - # wrapper sends the reader to "review your startup logs" for what is really - # a memory limit. So a backoff wrapper defers to the termination reason. - # Any other waiting reason (ImagePullBackOff, CreateContainerError) IS the - # cause and still wins. FAILURE_REASON=$(echo "$pods_json" | jq -r ' [.items[].status.containerStatuses[]? | (.state.waiting.reason // "") as $w @@ -194,11 +187,6 @@ diagnose_failure() { HUMAN_MESSAGE="The container started and crashed repeatedly." SUGGESTED_FIX="Review application logs for startup errors (failed dependencies, bad config, panics)." ;; Error|ContainerStatusUnknown) - # The ordinary crash-loop cause: the app ran and exited non-zero. Now - # that the backoff wrapper defers to the termination reason, this is - # where a panic or a bad config lands, so it needs the crash-loop - # wording the wrapper used to give it — plus the exit code, which is - # the first thing an operator greps their logs for. if [[ -n "$FAILURE_EXIT_CODE" ]]; then HUMAN_MESSAGE="The container started and exited with code ${FAILURE_EXIT_CODE}." else @@ -268,9 +256,7 @@ diagnose_failure() { SUGGESTED_FIX="" ;; *) HUMAN_MESSAGE="Pods are failing with reason: $FAILURE_REASON" - # Deliberately empty: an empty fix is the signal that hands the reader - # the full generic troubleshooting checklist below, which is richer than - # any one line this branch could invent for a reason it does not know. + # Empty on purpose: an empty fix routes the reader to the generic checklist below. SUGGESTED_FIX="" ;; esac } diff --git a/k8s/deployment/scale_deployments b/k8s/deployment/scale_deployments index 34a503f9..8ff15df3 100755 --- a/k8s/deployment/scale_deployments +++ b/k8s/deployment/scale_deployments @@ -8,8 +8,6 @@ BLUE_REPLICAS=$(echo "$CONTEXT" | jq -r .blue_replicas) BLUE_DEPLOYMENT_ID=$(echo "$CONTEXT" | jq .scope.current_active_deployment -r) if [ "$DEPLOY_STRATEGY" != "rolling" ]; then - # Blue-green carries its replica counts in the manifests themselves — this step - # has nothing to do, and the trace says so instead of showing a hollow success. if command -v np_step_skip >/dev/null 2>&1; then np_step_skip "instance counts are set by the manifests for the $DEPLOY_STRATEGY strategy — nothing to scale" fi @@ -49,7 +47,6 @@ if [ "$DEPLOY_STRATEGY" = "rolling" ]; then unset TIMEOUT unset SKIP_DEPLOYMENT_STATUS_CHECK - # What this step DID, on the row a user clicks: the resulting replica split. if command -v np_scope_output >/dev/null 2>&1; then np_scope_output replicas "{\"green\": $GREEN_REPLICAS, \"previous\": $BLUE_REPLICAS}" fi diff --git a/k8s/deployment/tests/wait_deployment_active.bats b/k8s/deployment/tests/wait_deployment_active.bats index ce449930..368c8ba8 100644 --- a/k8s/deployment/tests/wait_deployment_active.bats +++ b/k8s/deployment/tests/wait_deployment_active.bats @@ -657,29 +657,21 @@ teardown() { } @test "wait_deployment_active: a multi-word reason survives the reason list whole" { - # The event sweep contributes reasons already in human words ("Startup probe - # failing"). Splitting the list on whitespace as well as the comma shredded those - # into three, and the phase read "1 with Startup, probe, failing". source "$BATS_TEST_DIRNAME/../wait_deployment_active" 2>/dev/null || true run bash -c " source '$BATS_TEST_DIRNAME/../print_failed_deployment_hints' 2>/dev/null || true \$(declare -f humanize_k8s_reason humanize_k8s_reasons 2>/dev/null) true " - # Exercise the helpers directly out of the script's own source. eval "$(sed -n '/^humanize_k8s_reason()/,/^}/p;/^humanize_k8s_reasons()/,/^}/p' "$BATS_TEST_DIRNAME/../wait_deployment_active")" [ "$(humanize_k8s_reasons 'Startup probe failing')" = "Startup probe failing" ] [ "$(humanize_k8s_reasons 'OOMKilled, CrashLoopBackOff')" = "out of memory, crashing repeatedly" ] [ "$(humanize_k8s_reasons 'Startup probe failing, OOMKilled')" = "Startup probe failing, out of memory" ] - # Two codes meaning the same thing still read once. [ "$(humanize_k8s_reasons 'ImagePullBackOff, ErrImagePull')" = "can't pull the container image" ] } @test "wait_deployment_active: the trace carries WHY it is stuck and what to do, not just the counts" { - # The console hints already classified this failure; the step's explain must say the - # same thing, so a reader who opens the phase gets the reason and the fix without - # going to the logs. run bash -c " sleep() { :; } export -f sleep @@ -703,9 +695,6 @@ teardown() { np() { echo 'running'; } export -f np - # Capture what the step's facets are told, without a tracing backend. The - # timeout's trace block is guarded on np_scope_step_timeout existing, so the - # stubs must cover the whole terminal trio. np_scope_explain() { echo \"EXPLAIN \$*\"; } np_scope_error() { echo \"ERROR \$*\"; } np_scope_step_timeout() { echo \"TIMEOUT \$*\"; } @@ -719,11 +708,9 @@ teardown() { " [ "$status" -eq 1 ] - # WHY, in the reader's words, naming the configured path and what was detected. assert_contains "$output" "did not pass its health check at /health-bad" assert_contains "$output" "Detected: Startup probe" assert_contains "$output" "HTTP 404" - # …and the actionable next step. assert_contains "$output" "--next" } @@ -930,9 +917,6 @@ teardown() { } @test "wait_deployment_active: an OOM kill in a restart loop is reported as out of memory, not as the loop" { - # CrashLoopBackOff is the MECHANISM; OOMKilled is why the container died. Reading - # the wrapper sent the operator to "review your startup logs" for what was really - # a memory limit, so the backoff wrappers defer to the termination reason. cat > "$BATS_TMPDIR/oom-pods.json" <<'JSON' {"items":[{"metadata":{"name":"d-1-2-abc"}, "status":{"containerStatuses":[{"name":"application", @@ -941,7 +925,6 @@ teardown() { "restartCount":3}]}}]} JSON - # The narrative's reason list: the cause wins. run jq -r '[.items[] | .status.containerStatuses[]? | .state.waiting.reason as $w | (.lastState.terminated.reason // "") as $t0 @@ -956,8 +939,6 @@ JSON } @test "wait_deployment_active: the mechanism stays on the io even though the narrative names the cause" { - # An agent reading the step must still see that it was looping — the wrapper is - # kept as `reason`, the cause added as `cause`. Only the human line is narrowed. cat > "$BATS_TMPDIR/oom-pods.json" <<'JSON' {"items":[{"metadata":{"name":"d-1-2-abc"}, "status":{"containerStatuses":[{"name":"application", @@ -978,8 +959,6 @@ JSON } @test "wait_deployment_active: a real cause still wins over the termination reason" { - # Only the backoff wrappers defer. ImagePullBackOff IS the cause and must not be - # replaced by whatever the container happened to exit with last time. cat > "$BATS_TMPDIR/pull-pods.json" <<'JSON' {"items":[{"metadata":{"name":"d-1-2-abc"}, "status":{"containerStatuses":[{"name":"application", @@ -996,9 +975,6 @@ JSON } @test "wait_deployment_active: the back-off boilerplate never reaches the phase line" { - # "back-off 20s restarting failed container=… pod=…(uuid)" repeats the reason and - # then runs the line off the page with two ids. It is dropped from the narrative; - # a registry's real error is information and is kept (bounded). detail_of() { local d="$1" case "$d" in diff --git a/k8s/deployment/validate_alb_target_group_capacity b/k8s/deployment/validate_alb_target_group_capacity index 94a0bad8..2ec73134 100755 --- a/k8s/deployment/validate_alb_target_group_capacity +++ b/k8s/deployment/validate_alb_target_group_capacity @@ -185,7 +185,6 @@ fi log info "✅ ALB listener capacity validated: $LISTENER_COUNT/$ALB_MAX_LISTENERS" -# The row's summary: the quota headroom this pre-flight verified, in numbers. if command -v np_scope_explain >/dev/null 2>&1; then np_scope_explain --title "Validate load balancer capacity" --what "Load balancer has room: $TARGET_GROUP_COUNT/$ALB_MAX_TARGET_GROUPS target groups, $LISTENER_COUNT/$ALB_MAX_LISTENERS listeners in use" np_scope_output capacity "{\"target_groups\": {\"used\": $TARGET_GROUP_COUNT, \"max\": $ALB_MAX_TARGET_GROUPS}, \"listeners\": {\"used\": $LISTENER_COUNT, \"max\": $ALB_MAX_LISTENERS}}" diff --git a/k8s/deployment/verify_http_route_reconciliation b/k8s/deployment/verify_http_route_reconciliation index c4b914fb..01113dc3 100644 --- a/k8s/deployment/verify_http_route_reconciliation +++ b/k8s/deployment/verify_http_route_reconciliation @@ -11,8 +11,6 @@ elapsed=0 log debug "🔍 Verifying HTTPRoute reconciliation..." log debug "📋 HTTPRoute: $HTTPROUTE_NAME | Namespace: $K8S_NAMESPACE | Timeout: ${MAX_WAIT_SECONDS}s" -# The reconciliation wait is its own SUB-STEP in the trace. (Guarded: -# overrides may reuse this script without k8s/logging loaded.) if command -v np_scope_step_begin >/dev/null 2>&1; then np_scope_step_begin verify-httproute --title "Verify HTTPRoute reconciliation ($HTTPROUTE_NAME)" np_scope_wait_heartbeat "httproute-reconciliation" 0 "$MAX_WAIT_SECONDS" "pending" diff --git a/k8s/deployment/verify_ingress_reconciliation b/k8s/deployment/verify_ingress_reconciliation index 942a9994..af4f2d8a 100644 --- a/k8s/deployment/verify_ingress_reconciliation +++ b/k8s/deployment/verify_ingress_reconciliation @@ -250,8 +250,6 @@ validate_alb_config() { fi } -# The reconciliation wait is its own SUB-STEP in the trace. (Guarded: -# overrides may reuse this script without k8s/logging loaded.) if command -v np_scope_step_begin >/dev/null 2>&1; then np_scope_step_begin verify-ingress --title "Verify ingress reconciliation ($INGRESS_NAME)" np_scope_wait_heartbeat "ingress-reconciliation" 0 "$MAX_WAIT_SECONDS" "pending" @@ -263,8 +261,6 @@ while [ $elapsed -lt $MAX_WAIT_SECONDS ]; do log info "✅ ALB configuration validated successfully" if command -v np_scope_step_end >/dev/null 2>&1; then np_scope_step_end 0 - # The planned row's plain-language summary — what was verified, on the - # step the user clicks (after the sub-step closes). np_scope_explain --title "Verify networking" --what "Load-balancer routing verified for $INGRESS_NAME — rules and weights match the deployment" fi return 0 diff --git a/k8s/deployment/verify_networking_reconciliation b/k8s/deployment/verify_networking_reconciliation index c01dd16b..ce3cab33 100644 --- a/k8s/deployment/verify_networking_reconciliation +++ b/k8s/deployment/verify_networking_reconciliation @@ -12,8 +12,6 @@ case "$DNS_TYPE" in ;; *) log warn "⚠️ Ingress reconciliation not available for DNS type: $DNS_TYPE, skipping" - # On azure/ARO there is nothing to verify: close the step as `skipped`, - # not a hollow `completed`. (Guarded: defined by the np CLI's preamble.) if command -v np_step_skip >/dev/null 2>&1; then np_step_skip "no networking verification for DNS type '$DNS_TYPE'" fi diff --git a/k8s/deployment/wait_deployment_active b/k8s/deployment/wait_deployment_active index 1f6d2d0a..e2e3c839 100755 --- a/k8s/deployment/wait_deployment_active +++ b/k8s/deployment/wait_deployment_active @@ -55,25 +55,14 @@ iteration=0 LATEST_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") SKIP_DEPLOYMENT_STATUS_CHECK="${SKIP_DEPLOYMENT_STATUS_CHECK:=false}" LAST_REPORTED_COUNTS="" -# The wait's memory for the trace narrative: how many pods the LAST event -# sweep saw failing health checks (and why), and the pods' restart history — -# a crash-loop that heals mid-wait must not read as an uneventful wait. UNHEALTHY_POD_COUNT=0 UNHEALTHY_POD_REASONS="" -# The row's phase-specific words, derived from the step this wait is running AS -# (NP_TRACE ends with the step's own run path, `...~@attempt.iteration`) — -# they must match the trace titles the workflows declare. Never passed through -# step configuration: the engine splices configuration values into the shell -# fragment, where a spaced value breaks the command. case "${NP_TRACE:-}" in *~switch-instances-check@*) WAIT_TITLE="Verify scaled instances" ;; *~finalize-instances-check@*) WAIT_TITLE="Verify final capacity" ;; *) WAIT_TITLE="Instance health check" ;; esac -# Provider reason CODES -> plain words for the narrative surfaces (same table the -# native scopes use). The raw code is machine detail — it stays on the io facets -# for the dialog; the narrative speaks human. Unknown codes fall through verbatim. humanize_k8s_reason() { case "$1" in OOMKilled) echo "out of memory" ;; @@ -92,18 +81,10 @@ humanize_k8s_reason() { esac } -# A COMMA-separated reason list, translated and deduped AFTER translation — two -# codes that mean the same thing read once. -# -# Split on the comma ALONE. Not every entry is a one-word kubernetes code: the event -# sweep contributes phrases it has already put into human words ("Startup probe -# failing"), and splitting on whitespace too shredded those into their own "reasons" — -# the phase read "1 with Startup, probe, failing". humanize_k8s_reasons() { local _hr_out="" _hr_word IFS=',' read -ra _hr_parts <<< "$1" for _hr_part in "${_hr_parts[@]}"; do - # Trim the space that follows the comma in the joined form. _hr_part="${_hr_part#"${_hr_part%%[![:space:]]*}"}" _hr_part="${_hr_part%"${_hr_part##*[![:space:]]}"}" [ -n "$_hr_part" ] || continue @@ -116,12 +97,6 @@ humanize_k8s_reasons() { echo "$_hr_out" } -# The classified diagnosis, for the TRACE — the same words the console hints print, -# from the same classifier (diagnose_failure in print_failed_deployment_hints, which -# reads ALL_EVENTS): WHY it is stuck ("The application did not pass its health check at -# /health — Detected: Startup probe — app responded with HTTP 404 (expected 2xx)") and -# what to do about it. Sets WAIT_WHY / WAIT_NEXT, both empty when nothing classifies. -# Cheap enough per heartbeat because it only runs once something is actually wrong. WAIT_WHY="" WAIT_NEXT="" classify_wait_failure() { @@ -134,20 +109,10 @@ classify_wait_failure() { return 0 } -# Report the wait's live narrative onto the trace — the counted io, the -# instances-health meter a host renders as pips, and the plain-language -# explain with the severity an operator should read it at. Re-emitted per -# heartbeat: facets fold last-writer-wins, so the latest state (and a -# recovered warning) always wins. Best-effort, like every trace call. report_wait_narrative() { command -v np_scope_explain >/dev/null 2>&1 || return 0 local ready_now="$1" desired_now="$2" launched_now="$3" all_healthy="${4:-false}" - # ONE pod-state read feeds everything: the restart history (with its - # reasons), and the LIVE problem classification with kubernetes' own - # verbatim message — an ImagePullBackOff shows the registry's actual - # error, not a "check your image" hint. Boot churn (ContainerCreating, - # PodInitializing) is never presented as a problem. local pods_json restarted restart_total restart_reasons restart_clause local problems problem_count problem_reasons real_detail pods_json=$(kubectl get pods -n "$K8S_NAMESPACE" -l "deployment_id=${DEPLOYMENT_ID}" -o json 2>/dev/null) || pods_json="" @@ -175,15 +140,10 @@ report_wait_narrative() { | with_entries(select(.value != ""))]' 2>/dev/null) || problems="[]" [ -n "$problems" ] || problems="[]" problem_count=$(echo "$problems" | jq 'length' 2>/dev/null) || problem_count=0 - # The narrative names the CAUSE (out of memory), not the wrapper that kept - # restarting it; `reason` stays on the io so an agent still sees the loop. problem_reasons=$(echo "$problems" | jq -r '[.[] | (.cause // .reason)] | unique | join(", ")' 2>/dev/null) || problem_reasons="" - # Kubernetes' own words for the FIRST problem — the real error, verbatim. real_detail=$(echo "$problems" | jq -r '[.[].message // empty] | first // ""' 2>/dev/null \ | tr '\n' ' ' | sed 's/[[:space:]]*$//') || real_detail="" - # The event sweep fills in what pod state alone cannot name (which probe, - # on which path) when the state itself shows nothing. if [ "${problem_count:-0}" -eq 0 ] && [ "$UNHEALTHY_POD_COUNT" -gt 0 ]; then problem_count=$UNHEALTHY_POD_COUNT problem_reasons=$UNHEALTHY_POD_REASONS @@ -191,8 +151,6 @@ report_wait_narrative() { WAIT_EFFECTIVE_REASONS="${problem_reasons:-$restart_reasons}" WAIT_REAL_DETAIL="$real_detail" - # A crash-loop's REAL error is what the app printed before it died: fetch - # the previous container's last lines once per situation change, bounded. local crash_log="" if [ "$restart_total" -gt 0 ]; then local crash_pod @@ -205,23 +163,13 @@ report_wait_narrative() { fi fi - # A back-off message is pure MECHANISM: "back-off 20s restarting failed - # container=application pod=d-253247585-…_nullplatform(498dc0ab-…)" repeats the - # reason we just named and then buries the line in two ids no reader parses. - # It is the whole reason the phase line ran off the page. Drop it from the - # narrative — the cause is on `why`, and the verbatim message is still on the - # io for an agent. Everything else (a registry's real error) is information. case "$WAIT_REAL_DETAIL" in "back-off "*"restarting failed container="*) WAIT_REAL_DETAIL="" ;; esac - # What is left is read at a glance, so keep it to about one line; the io - # carries it untruncated. if [ ${#WAIT_REAL_DETAIL} -gt 140 ]; then WAIT_REAL_DETAIL="${WAIT_REAL_DETAIL:0:137}..." fi - # Realtime without noise: emit only when the SITUATION changes (counts, a - # new problem, a recovery) — the loop calls this every poll. local snapshot="$ready_now/$desired_now/$launched_now/$problem_count/$problem_reasons/$restart_total/$all_healthy" if [ "$snapshot" = "${WAIT_NARRATIVE_SNAPSHOT:-}" ]; then return 0 @@ -237,8 +185,6 @@ report_wait_narrative() { + (if $crash != "" then {last_crash_log: $crash} else {} end)') np_scope_output instances "$instances" - # The meter is user-facing: its reason captions speak human words; the io - # above keeps the raw codes for the dialog. meter=$(jq -nc --argjson h "$ready_now" --argjson l "$launched_now" --argjson d "$desired_now" \ --argjson u "$problem_count" --arg reasons "$(humanize_k8s_reasons "$problem_reasons")" --argjson t "$restart_total" \ --arg rreasons "$(humanize_k8s_reasons "$restart_reasons")" \ @@ -254,8 +200,6 @@ report_wait_narrative() { if [ "$problem_count" -gt 0 ]; then local problem_words problem_words=$(humanize_k8s_reasons "${problem_reasons:-}") - # The reader asks "why?" the moment the meter goes amber, not only when the - # wait finally gives up — so the diagnosis rides the LIVE narrative too. classify_wait_failure np_scope_explain --title "$WAIT_TITLE" --severity warn \ --what "Waiting for $ready_now/$desired_now instances to be healthy — $problem_count with ${problem_words:-failing health checks}$detail_clause" \ @@ -263,11 +207,6 @@ report_wait_narrative() { --impact "The deployment fails if the instances don't become healthy before the health-check timeout." \ ${WAIT_NEXT:+--next "$WAIT_NEXT"} elif [ "$restart_total" -gt 0 ]; then - # A container that crashed and is running again is between waiting states, - # so nothing above classified it — and this branch used to say only THAT it - # restarted, never why or what to do. A restart has a cause (the - # termination reason is right there in `restart_reasons`), so ask for it - # here too rather than leaving the amber phase mute. classify_wait_failure if [ "$all_healthy" = "true" ]; then np_scope_explain --title "$WAIT_TITLE" --severity warn \ @@ -285,8 +224,6 @@ report_wait_narrative() { elif [ "$all_healthy" = "true" ]; then np_scope_explain --title "$WAIT_TITLE" --what "All $desired_now instances healthy" else - # Calm but not silent: instances exist and are booting — a heavy boot reads - # as a heavy boot, never as a black box (and never as a problem). local graced_clause="" if [ "$launched_now" -gt "$ready_now" ] 2>/dev/null; then graced_clause=" (instances starting — normal while the app boots)" @@ -335,10 +272,6 @@ log debug "📋 Namespace: $K8S_NAMESPACE" log debug "📋 Timeout: ${TIMEOUT}s (max $MAX_ITERATIONS iterations)" log debug "" -# The rollout wait IS the planned step ("Instance health check") — heartbeats, -# counts and the narrative attach to that step directly, so clicking its row -# shows the diagnosis. A sub-step would split one human moment across two -# nodes. (Guarded: overrides may reuse this script without k8s/logging loaded.) if command -v np_scope_wait_heartbeat >/dev/null 2>&1; then np_scope_wait_heartbeat "deployment-active" 0 "$TIMEOUT" "starting" fi @@ -353,11 +286,6 @@ while true; do source "$SERVICE_PATH/deployment/print_failed_deployment_hints" if command -v np_scope_step_timeout >/dev/null 2>&1; then - # The terminal leads with the CAUSE — kubernetes' own words or the - # app's last crash line — and the mechanism (the timeout) follows - # in parentheses. ONE message everywhere: facets fold last-writer- - # wins, so the close must not overwrite the cause with the - # mechanism. timeout_cause="${WAIT_REAL_DETAIL:-}" timeout_reasons="${WAIT_EFFECTIVE_REASONS:-$UNHEALTHY_POD_REASONS}" timeout_message="deployment '$K8S_DEPLOYMENT_NAME' not active after ${TIMEOUT}s" @@ -365,8 +293,6 @@ while true; do timeout_message="$timeout_cause ($timeout_message)" fi timeout_reason_words=$(humanize_k8s_reasons "${timeout_reasons:-}") - # print_failed_deployment_hints (sourced above) has just classified the - # failure; the trace says the same thing the console said. np_scope_explain --title "$WAIT_TITLE" --severity error \ --what "Gave up with ${ready:-0}/${desired:-0} instances healthy${timeout_reason_words:+ — $timeout_reason_words}${timeout_cause:+: $timeout_cause}" \ ${HUMAN_MESSAGE:+--why "$HUMAN_MESSAGE"} \ @@ -414,9 +340,6 @@ while true; do report_instance_counts "$desired" "$launched" "$ready" - # Realtime WHY: the narrative re-evaluates every poll and emits only when - # the situation changes — a new problem appears on the trace within one - # poll interval, not at the next heartbeat. report_wait_narrative "$ready" "$desired" "$launched" if [ "$desired" = "$current" ] && [ "$desired" = "$updated" ] && [ "$desired" = "$ready" ] && [ "$desired" -gt 0 ]; then @@ -424,19 +347,10 @@ while true; do log info "✅ All pods in deployment '$K8S_DEPLOYMENT_NAME' are available and ready!" if command -v np_scope_progress >/dev/null 2>&1; then np_scope_progress "$ready" "$desired" count - # Every pod is ready: any earlier probe warning has recovered, and - # the terminal narrative must say so (a crash-loop on the way here - # stays visible as "healthy — after N restarts"). UNHEALTHY_POD_COUNT=0 UNHEALTHY_POD_REASONS="" report_wait_narrative "$ready" "$desired" "$launched" true - # Lineage, by the same canonical ids the platform's own scopes use, - # so a custom scope's graph joins by value: - # • the deployment CONSUMED the build's asset — the cross-flow - # edge that links this deploy back to the build that made it - # • it PRODUCED this deploy's logs; the pointer is the concrete - # log query, and the affordance is the UI's "view logs" control _wda_asset_url=$(echo "$CONTEXT" | jq -r '.asset.url // empty') _wda_asset_type=$(echo "$CONTEXT" | jq -r '.asset.type // "docker-image"') if [ -n "$_wda_asset_url" ]; then @@ -455,10 +369,6 @@ while true; do np_scope_affordance "{\"kind\":\"deploy-log\",\"application_id\":\"$_wda_app_id\",\"scope_id\":\"$_wda_scope_id\",\"type\":\"application\"${_wda_start_ms:+,\"start_time\":$_wda_start_ms}}" fi - # The wait's FINAL truth (all healthy, the lineage, the log offer) must not - # ride only the exit flush — a spent budget there loses exactly the tail. - # One bounded flush here: the wait already took its time; 10s to land the - # diagnosis is the cheapest part of it. NP_TRACE_FLUSH_TIMEOUT=10 np_trace_flush fi break @@ -543,8 +453,6 @@ while true; do while IFS=$'\t' read -r ts pod_name messages_concat; do [ -z "$pod_name" ] && continue ((UNHEALTHY_POD_COUNT++)) - # Remember WHY for the trace narrative — the probe kind when the - # message parses, a generic reason when it doesn't. first_msg=$(printf '%s' "$messages_concat" | tr '\001' '\n' | head -1) parsed=$(parse_probe_message "$first_msg" 2>/dev/null) || parsed="" probe_kind="${parsed%%|*}" diff --git a/k8s/deployment/workflows/blue_green.yaml b/k8s/deployment/workflows/blue_green.yaml index 89517c12..695046c7 100644 --- a/k8s/deployment/workflows/blue_green.yaml +++ b/k8s/deployment/workflows/blue_green.yaml @@ -3,20 +3,9 @@ include: configuration: INGRESS_TEMPLATE: "$BLUE_GREEN_INGRESS_PATH" trace: - # What a HUMAN calls this execution. It lands on the run's explain.title — - # the first rung of every consumer's naming contract — so the page shows the - # name we chose, never a wording derived from this file's basename. title: Blue/green deployment flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - # Declared steps only: engine fragments without a trace block (override - # plumbing) stay off the wire — the fold shows real observable acts, not - # 0ms no-ops. Script-emitted child steps (per-manifest applies) are - # unaffected. default: false - # The job's identity labels are how the deployment page FINDS this plan - # before any run exists (the pending checklist on a fresh deployment): - # the page queries entity + strategy + the scope's provider (the service - # specification id — stable across every scope this agent serves). job: name: k8s-deployment-blue-green namespace: "@context:scope.provider" @@ -27,15 +16,9 @@ trace: groups: - {key: setting-up, title: Setting up} - {key: waiting-instances, title: Waiting for instances to be healthy} - # Declared with no steps here on purpose: PLACEHOLDER stations the later - # workflows (switch_traffic, finalize) fill — the line shows the whole - # journey pending from the first paint, exactly like a native scope. - {key: switching-traffic, title: Switching traffic} - {key: finalize, title: Finalize} steps: - # No milestone title: scaling is a no-op for blue-green (counts ride the - # manifests; the script skips itself) and technical detail for rolling — - # native's Setting up declares no scaling milestone either. - name: update blue deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" diff --git a/k8s/deployment/workflows/delete.yaml b/k8s/deployment/workflows/delete.yaml index 46631f83..0896731e 100644 --- a/k8s/deployment/workflows/delete.yaml +++ b/k8s/deployment/workflows/delete.yaml @@ -3,23 +3,10 @@ include: trace: title: Remove deployment flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - # Declared steps only: engine fragments without a trace block (override - # plumbing) stay off the wire — the fold shows real observable acts, not - # 0ms no-ops. Script-emitted child steps (per-manifest applies) are - # unaffected. default: false - # Per-provider identity: jobs are scoped to the scope's service - # specification, so two providers sharing a workflow name never - # converge on one definition. job: name: k8s-deployment-delete namespace: "@context:scope.provider" - # Identity labels, the same grammar the deploy workflows declare: `entity` + - # `scope.provider` scope the search, and `operation` names the lifecycle act this - # job performs — the word the deployment entity's own statuses use. They are how - # the deployment page finds this plan BEFORE the run exists: a cancel shows the - # rollback checklist pending the moment the entity says the wind-down began, - # instead of an empty canvas until the agent picks the command up. labels: entity: deployment operation: delete diff --git a/k8s/deployment/workflows/diagnose.yaml b/k8s/deployment/workflows/diagnose.yaml index de0606a4..e15f33e5 100644 --- a/k8s/deployment/workflows/diagnose.yaml +++ b/k8s/deployment/workflows/diagnose.yaml @@ -1,8 +1,4 @@ trace: - # What a HUMAN calls this execution. It lands on the run's explain.title — the - # first rung of every consumer's naming contract. Without it a consumer has only - # the run id, and prints its own neutral word: this run rendered as - # "Activity · N steps", which names nothing the reader can act on. title: Diagnose the failed deployment continue_on_error: true include: @@ -35,8 +31,6 @@ steps: name: notify_check_running type: script file: "$SERVICE_PATH/diagnose/notify_check_running" - # Bookkeeping around each check — the CHECKS are the diagnosis's story; - # two notify steps per check would triple every disclosure. trace: false after_each: name: notify_check_results diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index 0ce7b0b9..7198acd3 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -4,28 +4,11 @@ configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" trace: title: Finalize deployment - # The scope's deploy strategy is a flavor DIMENSION: steps that only one - # strategy can ever run gate on it, so a checklist never advertises work - # this execution cannot do. Absent config reads as not-rolling, which is - # the same default the scripts take. flavors: ["$K8S_FLAVOR", "$DNS_TYPE", "@context:providers.scope-configurations.deployment.deployment_strategy"] - # Declared steps only: engine fragments without a trace block (override - # plumbing) stay off the wire — the fold shows real observable acts, not - # 0ms no-ops. Script-emitted child steps (per-manifest applies) are - # unaffected. default: false - # Per-provider identity: jobs are scoped to the scope's service - # specification, so two providers sharing a workflow name never - # converge on one definition. job: name: k8s-deployment-finalize namespace: "@context:scope.provider" - # Identity labels, the same grammar the deploy workflows declare: `entity` + - # `scope.provider` scope the search, and `operation` names the lifecycle act this - # job performs — the word the deployment entity's own statuses use. They are how - # the deployment page finds this plan BEFORE the run exists: a cancel shows the - # rollback checklist pending the moment the entity says the wind-down began, - # instead of an empty canvas until the agent picks the command up. labels: entity: deployment operation: finalize @@ -75,8 +58,6 @@ steps: trace: title: Promote new deployment description: Scales the new version up to full capacity before the old one is removed. - # ROLLING ONLY: every other strategy takes its instance counts from the - # manifests, so this step has nothing to do and never enters their plan. flavors: [rolling] group: finalize type: script @@ -87,8 +68,6 @@ steps: file: "$SERVICE_PATH/deployment/wait_deployment_active" trace: key: finalize-instances-check - # The same wait, in THIS phase's words: full capacity must be healthy - # before the previous version is removed. title: Verify final capacity description: Confirms every instance of the new version is healthy before the old one goes away. group: finalize diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index b4464c4a..a26cbd31 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -2,19 +2,10 @@ include: - "$SERVICE_PATH/values.yaml" configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" -# Trace identities speak the platform's deploy step vocabulary (the actual -# create-* steps are emitted per applied manifest by apply_templates, under -# the apply step). Step NAMES never change — overrides anchor on them. trace: title: Initial deployment flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - # Declared steps only: engine fragments without a trace block (override - # plumbing) stay off the wire — the fold shows real observable acts, not - # 0ms no-ops. Script-emitted child steps (per-manifest applies) are - # unaffected. default: false - # Identity labels: how the deployment page finds this plan before any run - # exists (see blue_green.yaml). job: name: k8s-deployment-initial namespace: "@context:scope.provider" @@ -25,7 +16,6 @@ trace: groups: - {key: setting-up, title: Setting up} - {key: waiting-instances, title: Waiting for instances to be healthy} - # Placeholder: the finalize workflow fills this station later. - {key: finalize, title: Finalize} steps: - name: load logging @@ -126,8 +116,6 @@ steps: flavors: [route53, external_dns] configuration: VERIFY_WEIGHTS: false - # Metrics publishing is machine bookkeeping — untraced, like the - # per-increment copy in switch_traffic. - name: publish_alb_metrics type: script file: "$SERVICE_PATH/deployment/publish_alb_metrics" diff --git a/k8s/deployment/workflows/kill_instance.yaml b/k8s/deployment/workflows/kill_instance.yaml index a7321054..b3942a7c 100644 --- a/k8s/deployment/workflows/kill_instance.yaml +++ b/k8s/deployment/workflows/kill_instance.yaml @@ -1,8 +1,4 @@ trace: - # What a HUMAN calls this execution. It lands on the run's explain.title — the - # first rung of every consumer's naming contract. Without it a consumer has only - # the run id, and prints its own neutral word: this run rendered as - # "Activity · N steps", which names nothing the reader can act on. title: Kill an instance include: - "$SERVICE_PATH/values.yaml" diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index afff1b77..01c336f1 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -4,26 +4,11 @@ configuration: INGRESS_TEMPLATE: "$INITIAL_INGRESS_PATH" trace: title: Roll back deployment - # The deploy strategy is a flavor dimension: a step only one strategy can - # run never enters another's checklist (see finalize.yaml). flavors: ["$K8S_FLAVOR", "$DNS_TYPE", "@context:providers.scope-configurations.deployment.deployment_strategy"] - # Declared steps only: engine fragments without a trace block (override - # plumbing) stay off the wire — the fold shows real observable acts, not - # 0ms no-ops. Script-emitted child steps (per-manifest applies) are - # unaffected. default: false - # Per-provider identity: jobs are scoped to the scope's service - # specification, so two providers sharing a workflow name never - # converge on one definition. job: name: k8s-deployment-rollback namespace: "@context:scope.provider" - # Identity labels, the same grammar the deploy workflows declare: `entity` + - # `scope.provider` scope the search, and `operation` names the lifecycle act this - # job performs — the word the deployment entity's own statuses use. They are how - # the deployment page finds this plan BEFORE the run exists: a cancel shows the - # rollback checklist pending the moment the entity says the wind-down began, - # instead of an empty canvas until the agent picks the command up. labels: entity: deployment operation: rollback @@ -75,8 +60,6 @@ steps: trace: title: Restore previous deployment description: Scales the previous version back up to full capacity. - # ROLLING ONLY, like finalize's promote step: other strategies never - # scaled anything down, so there is nothing to restore. flavors: [rolling] group: finalize - name: rollback traffic diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index e1affd3f..a22b83cd 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -2,27 +2,13 @@ include: - "$SERVICE_PATH/values.yaml" configuration: INGRESS_TEMPLATE: "$BLUE_GREEN_INGRESS_PATH" -# Trace identities speak the platform's deploy step vocabulary. Step NAMES -# never change — overrides anchor on them. trace: title: Switch traffic flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - # This workflow runs ONCE PER TRAFFIC INCREMENT (ten times in a 10%-step - # switch): declared steps only, so injected/override plumbing can never - # multiply into noise on the wire. default: false - # Per-provider identity: jobs are scoped to the scope's service - # specification, so two providers sharing a workflow name never - # converge on one definition. job: name: k8s-deployment-switch-traffic namespace: "@context:scope.provider" - # Identity labels, the same grammar the deploy workflows declare: `entity` + - # `scope.provider` scope the search, and `operation` names the lifecycle act this - # job performs — the word the deployment entity's own statuses use. They are how - # the deployment page finds this plan BEFORE the run exists: a cancel shows the - # rollback checklist pending the moment the entity says the wind-down began, - # instead of an empty canvas until the agent picks the command up. labels: entity: deployment operation: switch-traffic @@ -68,8 +54,6 @@ steps: type: environment - name: BLUE_DEPLOYMENT_ID type: environment - # Scaling is a no-op for blue-green (counts ride the manifests) and runs - # once per increment — no milestone; the post's health check is the story. - name: create deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" @@ -79,8 +63,6 @@ steps: file: "$SERVICE_PATH/deployment/wait_deployment_active" trace: key: switch-instances-check - # The same wait, in THIS phase's words: green must hold its scaled - # replicas healthy before any weight shifts. title: Verify scaled instances group: switching-traffic configuration: @@ -96,7 +78,6 @@ steps: - name: INGRESS_FILE type: file file: "$OUTPUT_DIR/ingress-$SCOPE_ID-$DEPLOYMENT_ID.yaml" - # Same no-op-for-blue-green scaling — untraced under default: false. - name: update blue deployment type: script file: "$SERVICE_PATH/deployment/scale_deployments" @@ -124,7 +105,6 @@ steps: flavors: [route53, external_dns] configuration: VERIFY_WEIGHTS: true - # Metrics publishing is machine bookkeeping, once per increment. - name: publish_alb_metrics type: script file: "$SERVICE_PATH/deployment/publish_alb_metrics" diff --git a/k8s/diagnose/tests/diagnose_utils.bats b/k8s/diagnose/tests/diagnose_utils.bats index 9cb6e00e..bf68f63b 100644 --- a/k8s/diagnose/tests/diagnose_utils.bats +++ b/k8s/diagnose/tests/diagnose_utils.bats @@ -422,7 +422,6 @@ strip_ansi() { # --- the verdict reaches the TRACE, not only the results file ----------------- -# Stand in for the logging shims so the test can see exactly what the check said. _stub_trace() { np_scope_explain() { echo "EXPLAIN $*" >> "$TRACE_LOG"; } np_scope_output() { echo "OUTPUT $1" >> "$TRACE_LOG"; } @@ -440,7 +439,6 @@ _stub_trace() { grep -q -- "--severity error" "$TRACE_LOG" grep -q -- "OOMKilled containers" "$TRACE_LOG" grep -q -- "--next Increase memory limits" "$TRACE_LOG" - # The evidence rides along so the dialog can show the affected pods. grep -q "^OUTPUT check_evidence" "$TRACE_LOG" rm -f "$TRACE_LOG" } @@ -467,7 +465,6 @@ _stub_trace() { } @test "diagnose_utils: the results file is still written when the workflow is untraced" { - # No np_scope_* in scope: the mirror is a no-op and the check still records. evidence=$(evidence_json "1 of 1 pod(s) had OOMKilled containers" "critical" '["pod-a"]' '{}' '[]') run update_check_result --status "failed" --evidence "$evidence" diff --git a/k8s/diagnose/utils/diagnose_utils b/k8s/diagnose/utils/diagnose_utils index 92fbc9f8..ed327e92 100644 --- a/k8s/diagnose/utils/diagnose_utils +++ b/k8s/diagnose/utils/diagnose_utils @@ -322,23 +322,9 @@ update_check_result() { mv "$tmpfile" "$output_file" - # ...and say the same thing on the TRACE. A check's verdict lived only in this - # file, read by the diagnose API — so a reader on the deployment page saw 21 - # steps that exited 0 and a summary reading "all succeeded", while the check - # that had just found the OOM sat there green. The evidence is already built by - # the caller (summary, severity, recommendations); it only needed saying. - # - # The step's STATUS stays whatever the script did: a check that FOUND a problem - # did its job, and marking it failed would conflate "this check broke" with - # "this check found something". The FINDING rides `explain.severity`, which is - # the same channel every other producer surface uses for attention. _np_trace_check_result "$status_lower" "$evidence" } -# Mirror a check's verdict onto its traced step: the finding as the step's -# narrative, its first recommendation as what to do, and the whole evidence as an -# output so the dialog can show the affected pods and details. A no-op when the -# workflow is untraced (the logging shims return 0 without a trace). _np_trace_check_result() { local _cr_status="$1" _cr_evidence="$2" _cr_severity="" _cr_summary="" _cr_action="" command -v np_scope_explain >/dev/null 2>&1 || return 0 @@ -346,8 +332,6 @@ _np_trace_check_result() { case "$_cr_status" in failed) _cr_severity="error" ;; warning) _cr_severity="warn" ;; - # A passing or skipped check is not news: it would turn a clean diagnosis - # into 21 lines of green noise, which is how nothing gets read. *) return 0 ;; esac diff --git a/k8s/logging b/k8s/logging index ddd32c5e..2d1c62cf 100644 --- a/k8s/logging +++ b/k8s/logging @@ -39,46 +39,11 @@ log() { fi fi - # Every `log error`, anywhere in any script, also lands on the TRACE — as a - # tracing.error facet on the workflow step it happened in. This is what makes - # a failed custom scope read like a failed native one: not just "step exited - # 1", but the actual message. A no-op unless the platform launched this - # workflow traced (see the tracing section below). if [ "$msg_num" -ge 3 ]; then _np_scopes_trace_error "$message" || true fi } -# ============================================================================= -# Tracing — best-effort, structural -# -# When the platform launches a workflow, NP_TRACE carries the trace context of -# the step each fragment runs inside (the np CLI re-points it per step). With -# the vendored shell SDK sourced: -# -# • every `log error` lands as a tracing.error facet ON that step -# • an uncaught command failure surfaces the failing command itself -# • long waits report live progress (np_scope_wait_heartbeat) -# • a phase inside a script can be its own SUB-STEP in the trace -# (np_scope_step_begin / np_scope_step_end), keyed and — for things like -# progressive traffic switches — iterated, exactly like native scopes -# -# All of it is best-effort: no SDK file, no NP_API_KEY, or no NP_TRACE means -# plain logging, byte-identical to before. A down tracing API never blocks the -# workflow: every emit is a local file write, and flushes are hard-bounded. -# ============================================================================= - -# Resolve the node observations attach to: the open sub-step when one belongs -# to the CURRENT platform step, else the adoption of NP_TRACE. The adopted -# handle is CACHED per NP_TRACE value: successive observations on the same -# platform step must accumulate in ONE bag, because every foreign re-emit -# carries the handle's full current bag and the API folds each facet -# last-writer-wins — fresh handles per call would leave only the last fact -# standing (and a lost tail event would take its fact with it; a cumulative -# bag re-carries earlier facts on every later emit). NP_TRACE changes as the -# CLI moves between steps, so the cache keys on it — an observation always -# attaches to the step current at the moment it happened, and a sub-step or -# adoption from a PREVIOUS platform step must never swallow it. _np_scopes_node() { command -v np_trace_adopt >/dev/null 2>&1 || return 1 [ -n "${NP_TRACE:-}" ] || return 1 @@ -87,12 +52,8 @@ _np_scopes_node() { printf '%s' "$_NP_SCOPES_SUBSTEP" return 0 fi - # The platform moved on with a sub-step still open — stale; forget it. _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" fi - # The cache lives in the SDK's state dir, not a shell variable: every np_scope_* - # helper resolves the node inside a command substitution, where a variable write - # dies with the subshell — a FILE is the only memory all those subshells share. local _nd_node _nd_under _nd_cached if [ -n "${NP_TRACE_DIR:-}" ] && [ -f "$NP_TRACE_DIR/scopes_adopted" ]; then IFS=' ' read -r _nd_under _nd_cached < "$NP_TRACE_DIR/scopes_adopted" 2>/dev/null || true @@ -110,25 +71,13 @@ _np_scopes_node() { return 0 } -# Record an observed failure on the current step (or open sub-step). -# -# A failure is usually a BURST of log errors: the cause first, then the -# "possible causes" and "how to fix" hint lines. Facets fold last-writer-wins, -# so naively recording each line would leave the LAST HINT as the step's -# error and bury the cause. Instead: the burst's FIRST message is the error, -# and every later line of the same step accumulates into the error's -# structured details.hints — the cause leads, the guidance travels with it. _np_scopes_trace_error() { local _lt_node _lt_message _lt_node=$(_np_scopes_node) || return 0 - # The console line arrives verbatim; the trace carries the CAUSE, so strip - # the console decoration (leading indentation and the ❌ marker) first. _lt_message="$1" _lt_message="${_lt_message#"${_lt_message%%[![:space:]]*}"}" case "$_lt_message" in "❌ "*) _lt_message="${_lt_message#❌ }" ;; esac if [ "${_NP_SCOPES_ERRED_ON:-}" = "${NP_TRACE:-}" ] && [ -n "${_NP_SCOPES_ERR_MESSAGE:-}" ]; then - # Same step, later line: a hint. Re-emit the whole error with the - # original cause as the message and the growing hint list as evidence. if [ -n "${_NP_SCOPES_ERR_HINTS:-}" ]; then _NP_SCOPES_ERR_HINTS="$_NP_SCOPES_ERR_HINTS,$(np__json_str "$_lt_message")" else @@ -139,18 +88,9 @@ _np_scopes_trace_error() { return 0 fi np_trace_error "$_lt_node" --message "$_lt_message" ${2:+--code "$2"} - # State the cause to the workflow engine too (np_step_error is published by - # the engine's script preamble): if this step later exits nonzero, the - # engine's own failure report carries THIS message instead of a bare exit - # status — one diagnosis on the wire, no matter which side reports last. - # Only REAL diagnoses state: the exit trap's synthetic mechanism message - # must not outrank the engine's own evidence (the shell's stderr words). if [ -z "${_NP_SCOPES_TRAP_REPORT:-}" ] && command -v np_step_error >/dev/null 2>&1; then np_step_error "$_lt_message" fi - # Remember which step already carries a real message (so the exit trap does - # not shadow it with a generic one) AND the message itself — on a fatal - # exit, the run-level mirror repeats the real diagnosis, not the mechanism. _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" _NP_SCOPES_LAST_REASON="$_lt_message" _NP_SCOPES_ERR_MESSAGE="$_lt_message" @@ -159,13 +99,6 @@ _np_scopes_trace_error() { } # np_scope_step_begin [--iteration N] [--attempt N] [--title ] -# -# Open a keyed sub-step under the step the platform is running — a phase that -# deserves its own line in the trace (a long wait, one traffic increment). The -# key names WHAT the phase is (`[A-Za-z0-9_.-]+`); --iteration distinguishes -# repeats of the same phase (traffic at 10, then 50, then 100). One sub-step -# open at a time: while open, `log error` and heartbeats attach to it, and -# np_scope_step_end closes it. Always defined; a no-op when untraced. np_scope_step_begin() { command -v np_trace_adopt >/dev/null 2>&1 || return 0 [ -n "${NP_TRACE:-}" ] || return 0 @@ -178,9 +111,6 @@ np_scope_step_begin() { *) _sb_args+=("$1"); shift ;; esac done - # A still-open sub-step would silently become the parent — close it instead: - # phases at this level are sequential, and a dangling open one is a bug here, - # not a hierarchy. [ -n "${_NP_SCOPES_SUBSTEP:-}" ] && np_scope_step_end 0 _sb_parent=$(np_trace_adopt 2>/dev/null) || return 0 [ -n "$_sb_parent" ] || return 0 @@ -193,9 +123,6 @@ np_scope_step_begin() { } # np_scope_step_end [rc] [message] -# -# Close the open sub-step: completed on rc 0, failed (with the message) on -# anything else. Flushed immediately so the phase transition is visible live. np_scope_step_end() { [ -n "${_NP_SCOPES_SUBSTEP:-}" ] || return 0 local _se_h="$_NP_SCOPES_SUBSTEP" _se_rc="${1:-0}" @@ -203,9 +130,6 @@ np_scope_step_end() { if [ "$_se_rc" -eq 0 ] 2>/dev/null; then np_trace_complete "$_se_h" else - # Only fabricate a generic message when the step does not already carry a - # real one (from `log error` or the exit trap) — never shadow the actual - # failure with "phase exited with status 1". if [ -n "${2:-}" ]; then np_trace_fail "$_se_h" "$2" elif [ "${_NP_SCOPES_ERRED_ON:-}" != "${NP_TRACE:-}" ]; then @@ -220,10 +144,6 @@ np_scope_step_end() { } # np_scope_step_timeout [message] -# -# Close the open sub-step as timed_out — the truthful terminal state for a -# wait that hit its deadline (distinct from failed: nothing broke, time ran -# out). The message, when given, is recorded as the step's error detail. np_scope_step_timeout() { [ -n "${_NP_SCOPES_SUBSTEP:-}" ] || return 0 local _st_h="$_NP_SCOPES_SUBSTEP" @@ -235,21 +155,10 @@ np_scope_step_timeout() { } # np_scope_wait_heartbeat [state] [k=v ...] -# -# Mark the current step (or open sub-step) `waiting` and declare WHAT it -# waits on as the tracing.signal facet — never as labels: tags are queryable -# filing, and per-poll diagnostics rendered as label chips are noise (counts -# ride the meter/progress facets; elapsed rides the node's own clock). The -# state word and extra k=v pairs are accepted for call-site compatibility and -# deliberately not recorded. The flush is bounded to 2s per beat: with the -# API down, a 30s-cadence wait loses at most ~6% of its poll budget — the -# deadline is wall-clock, so the timeout is never extended. Always defined; -# a no-op when the workflow is untraced. np_scope_wait_heartbeat() { local _hb_node _hb_node=$(_np_scopes_node) || return 0 local _hb_what="${1:-}" _hb_timeout="${3:-0}" - # A non-numeric timeout (an unresolved configuration) just drops the deadline. case "$_hb_timeout" in '' | *[!0-9]*) np_trace_signal "$_hb_node" "$_hb_what" wait ;; *) np_trace_signal "$_hb_node" "$_hb_what" wait --timeout-ms $(( _hb_timeout * 1000 )) ;; @@ -261,17 +170,6 @@ np_scope_wait_heartbeat() { # np_scope_produces [ ] # np_scope_consumes [ ] -# -# Data lineage: declare what the current step (or open sub-step) WROTE or -# READ, by CANONICAL dataset id — the exact grammar the scope-workflow-manager -# uses, so a custom scope's lineage joins the platform's by value: -# -# k8s-namespace: k8s-deployment:/ -# k8s-service:/ k8s-ingress:/ -# dns-record: load-balancer: -# docker-image: deployment-log:// -# -# Always defined; a no-op when the workflow is untraced. np_scope_produces() { command -v np_trace_produces >/dev/null 2>&1 || return 0 local _pd_node @@ -289,9 +187,6 @@ np_scope_consumes() { } # np_scope_affordance -# -# What the current step OFFERS a human to do — the UI renders it as a control -# (e.g. '{"kind":"deploy-log","application_id":"7","scope_id":"42",...}'). np_scope_affordance() { command -v np_trace_affordances >/dev/null 2>&1 || return 0 local _ad_node @@ -301,8 +196,6 @@ np_scope_affordance() { } # np_scope_progress [unit] -# -# A converging phase's advance toward its target (instances 3 of 10). np_scope_progress() { command -v np_trace_progress >/dev/null 2>&1 || return 0 local _pg_node @@ -312,11 +205,6 @@ np_scope_progress() { } # np_scope_output / np_scope_input -# -# Record what the current step (or open sub-step) produced or consumed as an -# inline value carried in the event — the counted evidence a host renders -# next to the narrative ('instances' with healthy/desired, 'traffic' with the -# switched level). np_scope_output() { command -v np_trace_output >/dev/null 2>&1 || return 0 local _ot_node @@ -334,10 +222,6 @@ np_scope_input() { } # np_scope_explain --title T [--what W] [--severity ok|warn|error] ... -# -# The plain-language narrative on the current step (or open sub-step) — what -# the phase is doing, in words, with the severity an operator should read it -# at. Re-emitted per poll on waits so the latest state wins. np_scope_explain() { command -v np_trace_explain >/dev/null 2>&1 || return 0 local _ex_node @@ -347,8 +231,6 @@ np_scope_explain() { } # np_scope_error [] -# -# An observed failure with its structured evidence. np_scope_error() { command -v np_trace_error >/dev/null 2>&1 || return 0 local _sr_node @@ -368,10 +250,6 @@ np_scope_labels() { } # np_scope_k8s_applied -# -# Turn `kubectl apply` output lines (`deployment.apps/name created`) into -# lineage on the current step, for the resource kinds the platform's lineage -# model knows. One chokepoint (apply_templates) covers every workflow. np_scope_k8s_applied() { command -v np_trace_produces >/dev/null 2>&1 || return 0 local _ka_ns="$1" _ka_line _ka_kind _ka_name @@ -395,11 +273,6 @@ np_scope_k8s_applied() { # np_scope_k8s_deleted -# -# The removal counterpart of np_scope_k8s_applied: record what a delete removed -# ("deployment.apps \"name\" deleted" lines) as pointers on the planned step, so a -# finalize's "Remove previous deployment" (or a rollback's "Remove new deployment") -# names what it removed. np_scope_k8s_deleted() { command -v np_trace_output >/dev/null 2>&1 || return 0 local _kd_ns="$1" _kd_node _kd_line _kd_kind _kd_name @@ -427,18 +300,10 @@ np_scope_k8s_deleted() { return 0 } -# A silent failure (no `log error` on the way down) must still be clear: the -# ERR trap remembers the last failing top-level command, and the EXIT trap -# reports it against the step that was current when the shell died. _np_scopes_on_err() { _NP_SCOPES_LAST_ERR="$BASH_COMMAND (exit ${1:-1})" } -# On a fatal exit the RUN's terminal deserves the real reason too: the step -# carries it already; this mirrors it one level up, so list views (which lead -# with the run) name the cause without drilling into steps. The run id is the -# step path minus its last segment — an observed fact on an adopted node, -# like every other enrichment here. _np_scopes_error_on_run() { command -v np_trace_adopt >/dev/null 2>&1 || return 0 [ -n "${NP_TRACE:-}" ] || return 0 @@ -466,16 +331,11 @@ _np_scopes_on_exit() { if [ "$_ex_rc" -ne 0 ]; then _np_scopes_error_on_run "$_ex_reason" || true fi - # A sub-step still open when the shell dies inherits the shell's outcome — - # closed AFTER the error report above so the failure detail lands on it. [ -n "${_NP_SCOPES_SUBSTEP:-}" ] && np_scope_step_end "$_ex_rc" || true np_trace_flush } _NP_SCOPES_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# The SDK ships as the catalog-tracing-sh submodule; a repo copy that lost the -# submodule (archive download, docker build context) may carry the single file -# vendored at the root instead. Neither present -> plain logging, untraced. _NP_SCOPES_SDK="$_NP_SCOPES_ROOT/vendor/catalog-tracing-sh/nptrace.sh" [ -f "$_NP_SCOPES_SDK" ] || _NP_SCOPES_SDK="$_NP_SCOPES_ROOT/nptrace.sh" if [ -z "${NP_TRACE_LOADED:-}" ] \ @@ -484,8 +344,6 @@ if [ -z "${NP_TRACE_LOADED:-}" ] \ && [ -n "${NP_TRACE:-}" ]; then # shellcheck source=/dev/null . "$_NP_SCOPES_SDK" - # --no-trap: the exit flush is ours, so the uncaught-failure report and the - # flush share ONE trap in a defined order. np_trace_init --producer "nullplatform-scopes@1" --api-key "$NP_API_KEY" --no-trap trap '_np_scopes_on_err $?' ERR trap '_np_scopes_on_exit $?' EXIT @@ -493,10 +351,5 @@ elif [ -z "${NP_TRACE_LOADED:-}" ] \ && [ -n "${NP_TRACE:-}" ] \ && [ -n "${NP_API_KEY:-}" ] \ && [ ! -f "$_NP_SCOPES_SDK" ]; then - # The engine is tracing this run and the credentials are present, yet the shell - # SDK is not in this bundle: the scope-side story (sub-steps, live waits, real - # errors, lineage, affordances) will be ABSENT while everything else looks fine. - # A build that loses the submodule (archive download, docker build context) is - # exactly the failure nobody sees — so say it, once, on the workflow's own log. log warn "⚠️ tracing SDK not bundled (vendor/catalog-tracing-sh/nptrace.sh missing) — scope-side tracing disabled for this run" fi diff --git a/k8s/scope/build_context b/k8s/scope/build_context index 4d3cffef..8682995a 100755 --- a/k8s/scope/build_context +++ b/k8s/scope/build_context @@ -142,8 +142,6 @@ if ! kubectl get namespace "$K8S_NAMESPACE" &> /dev/null; then fi else log info " ✅ Namespace '$K8S_NAMESPACE' exists" - # This step's trace identity is create-namespace: when the namespace - # already exists, that branch wasn't taken — close as `skipped`. if command -v np_step_skip >/dev/null 2>&1; then np_step_skip "namespace '$K8S_NAMESPACE' already exists" fi diff --git a/k8s/scope/iam/create_role b/k8s/scope/iam/create_role index bec4ef3b..896ead48 100644 --- a/k8s/scope/iam/create_role +++ b/k8s/scope/iam/create_role @@ -127,8 +127,6 @@ if [[ -n "$BOUNDARY_ARN" && "$BOUNDARY_ARN" != "null" ]]; then CREATE_ROLE_ARGS+=(--permissions-boundary "$BOUNDARY_ARN") fi -# Captured (with stderr) so a failure carries AWS's actual reason onto the -# trace; on success the response is echoed exactly as before. CREATE_ROLE_OUT=$(aws iam create-role "${CREATE_ROLE_ARGS[@]}" 2>&1) || create_role_error [[ -n "$CREATE_ROLE_OUT" ]] && echo "$CREATE_ROLE_OUT" log info " ✅ IAM role created successfully" diff --git a/k8s/scope/networking/dns/manage_dns b/k8s/scope/networking/dns/manage_dns index f52e3fd0..82540a29 100755 --- a/k8s/scope/networking/dns/manage_dns +++ b/k8s/scope/networking/dns/manage_dns @@ -71,8 +71,6 @@ esac log info "✅ DNS records managed successfully" -# The DNS record joins the lineage under its real-world address, so any other -# operation naming the same FQDN links to it. Only a CREATE writes the record. if [ "${ACTION:-}" = "CREATE" ] && [ -n "${SCOPE_DOMAIN:-}" ] \ && command -v np_scope_produces >/dev/null 2>&1; then np_scope_produces "dns-record:$SCOPE_DOMAIN" dns_record "$SCOPE_DOMAIN" diff --git a/k8s/scope/networking/wait_for_alb b/k8s/scope/networking/wait_for_alb index 74207df8..e586493f 100644 --- a/k8s/scope/networking/wait_for_alb +++ b/k8s/scope/networking/wait_for_alb @@ -24,8 +24,6 @@ # time out and fail the scope creation. if [ "${DNS_TYPE:-}" != "route53" ]; then log debug "📋 DNS type is '${DNS_TYPE:-unset}', skipping ALB active-state wait" - # Tell the platform this step's branch wasn't taken: it closes as - # `skipped`, not `completed`. (Guarded: defined by the np CLI's preamble.) if command -v np_step_skip >/dev/null 2>&1; then np_step_skip "no ALB on DNS type '${DNS_TYPE:-unset}'" fi @@ -54,10 +52,6 @@ polls_since_heartbeat=0 heartbeats_emitted=0 log info "⏳ Waiting up to ${TIMEOUT_SECONDS}s for ALB '$ALB_NAME' to become active..." -# The wait is its own SUB-STEP in the trace, flipped to `waiting` right away — -# an operator watching the provision sees WHAT it is blocked on, live, not -# after the workflow ends. (Guarded: overrides may reuse this script without -# k8s/logging loaded.) if command -v np_scope_step_begin >/dev/null 2>&1; then np_scope_step_begin wait-alb-active --title "Wait for ALB '$ALB_NAME' to become active" np_scope_wait_heartbeat "alb-active" 0 "$TIMEOUT_SECONDS" "pending" @@ -117,14 +111,10 @@ if [ "$state" != "active" ]; then exit 1 fi -# The scope depends on this ALB: record the lineage under its ARN — the same -# address the platform's own scopes name — before closing the wait sub-step. if [ -n "$alb_arn" ] && command -v np_scope_consumes >/dev/null 2>&1; then np_scope_consumes "load-balancer:$alb_arn" load_balancer "$alb_arn" fi -# The wait phase is over and the ALB is active — close its sub-step; the audit -# tagging below is not part of the wait. if command -v np_scope_step_end >/dev/null 2>&1; then np_scope_step_end 0 fi diff --git a/k8s/scope/wait_on_balancer b/k8s/scope/wait_on_balancer index 2e45db78..3e0cee1d 100644 --- a/k8s/scope/wait_on_balancer +++ b/k8s/scope/wait_on_balancer @@ -18,8 +18,6 @@ case "$DNS_TYPE" in log debug "📋 Checking ExternalDNS record creation for domain: $SCOPE_DOMAIN" - # The DNS wait is its own SUB-STEP in the trace, heartbeating every ~30s. - # (Guarded: overrides may reuse this script without k8s/logging loaded.) if command -v np_scope_step_begin >/dev/null 2>&1; then np_scope_step_begin wait-dns-endpoint --title "Wait for ExternalDNS to process $DNS_ENDPOINT_NAME" np_scope_wait_heartbeat "dns-endpoint" 0 "$((MAX_ITERATIONS * 10))" "pending" diff --git a/k8s/scope/workflows/create.yaml b/k8s/scope/workflows/create.yaml index f43daef1..76bb082c 100644 --- a/k8s/scope/workflows/create.yaml +++ b/k8s/scope/workflows/create.yaml @@ -1,22 +1,11 @@ include: - "$SERVICE_PATH/values.yaml" -# The trace identity of each step (trace: blocks below) speaks the same step -# vocabulary the platform's own scopes emit, so a custom scope's provision -# reads identically in the timeline. Step NAMES never change — overrides -# anchor on them. trace: title: Create scope flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - # Per-provider identity: jobs are scoped to the scope's service - # specification, so two providers sharing a workflow name never - # converge on one definition. job: name: k8s-scope-create namespace: "@context:scope.provider" - # Identity labels, the same grammar the deployment workflows declare: - # `entity` + `scope.provider` scope the search, `operation` names the - # lifecycle act. They are how a consumer finds this plan BEFORE the run - # exists, instead of an empty canvas until the agent picks the command up. labels: entity: scope operation: create @@ -97,7 +86,6 @@ steps: - name: build service account type: script file: "$SERVICE_PATH/scope/iam/build_service_account" - # Template generation — the apply below is the observable act. trace: false configuration: ACTION: create @@ -120,7 +108,6 @@ steps: - name: generate domain type: script file: "$SERVICE_PATH/scope/networking/dns/domain/generate_domain" - # Pure computation — the DNS create below is the observable act. trace: false output: - name: SCOPE_DOMAIN diff --git a/k8s/scope/workflows/delete.yaml b/k8s/scope/workflows/delete.yaml index 934a2de0..8540b1c6 100644 --- a/k8s/scope/workflows/delete.yaml +++ b/k8s/scope/workflows/delete.yaml @@ -3,16 +3,9 @@ include: trace: title: Delete scope flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - # Per-provider identity: jobs are scoped to the scope's service - # specification, so two providers sharing a workflow name never - # converge on one definition. job: name: k8s-scope-delete namespace: "@context:scope.provider" - # Identity labels, the same grammar the deployment workflows declare: - # `entity` + `scope.provider` scope the search, `operation` names the - # lifecycle act. They are how a consumer finds this plan BEFORE the run - # exists, instead of an empty canvas until the agent picks the command up. labels: entity: scope operation: delete @@ -77,7 +70,6 @@ steps: - name: build service account type: script file: "$SERVICE_PATH/scope/iam/build_service_account" - # Template generation — the delete below is the observable act. trace: false configuration: ACTION: delete diff --git a/k8s/scope/workflows/diagnose.yaml b/k8s/scope/workflows/diagnose.yaml index 190e9aef..00eeea78 100644 --- a/k8s/scope/workflows/diagnose.yaml +++ b/k8s/scope/workflows/diagnose.yaml @@ -1,8 +1,4 @@ trace: - # What a HUMAN calls this execution. It lands on the run's explain.title — the - # first rung of every consumer's naming contract. Without it a consumer has only - # the run id, and prints its own neutral word: this run rendered as - # "Activity · N steps", which names nothing the reader can act on. title: Diagnose the scope continue_on_error: true include: @@ -35,7 +31,6 @@ steps: name: notify_check_running type: script file: "$SERVICE_PATH/diagnose/notify_check_running" - # Bookkeeping around each check — the CHECKS are the diagnosis's story. trace: false after_each: name: notify_check_results diff --git a/k8s/scope/workflows/pause-autoscaling.yaml b/k8s/scope/workflows/pause-autoscaling.yaml index 74a5cee5..70d65773 100644 --- a/k8s/scope/workflows/pause-autoscaling.yaml +++ b/k8s/scope/workflows/pause-autoscaling.yaml @@ -1,8 +1,4 @@ trace: - # What a HUMAN calls this execution. It lands on the run's explain.title — the - # first rung of every consumer's naming contract. Without it a consumer has only - # the run id, and prints its own neutral word: this run rendered as - # "Activity · N steps", which names nothing the reader can act on. title: Pause autoscaling include: - "$SERVICE_PATH/values.yaml" diff --git a/k8s/scope/workflows/restart-pods.yaml b/k8s/scope/workflows/restart-pods.yaml index fb37fcb8..e6e40d7b 100644 --- a/k8s/scope/workflows/restart-pods.yaml +++ b/k8s/scope/workflows/restart-pods.yaml @@ -1,8 +1,4 @@ trace: - # What a HUMAN calls this execution. It lands on the run's explain.title — the - # first rung of every consumer's naming contract. Without it a consumer has only - # the run id, and prints its own neutral word: this run rendered as - # "Activity · N steps", which names nothing the reader can act on. title: Restart the instances include: - "$SERVICE_PATH/values.yaml" diff --git a/k8s/scope/workflows/resume-autoscaling.yaml b/k8s/scope/workflows/resume-autoscaling.yaml index dcce1816..21c7640e 100644 --- a/k8s/scope/workflows/resume-autoscaling.yaml +++ b/k8s/scope/workflows/resume-autoscaling.yaml @@ -1,8 +1,4 @@ trace: - # What a HUMAN calls this execution. It lands on the run's explain.title — the - # first rung of every consumer's naming contract. Without it a consumer has only - # the run id, and prints its own neutral word: this run rendered as - # "Activity · N steps", which names nothing the reader can act on. title: Resume autoscaling include: - "$SERVICE_PATH/values.yaml" diff --git a/k8s/scope/workflows/set-desired-instance-count.yaml b/k8s/scope/workflows/set-desired-instance-count.yaml index 99b734c2..a5778f03 100644 --- a/k8s/scope/workflows/set-desired-instance-count.yaml +++ b/k8s/scope/workflows/set-desired-instance-count.yaml @@ -1,8 +1,4 @@ trace: - # What a HUMAN calls this execution. It lands on the run's explain.title — the - # first rung of every consumer's naming contract. Without it a consumer has only - # the run id, and prints its own neutral word: this run rendered as - # "Activity · N steps", which names nothing the reader can act on. title: Set the instance count include: - "$SERVICE_PATH/values.yaml" diff --git a/k8s/scope/workflows/update.yaml b/k8s/scope/workflows/update.yaml index 46036585..a715ea9b 100644 --- a/k8s/scope/workflows/update.yaml +++ b/k8s/scope/workflows/update.yaml @@ -3,16 +3,9 @@ include: trace: title: Update scope flavors: ["$K8S_FLAVOR", "$DNS_TYPE"] - # Per-provider identity: jobs are scoped to the scope's service - # specification, so two providers sharing a workflow name never - # converge on one definition. job: name: k8s-scope-update namespace: "@context:scope.provider" - # Identity labels, the same grammar the deployment workflows declare: - # `entity` + `scope.provider` scope the search, `operation` names the - # lifecycle act. They are how a consumer finds this plan BEFORE the run - # exists, instead of an empty canvas until the agent picks the command up. labels: entity: scope operation: update diff --git a/k8s/utils/tests/trace_logging.bats b/k8s/utils/tests/trace_logging.bats index b4647d48..fcbcf627 100644 --- a/k8s/utils/tests/trace_logging.bats +++ b/k8s/utils/tests/trace_logging.bats @@ -1,15 +1,5 @@ #!/usr/bin/env bats -# ============================================================================= # Unit tests for the tracing hooks in k8s/logging -# -# (Lives under utils/tests because the runner discovers k8s//tests; -# the file under test is k8s/logging.) -# -# The contract under test: with the platform's trace context present, every -# `log error` and every uncaught failure lands ON the workflow step as a -# tracing.error facet, waits report progress — and with anything missing, the -# behavior is byte-identical to plain logging. -# ============================================================================= setup() { export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" @@ -17,9 +7,6 @@ setup() { export LOGGING="$PROJECT_ROOT/k8s/logging" - # A traced environment, as the np CLI provides it: NP_TRACE points at the - # step this fragment runs inside; the state dir is per-test; the endpoint is - # unroutable so nothing ever leaves the machine and flushes fail fast. export NP_TRACE="1|trace-9|scope-provision-42~apply-manifests@0.0" export NP_API_KEY="test-key" export NP_TRACE_DIR="$BATS_TEST_TMPDIR/nptrace" @@ -34,8 +21,6 @@ teardown() { unset NP_TRACE_FLUSH_TIMEOUT NP_TRACE_MAX_RETRIES } -# Run a snippet in a fresh bash with logging sourced, then print every spooled -# envelope (the spool is the wire: it is what the API would receive). run_logged() { run "$BASH" -c " source '$LOGGING' @@ -56,8 +41,6 @@ run_logged() { } @test "the error attaches to the step CURRENT at the moment it happened" { - # NP_TRACE moves between steps; adoption is per-call, so a later error must - # land on the later step. run_logged ' log error "first failure" export NP_TRACE="1|trace-9|scope-provision-42~wait-for-alb@0.0" @@ -98,8 +81,6 @@ run_logged() { " [ "$status" -eq 0 ] echo "$output" | grep -q 'the real reason' - # exactly one error facet ON THE STEP: the generic exit report stood down - # (the run-level mirror is a separate node and is asserted elsewhere) [ "$(echo "$output" | grep 'apply-manifests@0.0"' | grep -c 'tracing.error')" -eq 1 ] } @@ -107,14 +88,10 @@ run_logged() { run_logged 'np_scope_wait_heartbeat "alb-active" 90 300 "pending"' [ "$status" -eq 0 ] echo "$output" | grep -q '"status":"waiting"' - # What is being waited for is a FACET, not labels: a wait is a first-class - # thing every consumer can render, and the deadline rides it in milliseconds. echo "$output" | grep -q '"tracing.signal":{"name":"alb-active","direction":"wait","timeout_ms":300000}' } @test "a wait with an unresolved timeout still states the wait, without a deadline" { - # An unresolved configuration ($DEPLOYMENT_MAX_WAIT_IN_SECONDS arriving - # literal) must not put a nonsense deadline on the wire. run_logged 'np_scope_wait_heartbeat "alb-active" 90 "MAX_WAIT" "pending"' [ "$status" -eq 0 ] echo "$output" | grep -q '"tracing.signal":{"name":"alb-active","direction":"wait"}' @@ -155,8 +132,6 @@ hello" ] # --- sub-steps -------------------------------------------------------------- @test "step_begin opens a keyed sub-step under the platform step" { - # The title rides the next lifecycle emit (own-node enrichment is bagged, - # not emitted eagerly), so close the step to see it on the wire. run_logged 'np_scope_step_begin wait-alb-active --title "Wait for the ALB"; np_scope_step_end 0' [ "$status" -eq 0 ] echo "$output" | grep -q '"run_id":"scope-provision-42~apply-manifests@0.0~wait-alb-active@0.0"' @@ -186,8 +161,6 @@ hello" ] } @test "while a sub-step is open, log error attaches to IT, not the platform step" { - # The facet rides the step's closing emit — and because a real message was - # already recorded, the close adds no generic shadow next to it. run_logged 'np_scope_step_begin wait-alb-active; log error "quota exceeded"; np_scope_step_end 1' [ "$status" -eq 0 ] echo "$output" | grep '"quota exceeded"' | grep -q 'wait-alb-active@0.0' @@ -201,9 +174,6 @@ hello" ] } @test "a heartbeat puts NO wait bookkeeping in the labels" { - # Labels are how a node is FILED, and a consumer renders them as tags. The - # wait's own bookkeeping is not filing metadata — it read as a row of code-y - # chips on the phase, which is what the signal facet replaced. run_logged 'np_scope_wait_heartbeat "deployment-active" 20 600 "progressing"' [ "$status" -eq 0 ] ! echo "$output" | grep -q '"wait\.' @@ -216,7 +186,6 @@ hello" ] log error "late failure" ' [ "$status" -eq 0 ] - # the error lands on the NEW platform step, not the stale sub-step echo "$output" | grep '"late failure"' | grep -q 'scope-provision-42~wait-for-alb@0.0"' } @@ -232,7 +201,6 @@ hello" ] [ "$status" -eq 0 ] echo "$output" | grep '"status":"failed"' | grep -q 'wait-alb-active@0.0' echo "$output" | grep '"the real reason"' | grep -q 'wait-alb-active@0.0' - # no generic 'phase exited' shadow next to the real message ! echo "$output" | grep -q 'phase exited with status' } @@ -262,10 +230,7 @@ hello" ] run_logged 'np_scope_produces "dns-record:api.example.com" dns_record "api.example.com"' [ "$status" -eq 0 ] echo "$output" | grep '"edge.produces"' | grep -q '"id":"dns-record:api.example.com"' - # The edge binding says WHICH io of the node the edge is about — a name, and - # nothing else. The descriptor itself stays whole on the node's io facet. echo "$output" | grep -q '"tracing.binding":{"name":"dns_record"}' - # foreign re-emit carries the io facet on the adopted step echo "$output" | grep '"tracing.output"' | grep -q 'apply-manifests@0.0' } @@ -296,7 +261,6 @@ secret/sec-1 created" echo "$output" | grep -q '"id":"k8s-deployment:ns-42/d-1-2"' echo "$output" | grep -q '"id":"k8s-service:ns-42/s-1-2"' echo "$output" | grep -q '"id":"k8s-ingress:ns-42/i-1-2"' - # kinds outside the platform lineage model are not datasets ! echo "$output" | grep -q 'sec-1' } @@ -307,8 +271,6 @@ secret/sec-1 created" ' [ "$status" -eq 0 ] echo "$output" | grep -q '"tracing.affordances":\[{"kind":"deploy-log","application_id":"7"}\]' - # The unit is a CLOSED set every consumer can render (percent/count/bytes/ - # milliseconds) — never a free-text noun the reader has to interpret. echo "$output" | grep -q '"tracing.progress":{"current":3,"target":10,"unit":"count"}' } @@ -383,9 +345,7 @@ secret/sec-1 created" true " [ "$status" -eq 0 ] - # the step carries it... echo "$output" | grep '"the real reason"' | grep -q 'scope-provision-42~apply-manifests@0.0"' - # ...and so does the run (the step path minus its last segment) echo "$output" | grep '"the real reason"' | grep -q '"run_id":"scope-provision-42"' } @@ -410,8 +370,6 @@ secret/sec-1 created" log error " • The role lacks route53:ListHostedZones" ' [ "$status" -eq 0 ] - # every emission of the burst carries the CAUSE as the message — with the - # console decoration (indentation, ❌ marker) stripped for the trace ! echo "$output" | grep '"tracing.error"' | grep -q '"message":"💡' echo "$output" | grep -q '"message":"HostedZone not found (AccessDenied)","details":{"hints":\["💡 Possible causes:","• The role lacks route53:ListHostedZones"\]}' } diff --git a/scheduled_task/logging b/scheduled_task/logging index ddd32c5e..2d1c62cf 100644 --- a/scheduled_task/logging +++ b/scheduled_task/logging @@ -39,46 +39,11 @@ log() { fi fi - # Every `log error`, anywhere in any script, also lands on the TRACE — as a - # tracing.error facet on the workflow step it happened in. This is what makes - # a failed custom scope read like a failed native one: not just "step exited - # 1", but the actual message. A no-op unless the platform launched this - # workflow traced (see the tracing section below). if [ "$msg_num" -ge 3 ]; then _np_scopes_trace_error "$message" || true fi } -# ============================================================================= -# Tracing — best-effort, structural -# -# When the platform launches a workflow, NP_TRACE carries the trace context of -# the step each fragment runs inside (the np CLI re-points it per step). With -# the vendored shell SDK sourced: -# -# • every `log error` lands as a tracing.error facet ON that step -# • an uncaught command failure surfaces the failing command itself -# • long waits report live progress (np_scope_wait_heartbeat) -# • a phase inside a script can be its own SUB-STEP in the trace -# (np_scope_step_begin / np_scope_step_end), keyed and — for things like -# progressive traffic switches — iterated, exactly like native scopes -# -# All of it is best-effort: no SDK file, no NP_API_KEY, or no NP_TRACE means -# plain logging, byte-identical to before. A down tracing API never blocks the -# workflow: every emit is a local file write, and flushes are hard-bounded. -# ============================================================================= - -# Resolve the node observations attach to: the open sub-step when one belongs -# to the CURRENT platform step, else the adoption of NP_TRACE. The adopted -# handle is CACHED per NP_TRACE value: successive observations on the same -# platform step must accumulate in ONE bag, because every foreign re-emit -# carries the handle's full current bag and the API folds each facet -# last-writer-wins — fresh handles per call would leave only the last fact -# standing (and a lost tail event would take its fact with it; a cumulative -# bag re-carries earlier facts on every later emit). NP_TRACE changes as the -# CLI moves between steps, so the cache keys on it — an observation always -# attaches to the step current at the moment it happened, and a sub-step or -# adoption from a PREVIOUS platform step must never swallow it. _np_scopes_node() { command -v np_trace_adopt >/dev/null 2>&1 || return 1 [ -n "${NP_TRACE:-}" ] || return 1 @@ -87,12 +52,8 @@ _np_scopes_node() { printf '%s' "$_NP_SCOPES_SUBSTEP" return 0 fi - # The platform moved on with a sub-step still open — stale; forget it. _NP_SCOPES_SUBSTEP="" _NP_SCOPES_SUBSTEP_UNDER="" fi - # The cache lives in the SDK's state dir, not a shell variable: every np_scope_* - # helper resolves the node inside a command substitution, where a variable write - # dies with the subshell — a FILE is the only memory all those subshells share. local _nd_node _nd_under _nd_cached if [ -n "${NP_TRACE_DIR:-}" ] && [ -f "$NP_TRACE_DIR/scopes_adopted" ]; then IFS=' ' read -r _nd_under _nd_cached < "$NP_TRACE_DIR/scopes_adopted" 2>/dev/null || true @@ -110,25 +71,13 @@ _np_scopes_node() { return 0 } -# Record an observed failure on the current step (or open sub-step). -# -# A failure is usually a BURST of log errors: the cause first, then the -# "possible causes" and "how to fix" hint lines. Facets fold last-writer-wins, -# so naively recording each line would leave the LAST HINT as the step's -# error and bury the cause. Instead: the burst's FIRST message is the error, -# and every later line of the same step accumulates into the error's -# structured details.hints — the cause leads, the guidance travels with it. _np_scopes_trace_error() { local _lt_node _lt_message _lt_node=$(_np_scopes_node) || return 0 - # The console line arrives verbatim; the trace carries the CAUSE, so strip - # the console decoration (leading indentation and the ❌ marker) first. _lt_message="$1" _lt_message="${_lt_message#"${_lt_message%%[![:space:]]*}"}" case "$_lt_message" in "❌ "*) _lt_message="${_lt_message#❌ }" ;; esac if [ "${_NP_SCOPES_ERRED_ON:-}" = "${NP_TRACE:-}" ] && [ -n "${_NP_SCOPES_ERR_MESSAGE:-}" ]; then - # Same step, later line: a hint. Re-emit the whole error with the - # original cause as the message and the growing hint list as evidence. if [ -n "${_NP_SCOPES_ERR_HINTS:-}" ]; then _NP_SCOPES_ERR_HINTS="$_NP_SCOPES_ERR_HINTS,$(np__json_str "$_lt_message")" else @@ -139,18 +88,9 @@ _np_scopes_trace_error() { return 0 fi np_trace_error "$_lt_node" --message "$_lt_message" ${2:+--code "$2"} - # State the cause to the workflow engine too (np_step_error is published by - # the engine's script preamble): if this step later exits nonzero, the - # engine's own failure report carries THIS message instead of a bare exit - # status — one diagnosis on the wire, no matter which side reports last. - # Only REAL diagnoses state: the exit trap's synthetic mechanism message - # must not outrank the engine's own evidence (the shell's stderr words). if [ -z "${_NP_SCOPES_TRAP_REPORT:-}" ] && command -v np_step_error >/dev/null 2>&1; then np_step_error "$_lt_message" fi - # Remember which step already carries a real message (so the exit trap does - # not shadow it with a generic one) AND the message itself — on a fatal - # exit, the run-level mirror repeats the real diagnosis, not the mechanism. _NP_SCOPES_ERRED_ON="${NP_TRACE:-}" _NP_SCOPES_LAST_REASON="$_lt_message" _NP_SCOPES_ERR_MESSAGE="$_lt_message" @@ -159,13 +99,6 @@ _np_scopes_trace_error() { } # np_scope_step_begin [--iteration N] [--attempt N] [--title ] -# -# Open a keyed sub-step under the step the platform is running — a phase that -# deserves its own line in the trace (a long wait, one traffic increment). The -# key names WHAT the phase is (`[A-Za-z0-9_.-]+`); --iteration distinguishes -# repeats of the same phase (traffic at 10, then 50, then 100). One sub-step -# open at a time: while open, `log error` and heartbeats attach to it, and -# np_scope_step_end closes it. Always defined; a no-op when untraced. np_scope_step_begin() { command -v np_trace_adopt >/dev/null 2>&1 || return 0 [ -n "${NP_TRACE:-}" ] || return 0 @@ -178,9 +111,6 @@ np_scope_step_begin() { *) _sb_args+=("$1"); shift ;; esac done - # A still-open sub-step would silently become the parent — close it instead: - # phases at this level are sequential, and a dangling open one is a bug here, - # not a hierarchy. [ -n "${_NP_SCOPES_SUBSTEP:-}" ] && np_scope_step_end 0 _sb_parent=$(np_trace_adopt 2>/dev/null) || return 0 [ -n "$_sb_parent" ] || return 0 @@ -193,9 +123,6 @@ np_scope_step_begin() { } # np_scope_step_end [rc] [message] -# -# Close the open sub-step: completed on rc 0, failed (with the message) on -# anything else. Flushed immediately so the phase transition is visible live. np_scope_step_end() { [ -n "${_NP_SCOPES_SUBSTEP:-}" ] || return 0 local _se_h="$_NP_SCOPES_SUBSTEP" _se_rc="${1:-0}" @@ -203,9 +130,6 @@ np_scope_step_end() { if [ "$_se_rc" -eq 0 ] 2>/dev/null; then np_trace_complete "$_se_h" else - # Only fabricate a generic message when the step does not already carry a - # real one (from `log error` or the exit trap) — never shadow the actual - # failure with "phase exited with status 1". if [ -n "${2:-}" ]; then np_trace_fail "$_se_h" "$2" elif [ "${_NP_SCOPES_ERRED_ON:-}" != "${NP_TRACE:-}" ]; then @@ -220,10 +144,6 @@ np_scope_step_end() { } # np_scope_step_timeout [message] -# -# Close the open sub-step as timed_out — the truthful terminal state for a -# wait that hit its deadline (distinct from failed: nothing broke, time ran -# out). The message, when given, is recorded as the step's error detail. np_scope_step_timeout() { [ -n "${_NP_SCOPES_SUBSTEP:-}" ] || return 0 local _st_h="$_NP_SCOPES_SUBSTEP" @@ -235,21 +155,10 @@ np_scope_step_timeout() { } # np_scope_wait_heartbeat [state] [k=v ...] -# -# Mark the current step (or open sub-step) `waiting` and declare WHAT it -# waits on as the tracing.signal facet — never as labels: tags are queryable -# filing, and per-poll diagnostics rendered as label chips are noise (counts -# ride the meter/progress facets; elapsed rides the node's own clock). The -# state word and extra k=v pairs are accepted for call-site compatibility and -# deliberately not recorded. The flush is bounded to 2s per beat: with the -# API down, a 30s-cadence wait loses at most ~6% of its poll budget — the -# deadline is wall-clock, so the timeout is never extended. Always defined; -# a no-op when the workflow is untraced. np_scope_wait_heartbeat() { local _hb_node _hb_node=$(_np_scopes_node) || return 0 local _hb_what="${1:-}" _hb_timeout="${3:-0}" - # A non-numeric timeout (an unresolved configuration) just drops the deadline. case "$_hb_timeout" in '' | *[!0-9]*) np_trace_signal "$_hb_node" "$_hb_what" wait ;; *) np_trace_signal "$_hb_node" "$_hb_what" wait --timeout-ms $(( _hb_timeout * 1000 )) ;; @@ -261,17 +170,6 @@ np_scope_wait_heartbeat() { # np_scope_produces [ ] # np_scope_consumes [ ] -# -# Data lineage: declare what the current step (or open sub-step) WROTE or -# READ, by CANONICAL dataset id — the exact grammar the scope-workflow-manager -# uses, so a custom scope's lineage joins the platform's by value: -# -# k8s-namespace: k8s-deployment:/ -# k8s-service:/ k8s-ingress:/ -# dns-record: load-balancer: -# docker-image: deployment-log:// -# -# Always defined; a no-op when the workflow is untraced. np_scope_produces() { command -v np_trace_produces >/dev/null 2>&1 || return 0 local _pd_node @@ -289,9 +187,6 @@ np_scope_consumes() { } # np_scope_affordance -# -# What the current step OFFERS a human to do — the UI renders it as a control -# (e.g. '{"kind":"deploy-log","application_id":"7","scope_id":"42",...}'). np_scope_affordance() { command -v np_trace_affordances >/dev/null 2>&1 || return 0 local _ad_node @@ -301,8 +196,6 @@ np_scope_affordance() { } # np_scope_progress [unit] -# -# A converging phase's advance toward its target (instances 3 of 10). np_scope_progress() { command -v np_trace_progress >/dev/null 2>&1 || return 0 local _pg_node @@ -312,11 +205,6 @@ np_scope_progress() { } # np_scope_output / np_scope_input -# -# Record what the current step (or open sub-step) produced or consumed as an -# inline value carried in the event — the counted evidence a host renders -# next to the narrative ('instances' with healthy/desired, 'traffic' with the -# switched level). np_scope_output() { command -v np_trace_output >/dev/null 2>&1 || return 0 local _ot_node @@ -334,10 +222,6 @@ np_scope_input() { } # np_scope_explain --title T [--what W] [--severity ok|warn|error] ... -# -# The plain-language narrative on the current step (or open sub-step) — what -# the phase is doing, in words, with the severity an operator should read it -# at. Re-emitted per poll on waits so the latest state wins. np_scope_explain() { command -v np_trace_explain >/dev/null 2>&1 || return 0 local _ex_node @@ -347,8 +231,6 @@ np_scope_explain() { } # np_scope_error [] -# -# An observed failure with its structured evidence. np_scope_error() { command -v np_trace_error >/dev/null 2>&1 || return 0 local _sr_node @@ -368,10 +250,6 @@ np_scope_labels() { } # np_scope_k8s_applied -# -# Turn `kubectl apply` output lines (`deployment.apps/name created`) into -# lineage on the current step, for the resource kinds the platform's lineage -# model knows. One chokepoint (apply_templates) covers every workflow. np_scope_k8s_applied() { command -v np_trace_produces >/dev/null 2>&1 || return 0 local _ka_ns="$1" _ka_line _ka_kind _ka_name @@ -395,11 +273,6 @@ np_scope_k8s_applied() { # np_scope_k8s_deleted -# -# The removal counterpart of np_scope_k8s_applied: record what a delete removed -# ("deployment.apps \"name\" deleted" lines) as pointers on the planned step, so a -# finalize's "Remove previous deployment" (or a rollback's "Remove new deployment") -# names what it removed. np_scope_k8s_deleted() { command -v np_trace_output >/dev/null 2>&1 || return 0 local _kd_ns="$1" _kd_node _kd_line _kd_kind _kd_name @@ -427,18 +300,10 @@ np_scope_k8s_deleted() { return 0 } -# A silent failure (no `log error` on the way down) must still be clear: the -# ERR trap remembers the last failing top-level command, and the EXIT trap -# reports it against the step that was current when the shell died. _np_scopes_on_err() { _NP_SCOPES_LAST_ERR="$BASH_COMMAND (exit ${1:-1})" } -# On a fatal exit the RUN's terminal deserves the real reason too: the step -# carries it already; this mirrors it one level up, so list views (which lead -# with the run) name the cause without drilling into steps. The run id is the -# step path minus its last segment — an observed fact on an adopted node, -# like every other enrichment here. _np_scopes_error_on_run() { command -v np_trace_adopt >/dev/null 2>&1 || return 0 [ -n "${NP_TRACE:-}" ] || return 0 @@ -466,16 +331,11 @@ _np_scopes_on_exit() { if [ "$_ex_rc" -ne 0 ]; then _np_scopes_error_on_run "$_ex_reason" || true fi - # A sub-step still open when the shell dies inherits the shell's outcome — - # closed AFTER the error report above so the failure detail lands on it. [ -n "${_NP_SCOPES_SUBSTEP:-}" ] && np_scope_step_end "$_ex_rc" || true np_trace_flush } _NP_SCOPES_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# The SDK ships as the catalog-tracing-sh submodule; a repo copy that lost the -# submodule (archive download, docker build context) may carry the single file -# vendored at the root instead. Neither present -> plain logging, untraced. _NP_SCOPES_SDK="$_NP_SCOPES_ROOT/vendor/catalog-tracing-sh/nptrace.sh" [ -f "$_NP_SCOPES_SDK" ] || _NP_SCOPES_SDK="$_NP_SCOPES_ROOT/nptrace.sh" if [ -z "${NP_TRACE_LOADED:-}" ] \ @@ -484,8 +344,6 @@ if [ -z "${NP_TRACE_LOADED:-}" ] \ && [ -n "${NP_TRACE:-}" ]; then # shellcheck source=/dev/null . "$_NP_SCOPES_SDK" - # --no-trap: the exit flush is ours, so the uncaught-failure report and the - # flush share ONE trap in a defined order. np_trace_init --producer "nullplatform-scopes@1" --api-key "$NP_API_KEY" --no-trap trap '_np_scopes_on_err $?' ERR trap '_np_scopes_on_exit $?' EXIT @@ -493,10 +351,5 @@ elif [ -z "${NP_TRACE_LOADED:-}" ] \ && [ -n "${NP_TRACE:-}" ] \ && [ -n "${NP_API_KEY:-}" ] \ && [ ! -f "$_NP_SCOPES_SDK" ]; then - # The engine is tracing this run and the credentials are present, yet the shell - # SDK is not in this bundle: the scope-side story (sub-steps, live waits, real - # errors, lineage, affordances) will be ABSENT while everything else looks fine. - # A build that loses the submodule (archive download, docker build context) is - # exactly the failure nobody sees — so say it, once, on the workflow's own log. log warn "⚠️ tracing SDK not bundled (vendor/catalog-tracing-sh/nptrace.sh missing) — scope-side tracing disabled for this run" fi