From bdfb489c493f9a911d9209bbc274ea5c16579ebd Mon Sep 17 00:00:00 2001
From: Agustin Celentano <12614595+agustincelentano@users.noreply.github.com>
Date: Mon, 3 Aug 2026 09:25:35 -0300
Subject: [PATCH 1/2] feat(lambda): resolve each metric's statistic and expose
 the stream iterator age
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

fetch_metric read METRIC_STATISTIC, and nothing ever set it: build_context does
not export it and the workflow does not declare it, so every metric fell back to
Sum. Three of them are counts and were fine; the other two reported numbers that
do not mean what their name says. Measured on a live function: Duration read
1076 ms where the latency was 5.25 ms, and ConcurrentExecutions read 205 where
the peak was 1.

The statistic now comes from the metric name, the way the containers scope
resolves type and unit in scopes/k8s/metric/metric — it belongs to the metric,
not to the request.

A failed or empty query now returns the response with an empty series instead of
exiting non-zero. Reaching the platform as an error turned a graph with no data
into "Error fetching data from external provider".

Adds stream.iterator_age, which is the one metric that says whether the consumer
of a DynamoDB stream is falling behind. The others describe the function's
health, and it can be healthy while the stream piles up behind it.
---
 lambda/metric/fetch_metric | 168 ++++++++++++++++++++++++-------------
 lambda/metric/list_metrics |   7 ++
 2 files changed, 117 insertions(+), 58 deletions(-)

diff --git a/lambda/metric/fetch_metric b/lambda/metric/fetch_metric
index e49568e..f4b48d2 100755
--- a/lambda/metric/fetch_metric
+++ b/lambda/metric/fetch_metric
@@ -1,24 +1,107 @@
 #!/bin/bash
 # Fetch Lambda metrics from CloudWatch
 
-if [ -z "${LAMBDA_FUNCTION_NAME:-}" ]; then
-   echo "❌ Lambda function name not available" >&2
-   echo "💡 Possible causes:" >&2
-   echo "   - build_context failed to resolve the function name" >&2
-   echo "   - LAMBDA_FUNCTION_NAME was not exported" >&2
-   echo "🔧 How to fix:" >&2
-   echo "   - Check that build_context ran successfully" >&2
-   echo "   - Verify the NRN contains LAMBDA_FUNCTION_NAME" >&2
+# Validate required parameters
+if [[ -z "$METRIC_NAME" ]]; then
+   echo '{"metric":"","type":"","period_in_seconds":0,"unit":"","results":[]}'
    exit 1
 fi
 
-# Get metric parameters from context
-metric_name="${METRIC_NAME:-Invocations}"
-period="${METRIC_PERIOD:-300}"
-statistic="${METRIC_STATISTIC:-Sum}"
+if [[ -z "$LAMBDA_FUNCTION_NAME" ]]; then
+   echo '{"metric":"","type":"","period_in_seconds":0,"unit":"","results":[]}'
+   exit 1
+fi
+
+# Every metric carries its own aggregation: a duration is averaged, a count is
+# summed and a concurrency level is a maximum. It belongs to the metric, not to
+# the request, so it is resolved here from the name.
+get_metric_config() {
+   case "$METRIC_NAME" in
+      "Invocations")
+         echo "gauge count Sum"
+         ;;
+      "Duration")
+         echo "gauge milliseconds Average"
+         ;;
+      "Errors")
+         echo "gauge count Sum"
+         ;;
+      "Throttles")
+         echo "gauge count Sum"
+         ;;
+      "ConcurrentExecutions")
+         echo "gauge count Maximum"
+         ;;
+      "stream.iterator_age")
+         echo "gauge milliseconds Maximum"
+         ;;
+      *)
+         echo "gauge unknown Sum"
+         ;;
+   esac
+}
+
+# The CloudWatch metric behind each name. Only the stream one is renamed; the
+# rest map straight through.
+get_cloudwatch_metric() {
+   case "$METRIC_NAME" in
+      "stream.iterator_age")
+         echo "IteratorAge"
+         ;;
+      *)
+         echo "$METRIC_NAME"
+         ;;
+   esac
+}
+
+query_cloudwatch() {
+   local metric="$1"
+   local statistic="$2"
+   local start_time="$3"
+   local end_time="$4"
+   local period="$5"
+
+   aws cloudwatch get-metric-statistics \
+      --namespace AWS/Lambda \
+      --metric-name "$metric" \
+      --dimensions Name=FunctionName,Value="$LAMBDA_FUNCTION_NAME" \
+      --start-time "$start_time" \
+      --end-time "$end_time" \
+      --period "$period" \
+      --statistics "$statistic" \
+      --output json 2>/dev/null
+}
+
+# A failed or empty query is an empty series, not an error: the graph shows no
+# data instead of the panel reporting a provider failure.
+transform_response() {
+   local response="$1"
+   local statistic="$2"
+
+   if [[ -z "$response" ]]; then
+      echo "[]"
+      return
+   fi
+
+   local datapoints
+   datapoints=$(echo "$response" | jq '.Datapoints // []' 2>/dev/null)
+
+   if [[ -z "$datapoints" || "$datapoints" == "[]" || "$datapoints" == "null" ]]; then
+      echo "[]"
+      return
+   fi
 
-# Use time range from context or fall back to last hour
-if [ -n "${METRIC_START_TIME:-}" ] && [ -n "${METRIC_END_TIME:-}" ]; then
+   echo "$datapoints" | jq \
+      --arg stat "$statistic" \
+      --arg function_name "$LAMBDA_FUNCTION_NAME" \
+      '[{
+         selector: {FunctionName: $function_name},
+         data: [.[] | {timestamp: .Timestamp, value: .[$stat]}] | sort_by(.timestamp)
+      }]'
+}
+
+# Use the requested window, or fall back to the last hour.
+if [[ -n "$METRIC_START_TIME" && -n "$METRIC_END_TIME" ]]; then
    start_time="$METRIC_START_TIME"
    end_time="$METRIC_END_TIME"
 else
@@ -28,48 +111,17 @@ else
    unset _now
 fi
 
-# Fetch metric
-result=$(aws cloudwatch get-metric-statistics \
-   --namespace AWS/Lambda \
-   --metric-name "$metric_name" \
-   --dimensions Name=FunctionName,Value="$LAMBDA_FUNCTION_NAME" \
-   --start-time "$start_time" \
-   --end-time "$end_time" \
-   --period "$period" \
-   --statistics "$statistic" \
-   --query 'Datapoints[*]' \
-   --output json 2>&1)
-
-if [ $? -ne 0 ]; then
-   echo "❌ Failed to fetch metric '$metric_name' from CloudWatch" >&2
-   echo "💡 Possible causes:" >&2
-   echo "   - Invalid metric name '$metric_name'" >&2
-   echo "   - AWS credentials lack cloudwatch:GetMetricStatistics permission" >&2
-   echo "   - Function '$LAMBDA_FUNCTION_NAME' has no data for this metric" >&2
-   echo "🔧 How to fix:" >&2
-   echo "   - Verify the metric name is valid (use list_metrics to see options)" >&2
-   echo "   - Check the agent IAM role permissions" >&2
-   exit 1
-fi
+step=${METRIC_PERIOD:-300}
 
-unit=$(echo "$result" | jq -r 'first | .Unit // ""')
-sorted_data=$(echo "$result" | jq --arg stat "$statistic" \
-   '[.[] | {timestamp: .Timestamp, value: .[$stat]}] | sort_by(.timestamp)')
-
-jq -n \
-   --arg metric "$metric_name" \
-   --arg type "$statistic" \
-   --argjson period "$period" \
-   --arg unit "$unit" \
-   --arg function_name "$LAMBDA_FUNCTION_NAME" \
-   --argjson data "$sorted_data" \
-   '{
-      metric: $metric,
-      type: $type,
-      period_in_seconds: $period,
-      unit: $unit,
-      results: [{
-         selector: {FunctionName: $function_name},
-         data: $data
-      }]
-   }'
+config=$(get_metric_config)
+metric_type=$(echo "$config" | cut -d' ' -f1)
+unit=$(echo "$config" | cut -d' ' -f2)
+statistic=$(echo "$config" | cut -d' ' -f3)
+
+cloudwatch_metric=$(get_cloudwatch_metric)
+
+response=$(query_cloudwatch "$cloudwatch_metric" "$statistic" "$start_time" "$end_time" "$step")
+
+transformed_results=$(transform_response "$response" "$statistic")
+
+echo "{\"metric\":\"$METRIC_NAME\",\"type\":\"$metric_type\",\"period_in_seconds\":$step,\"unit\":\"$unit\",\"results\":$transformed_results}"
diff --git a/lambda/metric/list_metrics b/lambda/metric/list_metrics
index e5373ca..8b4c389 100755
--- a/lambda/metric/list_metrics
+++ b/lambda/metric/list_metrics
@@ -36,6 +36,13 @@ echo '{
       "unit": "count",
       "available_filters": ["scope_id"],
       "available_group_by": []
+    },
+    {
+      "name": "stream.iterator_age",
+      "title": "Stream iterator age",
+      "unit": "ms",
+      "available_filters": ["scope_id"],
+      "available_group_by": []
     }
   ]
 }'

From 6fbbe611be7cf173ca381f5c914e494179e1971f Mon Sep 17 00:00:00 2001
From: Agustin Celentano <12614595+agustincelentano@users.noreply.github.com>
Date: Mon, 3 Aug 2026 10:24:22 -0300
Subject: [PATCH 2/2] fix(lambda): keep progress messages out of the telemetry
 response body
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The telemetry workflows hand their stdout to the platform as the response body,
and assume_role wrote its progress there. The catalog arrived behind two lines of
text:

       Assuming role: arn:aws:iam::...:role/..._lambda_role
       Role assumed successfully
    { "results": [ ... ] }

The platform could not parse that, so GET /metric answered 400 "Error fetching
data from external provider", the UI was left without a catalog and fell back to
the container metrics — CPU, memory, throughput — none of which a Lambda scope
can answer. Every one of those returned 400 too.

Only _ar_log changes, including the branch that delegates to the pretty logger.
utils/log keeps writing to stdout because the deployment workflows show that
output as progress, and silencing it there would leave deploys mute.

instance/build_context printed four diagnostic lines to stdout and broke
instance:data the same way.
---
 lambda/instance/build_context | 8 ++++----
 lambda/utils/assume_role      | 9 +++++++--
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/lambda/instance/build_context b/lambda/instance/build_context
index e00bafd..58712d7 100755
--- a/lambda/instance/build_context
+++ b/lambda/instance/build_context
@@ -41,7 +41,7 @@ export SCOPE_ID
 export SCOPE_NRN
 export LAMBDA_FUNCTION_NAME
 
-echo "Instance build_context completed"
-echo "   SCOPE_ID=$SCOPE_ID"
-echo "   SCOPE_NRN=$SCOPE_NRN"
-echo "   LAMBDA_FUNCTION_NAME=$LAMBDA_FUNCTION_NAME"
+echo "Instance build_context completed" >&2
+echo "   SCOPE_ID=$SCOPE_ID" >&2
+echo "   SCOPE_NRN=$SCOPE_NRN" >&2
+echo "   LAMBDA_FUNCTION_NAME=$LAMBDA_FUNCTION_NAME" >&2
diff --git a/lambda/utils/assume_role b/lambda/utils/assume_role
index 0c851de..7870fc2 100755
--- a/lambda/utils/assume_role
+++ b/lambda/utils/assume_role
@@ -8,11 +8,16 @@
 # Expects:  ASSUME_ROLE_ARN (exported by fetch_scope_configuration or values.yaml)
 #           SCOPE_ID (optional, used for the session name)
 
+# Progress goes to stderr, including when the pretty logger is available: the
+# telemetry workflows (metric, log, instance) hand their stdout to the platform as
+# the response body, so anything printed here lands in front of the JSON and the
+# parse fails. utils/log keeps writing to stdout for the deployment workflows,
+# where that output is the progress the user sees.
 _ar_log() {
   if declare -f log > /dev/null 2>&1; then
-    log "$1" "$2"
+    log "$1" "$2" >&2
   else
-    echo "$2"
+    echo "$2" >&2
   fi
 }