diff --git a/lambda/instance/build_context b/lambda/instance/build_context index e00bafd..58712d7 100755 --- a/lambda/instance/build_context +++ b/lambda/instance/build_context @@ -41,7 +41,7 @@ export SCOPE_ID export SCOPE_NRN export LAMBDA_FUNCTION_NAME -echo "Instance build_context completed" -echo " SCOPE_ID=$SCOPE_ID" -echo " SCOPE_NRN=$SCOPE_NRN" -echo " LAMBDA_FUNCTION_NAME=$LAMBDA_FUNCTION_NAME" +echo "Instance build_context completed" >&2 +echo " SCOPE_ID=$SCOPE_ID" >&2 +echo " SCOPE_NRN=$SCOPE_NRN" >&2 +echo " LAMBDA_FUNCTION_NAME=$LAMBDA_FUNCTION_NAME" >&2 diff --git a/lambda/metric/fetch_metric b/lambda/metric/fetch_metric index e49568e..f4b48d2 100755 --- a/lambda/metric/fetch_metric +++ b/lambda/metric/fetch_metric @@ -1,24 +1,107 @@ #!/bin/bash # Fetch Lambda metrics from CloudWatch -if [ -z "${LAMBDA_FUNCTION_NAME:-}" ]; then - echo "❌ Lambda function name not available" >&2 - echo "💡 Possible causes:" >&2 - echo " - build_context failed to resolve the function name" >&2 - echo " - LAMBDA_FUNCTION_NAME was not exported" >&2 - echo "🔧 How to fix:" >&2 - echo " - Check that build_context ran successfully" >&2 - echo " - Verify the NRN contains LAMBDA_FUNCTION_NAME" >&2 +# Validate required parameters +if [[ -z "$METRIC_NAME" ]]; then + echo '{"metric":"","type":"","period_in_seconds":0,"unit":"","results":[]}' exit 1 fi -# Get metric parameters from context -metric_name="${METRIC_NAME:-Invocations}" -period="${METRIC_PERIOD:-300}" -statistic="${METRIC_STATISTIC:-Sum}" +if [[ -z "$LAMBDA_FUNCTION_NAME" ]]; then + echo '{"metric":"","type":"","period_in_seconds":0,"unit":"","results":[]}' + exit 1 +fi + +# Every metric carries its own aggregation: a duration is averaged, a count is +# summed and a concurrency level is a maximum. It belongs to the metric, not to +# the request, so it is resolved here from the name. +get_metric_config() { + case "$METRIC_NAME" in + "Invocations") + echo "gauge count Sum" + ;; + "Duration") + echo "gauge milliseconds Average" + ;; + "Errors") + echo "gauge count Sum" + ;; + "Throttles") + echo "gauge count Sum" + ;; + "ConcurrentExecutions") + echo "gauge count Maximum" + ;; + "stream.iterator_age") + echo "gauge milliseconds Maximum" + ;; + *) + echo "gauge unknown Sum" + ;; + esac +} + +# The CloudWatch metric behind each name. Only the stream one is renamed; the +# rest map straight through. +get_cloudwatch_metric() { + case "$METRIC_NAME" in + "stream.iterator_age") + echo "IteratorAge" + ;; + *) + echo "$METRIC_NAME" + ;; + esac +} + +query_cloudwatch() { + local metric="$1" + local statistic="$2" + local start_time="$3" + local end_time="$4" + local period="$5" + + aws cloudwatch get-metric-statistics \ + --namespace AWS/Lambda \ + --metric-name "$metric" \ + --dimensions Name=FunctionName,Value="$LAMBDA_FUNCTION_NAME" \ + --start-time "$start_time" \ + --end-time "$end_time" \ + --period "$period" \ + --statistics "$statistic" \ + --output json 2>/dev/null +} + +# A failed or empty query is an empty series, not an error: the graph shows no +# data instead of the panel reporting a provider failure. +transform_response() { + local response="$1" + local statistic="$2" + + if [[ -z "$response" ]]; then + echo "[]" + return + fi + + local datapoints + datapoints=$(echo "$response" | jq '.Datapoints // []' 2>/dev/null) + + if [[ -z "$datapoints" || "$datapoints" == "[]" || "$datapoints" == "null" ]]; then + echo "[]" + return + fi -# Use time range from context or fall back to last hour -if [ -n "${METRIC_START_TIME:-}" ] && [ -n "${METRIC_END_TIME:-}" ]; then + echo "$datapoints" | jq \ + --arg stat "$statistic" \ + --arg function_name "$LAMBDA_FUNCTION_NAME" \ + '[{ + selector: {FunctionName: $function_name}, + data: [.[] | {timestamp: .Timestamp, value: .[$stat]}] | sort_by(.timestamp) + }]' +} + +# Use the requested window, or fall back to the last hour. +if [[ -n "$METRIC_START_TIME" && -n "$METRIC_END_TIME" ]]; then start_time="$METRIC_START_TIME" end_time="$METRIC_END_TIME" else @@ -28,48 +111,17 @@ else unset _now fi -# Fetch metric -result=$(aws cloudwatch get-metric-statistics \ - --namespace AWS/Lambda \ - --metric-name "$metric_name" \ - --dimensions Name=FunctionName,Value="$LAMBDA_FUNCTION_NAME" \ - --start-time "$start_time" \ - --end-time "$end_time" \ - --period "$period" \ - --statistics "$statistic" \ - --query 'Datapoints[*]' \ - --output json 2>&1) - -if [ $? -ne 0 ]; then - echo "❌ Failed to fetch metric '$metric_name' from CloudWatch" >&2 - echo "💡 Possible causes:" >&2 - echo " - Invalid metric name '$metric_name'" >&2 - echo " - AWS credentials lack cloudwatch:GetMetricStatistics permission" >&2 - echo " - Function '$LAMBDA_FUNCTION_NAME' has no data for this metric" >&2 - echo "🔧 How to fix:" >&2 - echo " - Verify the metric name is valid (use list_metrics to see options)" >&2 - echo " - Check the agent IAM role permissions" >&2 - exit 1 -fi +step=${METRIC_PERIOD:-300} -unit=$(echo "$result" | jq -r 'first | .Unit // ""') -sorted_data=$(echo "$result" | jq --arg stat "$statistic" \ - '[.[] | {timestamp: .Timestamp, value: .[$stat]}] | sort_by(.timestamp)') - -jq -n \ - --arg metric "$metric_name" \ - --arg type "$statistic" \ - --argjson period "$period" \ - --arg unit "$unit" \ - --arg function_name "$LAMBDA_FUNCTION_NAME" \ - --argjson data "$sorted_data" \ - '{ - metric: $metric, - type: $type, - period_in_seconds: $period, - unit: $unit, - results: [{ - selector: {FunctionName: $function_name}, - data: $data - }] - }' +config=$(get_metric_config) +metric_type=$(echo "$config" | cut -d' ' -f1) +unit=$(echo "$config" | cut -d' ' -f2) +statistic=$(echo "$config" | cut -d' ' -f3) + +cloudwatch_metric=$(get_cloudwatch_metric) + +response=$(query_cloudwatch "$cloudwatch_metric" "$statistic" "$start_time" "$end_time" "$step") + +transformed_results=$(transform_response "$response" "$statistic") + +echo "{\"metric\":\"$METRIC_NAME\",\"type\":\"$metric_type\",\"period_in_seconds\":$step,\"unit\":\"$unit\",\"results\":$transformed_results}" diff --git a/lambda/metric/list_metrics b/lambda/metric/list_metrics index e5373ca..8b4c389 100755 --- a/lambda/metric/list_metrics +++ b/lambda/metric/list_metrics @@ -36,6 +36,13 @@ echo '{ "unit": "count", "available_filters": ["scope_id"], "available_group_by": [] + }, + { + "name": "stream.iterator_age", + "title": "Stream iterator age", + "unit": "ms", + "available_filters": ["scope_id"], + "available_group_by": [] } ] }' diff --git a/lambda/utils/assume_role b/lambda/utils/assume_role index 0c851de..7870fc2 100755 --- a/lambda/utils/assume_role +++ b/lambda/utils/assume_role @@ -8,11 +8,16 @@ # Expects: ASSUME_ROLE_ARN (exported by fetch_scope_configuration or values.yaml) # SCOPE_ID (optional, used for the session name) +# Progress goes to stderr, including when the pretty logger is available: the +# telemetry workflows (metric, log, instance) hand their stdout to the platform as +# the response body, so anything printed here lands in front of the JSON and the +# parse fails. utils/log keeps writing to stdout for the deployment workflows, +# where that output is the progress the user sees. _ar_log() { if declare -f log > /dev/null 2>&1; then - log "$1" "$2" + log "$1" "$2" >&2 else - echo "$2" + echo "$2" >&2 fi }