From 512b1799c616bd6a0f11063bb8a6881c308330eb Mon Sep 17 00:00:00 2001 From: Vipul Gargav Date: Tue, 25 Aug 2026 11:35:04 -0400 Subject: [PATCH] feat: add agentcore-observability-setup skill Add a skill that validates and bootstraps Amazon Bedrock AgentCore observability using a verify-where-reachable / prescribe-everywhere-else model. Covers Runtime agents, Memory and Gateway resources, built-in tools, and non-runtime hosts (Lambda, ECS, EKS, on-prem, multi-cloud) with three read-only IAM tiers and graceful degradation. Register the skill in llms.txt and add a gated read-only inline policy (EnableAgentCoreObservabilitySetup) to the CloudFormation skill policies for the Tier 2/3 permissions not covered by AIDevOpsAgentAccessPolicy. --- .../devops-agent-skill-policies.yaml | 47 +++++ llms.txt | 1 + .../.skilleval.yaml | 3 + .../CHANGELOG.md | 12 ++ .../agentcore-observability-setup/README.md | 114 ++++++++++++ skills/agentcore-observability-setup/SKILL.md | 168 ++++++++++++++++++ .../evals/eval_queries.json | 8 + .../evals/evals.json | 73 ++++++++ .../evals/files/agent-context.json | 20 +++ .../references/checks-catalog.md | 84 +++++++++ .../references/iam-tiers.md | 113 ++++++++++++ .../references/remediation-memory-gateway.md | 89 ++++++++++ .../references/remediation-non-runtime.md | 99 +++++++++++ .../references/remediation-runtime.md | 99 +++++++++++ 14 files changed, 930 insertions(+) create mode 100644 skills/agentcore-observability-setup/.skilleval.yaml create mode 100644 skills/agentcore-observability-setup/CHANGELOG.md create mode 100644 skills/agentcore-observability-setup/README.md create mode 100644 skills/agentcore-observability-setup/SKILL.md create mode 100644 skills/agentcore-observability-setup/evals/eval_queries.json create mode 100644 skills/agentcore-observability-setup/evals/evals.json create mode 100644 skills/agentcore-observability-setup/evals/files/agent-context.json create mode 100644 skills/agentcore-observability-setup/references/checks-catalog.md create mode 100644 skills/agentcore-observability-setup/references/iam-tiers.md create mode 100644 skills/agentcore-observability-setup/references/remediation-memory-gateway.md create mode 100644 skills/agentcore-observability-setup/references/remediation-non-runtime.md create mode 100644 skills/agentcore-observability-setup/references/remediation-runtime.md diff --git a/cloudformation/devops-agent-skill-policies.yaml b/cloudformation/devops-agent-skill-policies.yaml index f4bd005..c38e79c 100644 --- a/cloudformation/devops-agent-skill-policies.yaml +++ b/cloudformation/devops-agent-skill-policies.yaml @@ -27,6 +27,7 @@ Metadata: - EnableMskOperations - EnableServiceQuotaCheck - EnableDmsOperationReview + - EnableAgentCoreObservabilitySetup - Label: default: Optional Resource Scoping Parameters: @@ -111,6 +112,12 @@ Parameters: AllowedValues: ['true', 'false'] Default: 'true' + EnableAgentCoreObservabilitySetup: + Type: String + Description: AgentCore Observability Setup skill (adds read-only bedrock-agentcore, X-Ray, log-delivery, and Lambda/ECS/EKS host-config permissions). + AllowedValues: ['true', 'false'] + Default: 'true' + Conditions: CreateNewRole: !Equals [!Ref ExistingRoleName, ''] SkillAwsHealthEvents: !Equals [!Ref EnableAwsHealthEvents, 'true'] @@ -120,6 +127,7 @@ Conditions: SkillMskOperations: !Equals [!Ref EnableMskOperations, 'true'] SkillServiceQuotaCheck: !Equals [!Ref EnableServiceQuotaCheck, 'true'] SkillDmsOperationReview: !Equals [!Ref EnableDmsOperationReview, 'true'] + SkillAgentCoreObservabilitySetup: !Equals [!Ref EnableAgentCoreObservabilitySetup, 'true'] HasRegionRestriction: !Not [!Equals [!Join ['', !Ref AllowedRegions], '']] Resources: @@ -302,6 +310,44 @@ Resources: - dms:TestConnection Resource: '*' + # agentcore-observability-setup: adds read-only AgentCore control-plane, X-Ray Transaction Search, + # log-delivery/resource-policy describes, and Lambda/ECS/EKS host-config reads (Tier 2 + Tier 3). + # Tier 1 (CloudWatch Logs/Metrics telemetry-arrival reads) is covered by AIDevOpsAgentAccessPolicy. + PolicyAgentCoreObservabilitySetup: + Type: AWS::IAM::Policy + Condition: SkillAgentCoreObservabilitySetup + Properties: + PolicyName: DevOpsAgentSkill-AgentCoreObservabilitySetup + Roles: + - !If [CreateNewRole, !Ref DevOpsAgentRole, !Ref ExistingRoleName] + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: AgentCoreRuntimeConfigRead + Effect: Allow + Action: + - bedrock-agentcore:GetAgentRuntime + - bedrock-agentcore:ListAgentRuntimes + Resource: '*' + - Sid: XRayTransactionSearchAndDeliveries + Effect: Allow + Action: + - xray:GetTraceSegmentDestination + - logs:DescribeDeliveries + - logs:DescribeDeliverySources + - logs:DescribeDeliveryDestinations + - logs:DescribeResourcePolicies + Resource: '*' + - Sid: NonRuntimeHostConfigRead + Effect: Allow + Action: + - lambda:GetFunctionConfiguration + - ecs:DescribeTaskDefinition + - ecs:DescribeServices + - ecs:ListTasks + - eks:DescribeCluster + Resource: '*' + # Optional: restrict agent to specific regions PolicyRegionalRestriction: Type: AWS::IAM::Policy @@ -349,6 +395,7 @@ Outputs: - msk-operations: ${EnableMskOperations} (kafka:GetBootstrapBrokers) - service-quota-check: ${EnableServiceQuotaCheck} (servicequotas:*, cloudwatch:GetMetricData/GetMetricStatistics) - database-migration-service-expertise: ${EnableDmsOperationReview} (dms:TestConnection) + - agentcore-observability-setup: ${EnableAgentCoreObservabilitySetup} (bedrock-agentcore:Get/ListAgentRuntime, xray:GetTraceSegmentDestination, logs:DescribeDeliveries/DeliverySources/DeliveryDestinations/ResourcePolicies, lambda:GetFunctionConfiguration, ecs:DescribeTaskDefinition/DescribeServices/ListTasks, eks:DescribeCluster) Skills covered by AIDevOpsAgentAccessPolicy (no extra policy needed): - eks-operation-review, enrich-with-aws-security-agent, crm-production-investigation-guidelines No IAM required: diff --git a/llms.txt b/llms.txt index 6031d00..ecd6e09 100644 --- a/llms.txt +++ b/llms.txt @@ -27,6 +27,7 @@ Skills can be used with these AWS DevOps Agent types: - [S3 Resiliency Review Skill](skills/storage-s3-resiliency-expertise/SKILL.md): Reviews one or many S3 buckets across nine resiliency, security, and data-protection dimensions using read-only control-plane calls, producing a rated report with prioritized findings and remediation guidance - [VPC DNS Investigation Skill](skills/aws-vpc-dns-investigation/SKILL.md): Diagnoses VPC DNS resolution failures and validates DNS control-plane changes before they are applied, driving the aws-vpc-dns-diagnostics MCP server to observe live resolution from inside the affected subnet and to simulate a proposed change - [Bedrock Adoption Readiness Skill](skills/bedrock-adoption-readiness/SKILL.md): Assesses an AWS account's readiness to run Amazon Bedrock at production scale across IAM governance, data retention (ZDR), quota and capacity headroom, and operational observability, covering both the standard Bedrock and bedrock-mantle (OpenAI-compatible) surfaces with multi-region discovery +- [AgentCore Observability Setup Skill](skills/agentcore-observability-setup/SKILL.md): Validates and bootstraps Amazon Bedrock AgentCore observability across runtime agents, Memory and Gateway resources, built-in tools, and agents hosted outside the runtime, verifying telemetry wiring via read-only CloudWatch, X-Ray, and AgentCore APIs and prescribing exact remediation for gaps it cannot directly read ## Key Concepts diff --git a/skills/agentcore-observability-setup/.skilleval.yaml b/skills/agentcore-observability-setup/.skilleval.yaml new file mode 100644 index 0000000..686a9c7 --- /dev/null +++ b/skills/agentcore-observability-setup/.skilleval.yaml @@ -0,0 +1,3 @@ +audit: + ignore: + - STR-016 # README alongside SKILL.md is intentional diff --git a/skills/agentcore-observability-setup/CHANGELOG.md b/skills/agentcore-observability-setup/CHANGELOG.md new file mode 100644 index 0000000..219875d --- /dev/null +++ b/skills/agentcore-observability-setup/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +## 1.0.0 + +- Initial version. +- Validates and bootstraps Amazon Bedrock AgentCore observability using a verify-where-reachable / + prescribe-everywhere-else model. +- Covers Runtime agents, Memory and Gateway resources, built-in tools, and non-runtime hosts + (Lambda, ECS, EKS, on-prem, multi-cloud). +- Three read-only IAM tiers with graceful degradation and scoped inline policy JSON. +- Check catalog and per-host remediation references (Transaction Search, ADOT SDK, unified span + destination, X-Ray log-group resource policy, Memory/Gateway log & trace delivery). diff --git a/skills/agentcore-observability-setup/README.md b/skills/agentcore-observability-setup/README.md new file mode 100644 index 0000000..00db535 --- /dev/null +++ b/skills/agentcore-observability-setup/README.md @@ -0,0 +1,114 @@ +# AgentCore Observability Setup & Validation — AWS DevOps Agent Skill + +A readiness-checklist skill for [AWS DevOps Agent](https://docs.aws.amazon.com/devopsagent/latest/userguide/about-aws-devops-agent.html) +that validates and bootstraps observability for Amazon Bedrock AgentCore workloads, aligned with the +[AgentCore observability documentation](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-configure.html). + +> ⚠️ This skill is sample code, not intended for production use without additional review and +> testing. Users should validate in a non-production environment first. + +## What It Does + +Given an AgentCore deployment, the skill instructs the agent to report **what is configured vs. what +should be** for observability, and to generate the exact remediation for each gap. It operates on a +single principle: **verify where reachable; prescribe everywhere else.** + +- **VERIFY** — confirm via read-only APIs that log groups exist and receive data, spans flow, + session metrics emit, CloudWatch Transaction Search is enabled, runtime tracing is on, + Memory/Gateway log & trace deliveries exist, and the X-Ray log-group resource policy is present. +- **PRESCRIBE** — for code-level OTEL instrumentation (which cannot be read directly), validate the + *effect* (are spans arriving?) and emit the exact steps. The skill does **not** claim to read + source code. + +Covers AgentCore **Runtime** agents (primary), **Memory** and **Gateway** resources, **built-in +tools**, and agents hosted **outside the runtime** (Lambda, ECS, EKS, on-prem, multi-cloud). + +It is **read-only**: it validates and generates config/commands. The customer decides whether to +apply them. It never mutates IAM, resources, or configuration. + +Output: a shareable report artifact `agentcore-observability-review--.md`. + +## Agent Types + +- **On-demand (Chat)** — "is AgentCore observability configured correctly?", "why can't I see traces + for my agent?", "set up OTEL for my agent". +- **Evaluation** — proactive observability-readiness recommendations. + +Select **Generic** to make it available to all agent types. + +## Prerequisites + +### 1. An AWS DevOps Agent Space with the target AWS account configured as a cloud source. + +### 2. IAM — three read-only tiers (customer-applied) + +The skill degrades gracefully by tier and reports the exact policy to attach to unlock more checks. +Tier 1 is generally covered by the standard DevOps Agent managed access policy +([AWS-managed policies for DevOps Agent](https://docs.aws.amazon.com/devopsagent/latest/userguide/security-iam-awsmanpol.html)). +Attach the Tier 2/3 permissions as a **scoped inline policy on the DevOps Agent role** in your +account. The shared managed policy is cross-tenant and is not modified per-skill. + +| Tier | Unlocks | Key permissions | +|---|---|---| +| **1 — Standard** | telemetry-arrival verification (any host) | `logs:DescribeLogGroups`, `logs:FilterLogEvents`, `logs:GetLogEvents`, `logs:StartQuery`, `cloudwatch:GetMetricData`, `cloudwatch:ListMetrics`, `cloudwatch:DescribeAlarms` | +| **2 — Runtime config** | Transaction Search state, runtime tracing/env, Memory/Gateway delivery, X-Ray resource policy | `bedrock-agentcore:GetAgentRuntime`, `xray:GetTraceSegmentDestination`, `logs:DescribeDeliveries`, `logs:DescribeResourcePolicies` | +| **3 — Non-runtime host** | Lambda / ECS / EKS host config | `lambda:GetFunctionConfiguration`, `ecs:DescribeTaskDefinition`, `eks:DescribeCluster` | + +Full policy JSON: [`references/iam-tiers.md`](references/iam-tiers.md). All permissions are read-only. + +### 3. Capability providers connected to the Agent Space + +CloudWatch Logs + Metrics (Tier 1, required baseline); AWS X-Ray and AgentCore control plane +(Tier 2); Lambda / ECS / EKS (Tier 3). Source-repository file read is **not** required — code-level +checks are prescriptive. + +## Usage + +In the DevOps Agent Chat, describe the problem in natural language (don't name the skill): + +- *"Validate that AgentCore observability is correctly configured for this account."* +- *"Why can't I see any traces for my Bedrock agent?"* +- *"My agent runtime has no spans in CloudWatch — what's wrong?"* +- *"Set up observability for my agent running on Lambda."* +- *"Run an AgentCore observability readiness review."* + +The agent will detect the permission tier, determine the host and surfaces (prompting where it can't +auto-detect), run the checks the tier allows, and produce the report artifact plus any policy JSON +needed to unlock more checks. + +## Skill Contents + +``` +agentcore-observability-setup/ +├── SKILL.md # main instructions (frontmatter + decision tree) +├── README.md # this file +├── CHANGELOG.md # version history +├── references/ +│ ├── checks-catalog.md # full per-check catalog (API, logic, severity) +│ ├── iam-tiers.md # three read-only tiers + scoped inline policy JSON +│ ├── remediation-runtime.md # Transaction Search, ADOT, tracing, X-Ray policy +│ ├── remediation-memory-gateway.md # log delivery + tracing (console + SDK) +│ └── remediation-non-runtime.md # Lambda / ECS / EKS / on-prem OTEL env +└── evals/ # evaluation data (not included in upload zip) + ├── evals.json + └── eval_queries.json +``` + +## Severity Definitions + +| Severity | Definition | +|---|---| +| CRITICAL | Observability is broken — telemetry cannot flow (e.g. Transaction Search disabled). | +| HIGH | Significant gap — a required piece is missing (log group, delivery, resource policy, host env). | +| MEDIUM | Notable gap — metrics/traces partially configured or best-practice not met. | +| LOW | Minor hardening or optimization. | +| INFO | Observation, no action required. | + +## Limitations + +- **Code-level instrumentation** (ADOT distro, `opentelemetry-instrument`, framework tracing) is + prescriptive — validated by effect (spans arriving), not by reading source. +- **EKS pod-level env** is governed by Kubernetes RBAC and is not verifiable via IAM; the skill + verifies telemetry arrival and prescribes the pod/ConfigMap configuration. +- **On-prem / multi-cloud** agents are outside the DevOps Agent's reach — prescriptive only. +- **Read-only** — the skill validates and generates config/commands; the customer applies changes. diff --git a/skills/agentcore-observability-setup/SKILL.md b/skills/agentcore-observability-setup/SKILL.md new file mode 100644 index 0000000..a34128d --- /dev/null +++ b/skills/agentcore-observability-setup/SKILL.md @@ -0,0 +1,168 @@ +--- +name: agentcore-observability-setup +description: Validates and bootstraps Amazon Bedrock AgentCore observability so customers can trace + agent reasoning, detect silent failures, and measure performance before an outage. Use this skill + when a user asks to check, validate, audit, verify, fix, or set up AgentCore observability, + tracing, metrics, or logging - for example "is AgentCore observability configured correctly", + "why can't I see traces for my agent", "my Bedrock agent has no spans in CloudWatch", "validate + agent monitoring", "AgentCore observability readiness", or "set up OTEL for my agent". Covers + AgentCore Runtime agents, Memory and Gateway resources, built-in tools, and agents hosted outside + the runtime (Lambda, ECS, EKS, on-prem, multi-cloud). Verifies what is reachable via read-only + CloudWatch, X-Ray, and AgentCore control-plane APIs, and prescribes exact remediation for gaps it + cannot directly read such as code-level OTEL instrumentation. +metadata: + author: vggargav + version: "1.0.0" + aws-devops-agent-skills.agent-types: "Chat tasks, Evaluation" + aws-devops-agent-skills.aws-services: "Amazon Bedrock AgentCore, Amazon CloudWatch, AWS X-Ray" + aws-devops-agent-skills.technical-domains: "AI/ML, Observability" +--- + +# AgentCore Observability Setup & Validation + +Validate and bootstrap observability for Amazon Bedrock AgentCore workloads, aligned with the +[AgentCore observability documentation](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-configure.html). +Produce a readiness report that states, per check, what is **configured vs. what should be**, and +generate the exact remediation for every gap. + +## Operating Principle + +**Verify where reachable; prescribe everywhere else.** Every check is exactly one of: + +- **VERIFY** — confirmable through a read-only API (log group exists, data arriving, spans flowing, + Transaction Search enabled, tracing toggle, delivery configured). +- **PRESCRIBE** — cannot be read directly (code-level OTEL instrumentation in the agent's source). + Validate the *effect* (are spans arriving?) and emit the exact steps. Do **not** claim to read + source code. + +This skill is **read-only**. It validates and generates configuration/commands. The customer decides +whether to apply them. Never mutate IAM, resources, or configuration. + +## When to Use + +Activate when the user asks to: +- Check / validate / audit whether AgentCore observability is configured correctly +- Diagnose missing traces, spans, metrics, or logs for a Bedrock agent +- Set up or bootstrap observability (tracing, metrics, logging) for an agent +- Run an AgentCore observability readiness review +- Understand why agent telemetry isn't appearing in CloudWatch GenAI Observability + +## Step 1: Detect Permission Tier + +This skill degrades gracefully based on which read-only permissions the DevOps Agent role holds. +Detect the tier by attempting calls and noting failures; run the checks the tier allows and report +what to add to unlock the rest. **Never** attempt to modify IAM. + +| Tier | Permissions | Unlocks | +|---|---|---| +| **1 — Standard** (default DA IAM) | `logs:DescribeLogGroups`, `logs:FilterLogEvents`, `logs:GetLogEvents`, `logs:StartQuery`, `logs:GetQueryResults`, `cloudwatch:GetMetricData`, `cloudwatch:ListMetrics`, `cloudwatch:DescribeAlarms` | Telemetry-arrival verification for any host | +| **2 — Runtime config** | `bedrock-agentcore:GetAgentRuntime`, `bedrock-agentcore-control:List*`, `xray:GetTraceSegmentDestination`, `logs:DescribeDeliveries`, `logs:DescribeDeliverySources`, `logs:DescribeDeliveryDestinations`, `logs:DescribeResourcePolicies` | Runtime tracing/env, Transaction Search state, Memory/Gateway delivery, X-Ray resource policy | +| **3 — Non-runtime host** | `lambda:GetFunctionConfiguration`; `ecs:DescribeTaskDefinition`, `ecs:DescribeServices`, `ecs:ListTasks`; `eks:DescribeCluster` | Host-side config verification for Lambda / ECS / EKS | + +If Tier 2/3 permissions are absent, tell the user the check is **prescriptive-only** here and give the +exact scoped inline policy to attach to the DevOps Agent role (see `references/iam-tiers.md`) so a +re-run can verify it. The shared managed policy is cross-tenant and must not be modified per-skill. + +## Step 2: Determine Scope (Host & Surface) + +**Host detection:** +- **Runtime** — auto-detect with `bedrock-agentcore:GetAgentRuntime` / `bedrock-agentcore-control:ListAgentRuntimes` (Tier 2). If absent, ask the user. +- **Non-runtime** — the agent cannot reliably auto-detect the host. Ask: *"Is the agent on Lambda, ECS, EKS, or on-prem/another cloud?"* + +**Surfaces to assess** (ask which apply, or discover via Tier 2 `List*`): +Runtime agents · Memory resources · Gateway resources · Built-in tools (code interpreter, browser) · non-runtime host. + +**Outcome matrix:** + +| Host / Surface | Expected outcome | +|---|---| +| Runtime agent | Verify (Tier 1+2) + Prescribe | +| Memory resource | Verify delivery + tracing (Tier 2) + Prescribe | +| Gateway resource | Verify delivery + tracing (Tier 2) + Prescribe | +| Built-in tools | Prescribe; verify telemetry-arrival where present | +| Lambda | Verify host config (Tier 3) + Prescribe | +| ECS | Verify host config (Tier 3) + Prescribe | +| EKS | Verify telemetry arrival (Tier 1) + Prescribe config (pod env is K8s-RBAC-gated, not IAM-verifiable) | +| On-prem / multi-cloud | Prescribe only | + +## Step 3: Run Checks + +Assign each finding a severity (CRITICAL / HIGH / MEDIUM / LOW / INFO) and a type (VERIFY / PRESCRIBE). +The complete check catalog with APIs, log-group patterns, and pass/fail logic is in +`references/checks-catalog.md`. Summary below. + +### 3.1 Account prerequisite — CloudWatch Transaction Search (all hosts) +- **VERIFY (Tier 2):** `xray:GetTraceSegmentDestination` → destination must be `CloudWatchLogs` and status `ACTIVE`. If `XRay`/inactive, Transaction Search is **not** enabled → **CRITICAL** (spans will not be delivered to CloudWatch Logs; this is the single most common silent root cause). +- **VERIFY (Tier 1 fallback):** if the destination API is unavailable, infer from whether span log streams are receiving data (below). Report as inferred, not confirmed. +- **PRESCRIBE:** enable Transaction Search and ingest spans as structured logs (see `references/remediation-runtime.md`). + +### 3.2 Runtime agent path (start here — primary surface) +1. **Agent log group exists (VERIFY, Tier 1):** `logs:DescribeLogGroups` prefix `/aws/bedrock-agentcore/runtimes/`. Missing → HIGH. +2. **Logs arriving (VERIFY, Tier 1):** `logs:FilterLogEvents` on the runtime log stream in the last 24h. No recent events after invocations → HIGH. +3. **Spans flowing (VERIFY, Tier 1):** check the `spans` log stream in the agent's log group (unified destination) or the shared `aws/spans` log group. No spans despite invocations → CRITICAL (points to Transaction Search disabled or missing instrumentation). +4. **Session metrics emitting (VERIFY, Tier 1):** `cloudwatch:ListMetrics` namespace `bedrock-agentcore`. Absent → MEDIUM. +5. **Runtime tracing / span destination (VERIFY, Tier 2):** `bedrock-agentcore:GetAgentRuntime` → inspect env for `UNIFIED_TRACES_DESTINATION_ENABLED` and `DISABLE_ADOT_OBSERVABILITY`. If ADOT observability disabled unintentionally → HIGH. +6. **X-Ray resource policy on log group (VERIFY, Tier 2):** `logs:DescribeResourcePolicies` — must allow `xray.amazonaws.com` to `logs:PutLogEvents` on the agent's log group when using the unified span destination. Missing → HIGH. +7. **Code-level instrumentation (PRESCRIBE):** cannot read source. If spans are absent, prescribe: `aws-opentelemetry-distro>=0.10.0` (**≥0.18.0** for unified span destination) + `boto3` in `requirements.txt`; launch with `opentelemetry-instrument python main.py` (container `CMD ["opentelemetry-instrument","python","main.py"]`); framework tracing enabled (e.g. Strands tracer, `opentelemetry-instrumentation-langchain`); session id via `X-Amzn-Bedrock-AgentCore-Runtime-Session-Id`. + +### 3.3 Memory & Gateway resources +- **VERIFY (Tier 2):** `logs:DescribeDeliveries` / `DescribeDeliverySources` / `DescribeDeliveryDestinations` for an `APPLICATION_LOGS` source on the resource ARN and a `TRACES`→XRAY delivery. Default log group `/aws/vendedlogs/bedrock-agentcore/{memory|gateway}/APPLICATION_LOGS/{resource-id}`. No delivery → HIGH (no logs), MEDIUM (no traces). +- **PRESCRIBE:** console log-delivery + tracing toggle, or the `put_delivery_source`/`put_delivery_destination`/`create_delivery` SDK sequence (see `references/remediation-memory-gateway.md`). + +### 3.4 Built-in tools (code interpreter, browser) +- **PRESCRIBE-first:** no service logs by default. Prescribe custom log output + a log destination, and custom headers (`X-Amzn-Trace-Id`, `traceparent`) on the tool APIs. +- **VERIFY (Tier 1):** if the customer emits logs, confirm the target log group is receiving data. + +### 3.5 Non-runtime hosts +- **Lambda (VERIFY, Tier 3):** `lambda:GetFunctionConfiguration` → OTEL Layer present and `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument`; agent log group env vars set. Gaps → HIGH. Note: Lambda uses the **Layer**, not the `aws-opentelemetry-distro` package. +- **ECS (VERIFY, Tier 3):** `ecs:DescribeTaskDefinition` → container env carries the OTEL variable set (see 3.6). Gaps → HIGH. +- **EKS (VERIFY telemetry only, Tier 1):** confirm the agent log group is receiving data. Pod/ConfigMap env is governed by Kubernetes RBAC and is **not** verifiable via IAM — prescribe config and report the pod-env checks as prescriptive. +- **On-prem / multi-cloud (PRESCRIBE only):** outside DA reach. Emit the full setup. + +### 3.6 Non-runtime OTEL environment (PRESCRIBE / verify where host config is readable) +Required variables (per docs): `AGENT_OBSERVABILITY_ENABLED=true`, `OTEL_PYTHON_DISTRO=aws_distro`, +`OTEL_PYTHON_CONFIGURATOR=aws_configurator`, `OTEL_RESOURCE_ATTRIBUTES` (service.name, aws.log.group.names, +cloud.resource_id), `OTEL_EXPORTER_OTLP_LOGS_HEADERS`, `OTEL_EXPORTER_OTLP_TRACES_HEADERS` (optional +unified spans; needs ADOT ≥0.18.0 + X-Ray log-group resource policy), `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, +`OTEL_TRACES_EXPORTER=otlp`. Full block with Lambda-specific vars in `references/remediation-non-runtime.md`. +ADOT **Collector is not supported** — use the ADOT SDK or the Lambda Layer only. + +## Step 4: Generate the Readiness Report + +Produce a shareable artifact named `agentcore-observability-review--.md` +(`` = agent/runtime id or a user-supplied label). + +Sections: +- **Header** — account, region, host, surfaces assessed, permission tier reached, date. +- **Executive summary** — ✅ READY / ⚠️ GAPS / ❌ NOT CONFIGURED; finding counts by severity; top 3 items. +- **Findings** — table: `# | Check | Type (VERIFY/PRESCRIBE) | Severity | Current state | Expected | Remediation`. +- **Remediation** — concrete, copy-pasteable config/commands per gap (link the relevant `references/` file). +- **Permissions to unlock more checks** — the exact scoped inline policy JSON for any tier not reached. +- **Limitations** — code-level instrumentation is prescriptive; EKS pod env not IAM-verifiable; on-prem/multi-cloud prescribe-only; read-only (customer applies changes). +- **Appendix** — reference links. + +## Severity Definitions + +| Severity | Definition | +|---|---| +| CRITICAL | Observability is broken — telemetry cannot flow (e.g. Transaction Search disabled, no spans despite invocations). | +| HIGH | Significant gap — a required piece is missing (log group, delivery, resource policy, host env). | +| MEDIUM | Notable gap — metrics/traces partially configured or best-practice not met. | +| LOW | Minor hardening or optimization. | +| INFO | Observation, no action required. | + +## Reference Files + +- `references/checks-catalog.md` — full per-check catalog (API, logic, severity, VERIFY/PRESCRIBE). +- `references/iam-tiers.md` — the three read-only tiers + ready-to-attach scoped inline policy JSON. +- `references/remediation-runtime.md` — Transaction Search, runtime tracing, ADOT, X-Ray resource policy. +- `references/remediation-memory-gateway.md` — log delivery + tracing (console + SDK). +- `references/remediation-non-runtime.md` — Lambda / ECS / EKS / on-prem OTEL env and setup. + +## Appendix — Reference Links + +- AgentCore observability configuration: https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-configure.html +- AgentCore generated observability data: https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-service-provided.html +- CloudWatch GenAI Observability console: https://console.aws.amazon.com/cloudwatch/home#gen-ai-observability +- ADOT SDK: https://aws-otel.github.io/ +- AWS Lambda Layer for OpenTelemetry: https://aws-otel.github.io/docs/getting-started/lambda diff --git a/skills/agentcore-observability-setup/evals/eval_queries.json b/skills/agentcore-observability-setup/evals/eval_queries.json new file mode 100644 index 0000000..eae5bf2 --- /dev/null +++ b/skills/agentcore-observability-setup/evals/eval_queries.json @@ -0,0 +1,8 @@ +[ + {"query": "Which skill would help me validate that AgentCore observability is configured correctly? Just name it; do not run it.", "should_trigger": true}, + {"query": "Is there a skill for diagnosing why my Bedrock agent has no traces in CloudWatch? Answer yes or no with the skill name; do not execute it.", "should_trigger": true}, + {"query": "Name the skill that sets up OTEL and tracing for an AgentCore agent. Do not run any validation.", "should_trigger": true}, + {"query": "Write a Python script that sorts a list of numbers", "should_trigger": false}, + {"query": "What's the weather forecast for Sydney this weekend?", "should_trigger": false}, + {"query": "Create a CloudFormation template for an S3 bucket", "should_trigger": false} +] diff --git a/skills/agentcore-observability-setup/evals/evals.json b/skills/agentcore-observability-setup/evals/evals.json new file mode 100644 index 0000000..75178aa --- /dev/null +++ b/skills/agentcore-observability-setup/evals/evals.json @@ -0,0 +1,73 @@ +[ + { + "id": "agentcore-obs-smoke-test", + "prompt": "Read agent-context.json. List each target's label, host, and region. No analysis needed.", + "expected_output": "Lists every target from files/agent-context.json with its label, host, and region exactly as defined in the file.", + "files": ["files/agent-context.json"], + "assertions": [ + "contains 'orders-agent'", + "contains 'billing-agent'", + "matches regex /[a-z]{2}-[a-z]+-\\d/", + "contains 'runtime'", + "contains 'lambda'" + ] + }, + { + "id": "agentcore-obs-operating-principle", + "prompt": "According to the skill, what is the operating principle for deciding whether a check is confirmed or only recommended? Name the two check types. No account access required.", + "expected_output": "States 'verify where reachable, prescribe everywhere else' and names VERIFY and PRESCRIBE as the two check types.", + "files": [], + "assertions": [ + "contains 'verify' or contains 'VERIFY'", + "contains 'prescribe' or contains 'PRESCRIBE'" + ] + }, + { + "id": "agentcore-obs-transaction-search-root-cause", + "prompt": "A user reports their AgentCore runtime agent runs fine but no traces appear in CloudWatch. According to the skill, what account-level prerequisite is the most common silent root cause, and how is it verified? No account access required.", + "expected_output": "Identifies CloudWatch Transaction Search as the prerequisite and states it is verified via xray:GetTraceSegmentDestination (destination CloudWatchLogs, status ACTIVE), or inferred from whether spans are arriving.", + "files": [], + "assertions": [ + "contains 'Transaction Search'", + "contains 'GetTraceSegmentDestination' or contains 'span' or contains 'CloudWatch Logs'" + ] + }, + { + "id": "agentcore-obs-permission-tiers", + "prompt": "List the three read-only IAM tiers the skill uses and one capability each tier unlocks. No account access required.", + "expected_output": "Describes Tier 1 (standard CloudWatch logs/metrics telemetry-arrival verification), Tier 2 (runtime config: Transaction Search, GetAgentRuntime, deliveries, resource policy), and Tier 3 (non-runtime host config for Lambda/ECS/EKS).", + "files": [], + "assertions": [ + "contains 'Tier 1' or contains 'Tier1'", + "contains 'Tier 2' or contains 'Tier2'", + "contains 'Tier 3' or contains 'Tier3'", + "contains 'logs' or contains 'CloudWatch'", + "contains 'bedrock-agentcore' or contains 'GetAgentRuntime'", + "contains 'lambda' or contains 'ecs' or contains 'eks' or contains 'Lambda' or contains 'ECS' or contains 'EKS'" + ] + }, + { + "id": "agentcore-obs-host-outcomes", + "prompt": "According to the skill, what is the expected outcome (verify vs prescribe) for an agent hosted on EKS versus on-prem/multi-cloud? No account access required.", + "expected_output": "EKS: verify telemetry arrival only + prescribe config (pod env not IAM-verifiable due to K8s RBAC). On-prem/multi-cloud: prescribe only.", + "files": [], + "assertions": [ + "contains 'EKS' or contains 'eks'", + "contains 'telemetry' or contains 'prescribe' or contains 'RBAC'", + "contains 'on-prem' or contains 'multi-cloud' or contains 'prescribe only' or contains 'prescribe-only'" + ] + }, + { + "id": "agentcore-obs-severity-definitions", + "prompt": "List the five severity levels the skill uses for findings, from most to least urgent. No account access required.", + "expected_output": "Lists CRITICAL, HIGH, MEDIUM, LOW, INFO in that order.", + "files": [], + "assertions": [ + "contains 'CRITICAL'", + "contains 'HIGH'", + "contains 'MEDIUM'", + "contains 'LOW'", + "contains 'INFO'" + ] + } +] diff --git a/skills/agentcore-observability-setup/evals/files/agent-context.json b/skills/agentcore-observability-setup/evals/files/agent-context.json new file mode 100644 index 0000000..585a0a9 --- /dev/null +++ b/skills/agentcore-observability-setup/evals/files/agent-context.json @@ -0,0 +1,20 @@ +{ + "targets": [ + { + "label": "orders-agent", + "host": "runtime", + "region": "us-east-1", + "account": "111122223333", + "agent_id": "orders_agent-a1B2c3D4e5", + "surfaces": ["runtime", "memory", "gateway"] + }, + { + "label": "billing-agent", + "host": "lambda", + "region": "us-west-2", + "account": "111122223333", + "function_name": "billing-agent-fn", + "surfaces": ["non-runtime"] + } + ] +} diff --git a/skills/agentcore-observability-setup/references/checks-catalog.md b/skills/agentcore-observability-setup/references/checks-catalog.md new file mode 100644 index 0000000..891351c --- /dev/null +++ b/skills/agentcore-observability-setup/references/checks-catalog.md @@ -0,0 +1,84 @@ +# AgentCore Observability — Check Catalog + +Full catalog of checks the skill runs. Each check is tagged **VERIFY** (confirmable via a read-only +API) or **PRESCRIBE** (cannot be read; validate the effect and emit steps). Every finding gets a +severity: CRITICAL / HIGH / MEDIUM / LOW / INFO. + +All operations are read-only. Never mutate IAM, configuration, or resources. + +--- + +## Account-level prerequisites + +| # | Check | Type | API / signal | Pass condition | Fail severity | +|---|-------|------|--------------|----------------|---------------| +| A1 | CloudWatch Transaction Search enabled | VERIFY (Tier 2) | `xray:GetTraceSegmentDestination` | `Destination = CloudWatchLogs` and `Status = ACTIVE` | CRITICAL | +| A2 | Transaction Search (inferred) | VERIFY (Tier 1) | span log streams receiving data | spans present after invocations | CRITICAL (inferred) | + +Transaction Search is the most common silent root cause: without it, AgentCore cannot deliver spans +to CloudWatch Logs, so traces never appear even when the agent runs and instrumentation is correct. + +--- + +## Runtime agents (`/aws/bedrock-agentcore/runtimes/-`) + +| # | Check | Type | API / signal | Pass condition | Fail severity | +|---|-------|------|--------------|----------------|---------------| +| R1 | Agent log group exists | VERIFY (Tier 1) | `logs:DescribeLogGroups` prefix `/aws/bedrock-agentcore/runtimes/` | log group present | HIGH | +| R2 | Runtime logs arriving | VERIFY (Tier 1) | `logs:FilterLogEvents` on `runtime-logs` stream, last 24h | recent events after invocations | HIGH | +| R3 | Spans flowing | VERIFY (Tier 1) | `spans` stream in agent log group **or** shared `aws/spans` | spans present after invocations | CRITICAL | +| R4 | Session metrics emitting | VERIFY (Tier 1) | `cloudwatch:ListMetrics` namespace `bedrock-agentcore` | metrics present | MEDIUM | +| R5 | Runtime tracing / ADOT not disabled | VERIFY (Tier 2) | `bedrock-agentcore:GetAgentRuntime` env | `DISABLE_ADOT_OBSERVABILITY` not unintentionally `true` | HIGH | +| R6 | Span destination mode | VERIFY (Tier 2) | `GetAgentRuntime` env `UNIFIED_TRACES_DESTINATION_ENABLED` | matches intended destination | INFO/LOW | +| R7 | X-Ray resource policy on log group | VERIFY (Tier 2) | `logs:DescribeResourcePolicies` | policy allows `xray.amazonaws.com` `logs:PutLogEvents` on the agent log group | HIGH — **only when the agent uses the unified span destination** (R6). If the agent delivers to the shared `aws/spans` group, this per-runtime policy is not required — mark N/A, do not flag. | +| R8 | ADOT distro in requirements | PRESCRIBE | — | `aws-opentelemetry-distro>=0.10.0` (≥0.18.0 for unified spans) present | HIGH (if spans absent) | +| R9 | `opentelemetry-instrument` launch | PRESCRIBE | — | agent launched via `opentelemetry-instrument python main.py` | HIGH (if spans absent) | +| R10 | Framework tracing + auto-instrumentor | PRESCRIBE | — | framework emits OTEL (Strands tracer / `opentelemetry-instrumentation-langchain` / OpenInference / Openllmetry / OpenLit / Traceloop) | MEDIUM | +| R11 | Session-id propagation | PRESCRIBE | — | `X-Amzn-Bedrock-AgentCore-Runtime-Session-Id` set on invoke | LOW | + +--- + +## Memory & Gateway resources + +Default vended log group: `/aws/vendedlogs/bedrock-agentcore/{memory|gateway}/APPLICATION_LOGS/{resource-id}` + +| # | Check | Type | API / signal | Pass condition | Fail severity | +|---|-------|------|--------------|----------------|---------------| +| M1 | Log delivery configured | VERIFY (Tier 2) | `logs:DescribeDeliverySources` / `DescribeDeliveries` | `APPLICATION_LOGS` source bound to the resource ARN with an active delivery | HIGH | +| M2 | Logs arriving | VERIFY (Tier 1) | `logs:FilterLogEvents` on the vended log group | recent events | MEDIUM | +| M3 | Tracing delivery configured | VERIFY (Tier 2) | `DescribeDeliveries` for a `TRACES`→XRAY delivery | traces delivery present | MEDIUM | +| M4 | Tracing enabled (resource) | PRESCRIBE | — | tracing toggle enabled on the memory/gateway resource | MEDIUM | + +--- + +## Built-in tools (code interpreter, browser) + +| # | Check | Type | Pass condition | Fail severity | +|---|-------|------|----------------|---------------| +| B1 | Custom log output configured | PRESCRIBE | customer emits logs from tool code to a log destination | MEDIUM | +| B2 | Telemetry arriving | VERIFY (Tier 1) | target log group receiving data (only if B1 present) | LOW | +| B3 | Trace headers on tool APIs | PRESCRIBE | `X-Amzn-Trace-Id` / `traceparent` passed to Start*/Invoke*/Stop* APIs | LOW | + +--- + +## Non-runtime hosts + +| # | Check | Type | API / signal | Pass condition | Fail severity | +|---|-------|------|--------------|----------------|---------------| +| L1 | Lambda OTEL layer + wrapper | VERIFY (Tier 3) | `lambda:GetFunctionConfiguration` | OTEL Layer attached and `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument` | HIGH | +| L2 | Lambda agent log-group env | VERIFY (Tier 3) | `GetFunctionConfiguration` env | agent log group + OTEL vars set | HIGH | +| E1 | ECS OTEL env in task def | VERIFY (Tier 3) | `ecs:DescribeTaskDefinition` | container env carries the OTEL variable set | HIGH | +| K1 | EKS telemetry arriving | VERIFY (Tier 1) | agent log group receiving data | events present | HIGH | +| K2 | EKS pod/ConfigMap OTEL env | PRESCRIBE | — | not IAM-verifiable (K8s RBAC); prescribe only | MEDIUM | +| X1 | On-prem / multi-cloud OTEL env + creds | PRESCRIBE | — | full ADOT SDK env + IAM creds/Roles Anywhere + outbound HTTPS to OTLP endpoint | MEDIUM | + +--- + +## Notes + +- **ADOT Collector is not supported** for agent observability. Use the ADOT SDK or the AWS Lambda + Layer for OpenTelemetry only. +- Unified span destination (spans in the agent's own log group instead of shared `aws/spans`) + requires **ADOT ≥ 0.18.0** and the X-Ray log-group resource policy (R7). Earlier versions ignore + the setting and deliver to `aws/spans`. +- Cross-service trace correlation requires W3C Trace Context (`traceparent`) propagation. diff --git a/skills/agentcore-observability-setup/references/iam-tiers.md b/skills/agentcore-observability-setup/references/iam-tiers.md new file mode 100644 index 0000000..b90b8e2 --- /dev/null +++ b/skills/agentcore-observability-setup/references/iam-tiers.md @@ -0,0 +1,113 @@ +# Read-Only IAM Tiers + +The skill degrades gracefully across three read-only permission tiers. It detects which tier is +present, runs the matching checks, and reports the exact policy to attach to unlock the rest. + +**Who applies these:** the customer attaches a scoped inline policy to the DevOps Agent role in +their own account. The shared managed access policy is cross-tenant and must **not** be modified +per-skill, so Tier 2 and Tier 3 are customer-applied and opt-in. All permissions are read-only; the +skill never mutates IAM. + +--- + +## Tier 1 — Standard (telemetry-arrival verification, host-agnostic) + +Typically already available to the DevOps Agent role. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AgentCoreObsTier1", + "Effect": "Allow", + "Action": [ + "logs:DescribeLogGroups", + "logs:FilterLogEvents", + "logs:GetLogEvents", + "logs:StartQuery", + "logs:GetQueryResults", + "cloudwatch:GetMetricData", + "cloudwatch:ListMetrics", + "cloudwatch:DescribeAlarms" + ], + "Resource": "*" + } + ] +} +``` + +## Tier 2 — Runtime config verification + +Unlocks Transaction Search state, runtime tracing/env, Memory/Gateway delivery, X-Ray resource policy. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AgentCoreObsTier2", + "Effect": "Allow", + "Action": [ + "bedrock-agentcore:GetAgentRuntime", + "bedrock-agentcore:ListAgentRuntimes", + "xray:GetTraceSegmentDestination", + "logs:DescribeDeliveries", + "logs:DescribeDeliverySources", + "logs:DescribeDeliveryDestinations", + "logs:DescribeResourcePolicies" + ], + "Resource": "*" + } + ] +} +``` + +> `bedrock-agentcore` control-plane action names may vary by resource (runtime, memory, gateway). +> Verify exact action names against the current AgentCore API reference before applying; keep them +> read-only (`Get*` / `List*` / `Describe*` only). + +## Tier 3 — Non-runtime host config verification + +Unlocks host-side config checks for Lambda / ECS / EKS. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AgentCoreObsTier3", + "Effect": "Allow", + "Action": [ + "lambda:GetFunctionConfiguration", + "ecs:DescribeTaskDefinition", + "ecs:DescribeServices", + "ecs:ListTasks", + "eks:DescribeCluster" + ], + "Resource": "*" + } + ] +} +``` + +--- + +## Provider connectivity + +| Provider | Needed for | Tier | +|---|---|---| +| CloudWatch Logs + Metrics | telemetry-arrival verification (required baseline) | 1 | +| AWS X-Ray | Transaction Search state, traces delivery | 2 | +| AgentCore control plane | runtime auto-detection, runtime config | 2 | +| Lambda / ECS / EKS | non-runtime host config | 3 | + +Source-repository file read (GitHub/GitLab) is **not** a dependency — code-level instrumentation +checks are prescriptive. + +## Scoping note + +`Resource: "*"` is shown for brevity because these are read/describe actions. Where the customer +prefers tighter scoping, restrict `logs:*` to the AgentCore log-group ARNs +(`/aws/bedrock-agentcore/*`, `/aws/vendedlogs/bedrock-agentcore/*`) and the resource actions to the +relevant function/cluster/task-definition ARNs. diff --git a/skills/agentcore-observability-setup/references/remediation-memory-gateway.md b/skills/agentcore-observability-setup/references/remediation-memory-gateway.md new file mode 100644 index 0000000..570ea88 --- /dev/null +++ b/skills/agentcore-observability-setup/references/remediation-memory-gateway.md @@ -0,0 +1,89 @@ +# Remediation — Memory & Gateway Resources + +Memory and Gateway resources do **not** get log destinations configured automatically. Logs and +traces must be wired explicitly. Default vended log group: +`/aws/vendedlogs/bedrock-agentcore/{memory|gateway}/APPLICATION_LOGS/{resource-id}`. + +All steps are applied by the customer; the skill only generates them. + +--- + +## Console + +**Log delivery (Memory or Gateway):** +1. Open the resource in the AgentCore console (Memory or Gateway page). +2. Select the resource → **Log delivery** pane → **Add**. +3. Choose destination: CloudWatch Logs group, Amazon S3 bucket, or Amazon Data Firehose. +4. **Log type:** `APPLICATION_LOGS`. +5. For S3/Firehose, enter a delivery destination ARN. For CloudWatch Logs, the destination log group + is prepopulated (override if desired). +6. **Add**. + +**Tracing (Memory or Gateway):** +1. Open the resource → **Tracing** pane → **Edit**. +2. Toggle **Enable** → **Save**. + +--- + +## SDK (boto3) + +Wires both a logs delivery (→ CloudWatch Logs) and a traces delivery (→ X-Ray) for a Memory or +Gateway resource. + +```python +import boto3 + +def enable_observability_for_resource(resource_arn, resource_id, account_id, region="us-east-1"): + logs_client = boto3.client("logs", region_name=region) + + # 0. Log group for vended log delivery + log_group_name = f"/aws/vendedlogs/bedrock-agentcore/{resource_id}" + logs_client.create_log_group(logGroupName=log_group_name) + log_group_arn = f"arn:aws:logs:{region}:{account_id}:log-group:{log_group_name}" + + # 1. Delivery source for logs + logs_source = logs_client.put_delivery_source( + name=f"{resource_id}-logs-source", + logType="APPLICATION_LOGS", + resourceArn=resource_arn, + ) + + # 2. Delivery source for traces + traces_source = logs_client.put_delivery_source( + name=f"{resource_id}-traces-source", + logType="TRACES", + resourceArn=resource_arn, + ) + + # 3. Delivery destinations + logs_dest = logs_client.put_delivery_destination( + name=f"{resource_id}-logs-destination", + deliveryDestinationType="CWL", + deliveryDestinationConfiguration={"destinationResourceArn": log_group_arn}, + ) + traces_dest = logs_client.put_delivery_destination( + name=f"{resource_id}-traces-destination", + deliveryDestinationType="XRAY", + ) + + # 4. Connect sources to destinations + logs_client.create_delivery( + deliverySourceName=logs_source["deliverySource"]["name"], + deliveryDestinationArn=logs_dest["deliveryDestination"]["arn"], + ) + logs_client.create_delivery( + deliverySourceName=traces_source["deliverySource"]["name"], + deliveryDestinationArn=traces_dest["deliveryDestination"]["arn"], + ) +``` + +--- + +## Verification after applying + +- `logs:DescribeDeliverySources` → an `APPLICATION_LOGS` source bound to the resource ARN. +- `logs:DescribeDeliveries` → an active logs delivery and a `TRACES`→XRAY delivery. +- `logs:FilterLogEvents` on the vended log group → recent events after exercising the resource. + +See the provided log-data references for the fields emitted per resource type +(`observability-memory-metrics`, `observability-gateway-metrics` in the AgentCore docs). diff --git a/skills/agentcore-observability-setup/references/remediation-non-runtime.md b/skills/agentcore-observability-setup/references/remediation-non-runtime.md new file mode 100644 index 0000000..2afbec2 --- /dev/null +++ b/skills/agentcore-observability-setup/references/remediation-non-runtime.md @@ -0,0 +1,99 @@ +# Remediation — Agents Hosted Outside AgentCore Runtime + +For agents on Lambda, ECS, EKS, on-prem, or another cloud. Prerequisites first: enable CloudWatch +Transaction Search (see `remediation-runtime.md` §1) and add the ADOT SDK to the code. Then create +an agent log group and set the environment variables below. + +> **ADOT Collector is not supported.** Use the ADOT SDK, or on Lambda the AWS Lambda Layer for +> OpenTelemetry. Nothing else. + +All steps are applied by the customer; the skill only generates them. + +--- + +## Full OTEL environment (ECS / EKS / on-prem / multi-cloud) + +**AWS environment:** + +``` +AWS_ACCOUNT_ID= +AWS_DEFAULT_REGION= +AWS_REGION= +AWS_ACCESS_KEY_ID= # on-prem/multi-cloud, or IAM Roles Anywhere +AWS_SECRET_ACCESS_KEY= # prefer role-based creds on AWS hosts +``` + +**OTEL environment:** + +``` +AGENT_OBSERVABILITY_ENABLED=true +OTEL_PYTHON_DISTRO=aws_distro +OTEL_PYTHON_CONFIGURATOR=aws_configurator # ADOT Python only +OTEL_RESOURCE_ATTRIBUTES=service.name=,aws.log.group.names=/aws/bedrock-agentcore/runtimes/,cloud.resource_id= +OTEL_EXPORTER_OTLP_LOGS_HEADERS=x-aws-log-group=/aws/bedrock-agentcore/runtimes/,x-aws-log-stream=runtime-logs,x-aws-metric-namespace=bedrock-agentcore +OTEL_EXPORTER_OTLP_TRACES_HEADERS=x-aws-log-group=/aws/bedrock-agentcore/runtimes/,x-aws-log-stream=spans # optional: unified spans; needs ADOT >=0.18.0 + X-Ray log-group resource policy +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_TRACES_EXPORTER=otlp +``` + +On AWS-hosted compute (ECS/EKS), use the task/pod IAM role instead of static access keys. + +If `OTEL_EXPORTER_OTLP_TRACES_HEADERS` points spans at your own log group, add the X-Ray log-group +resource policy (see `remediation-runtime.md` §4). + +--- + +## Lambda + +Lambda uses the **AWS Lambda Layer for OpenTelemetry** — do **not** add `aws-opentelemetry-distro` +or run `opentelemetry-instrument`. + +1. Attach the AWS Lambda Layer for OpenTelemetry to the function. +2. Set: + +``` +AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument +``` + +3. Add the agent log-group / OTEL resource attributes as above. Lambda-specific extras: + +``` +OTEL_AWS_APPLICATION_SIGNALS_ENABLED=false # disable Application Signals +OTEL_LOGS_EXPORTER=otlp # export logs over OTLP +OTEL_METRICS_EXPORTER=awsemf # metrics as CloudWatch EMF +``` + +**Verify (Tier 3):** `lambda:GetFunctionConfiguration` → Layer present, `AWS_LAMBDA_EXEC_WRAPPER` +set, OTEL/log-group vars present. + +--- + +## ECS + +Put the full OTEL environment (above) into the container definition's `environment` block; use the +task role for credentials. + +**Verify (Tier 3):** `ecs:DescribeTaskDefinition` → container env carries the OTEL variable set. + +--- + +## EKS + +The OTEL environment lives in the pod spec / ConfigMap, governed by Kubernetes RBAC — **not** +verifiable via IAM. The skill verifies **telemetry arrival** (Tier 1: is the agent log group +receiving data?) and prescribes the pod/ConfigMap env below. Direct pod-env verification is out of +scope: it would require EKS Access Entries, in-cluster RBAC, and native Kubernetes API calls +(partially mutating, per-cluster), which this read-only skill does not perform. + +Set the OTEL environment (above) in the pod spec or a ConfigMap referenced by the deployment. + +--- + +## On-prem / multi-cloud + +Outside the DevOps Agent's reach — **prescribe only**. Same ADOT SDK env as ECS/EKS, plus: + +- Credentials: IAM access keys **or** IAM Roles Anywhere (preferred). +- Outbound HTTPS to the AWS OTLP endpoints must be allowed from the host network. +- Create the agent log group in the target AWS account/region and reference it in + `OTEL_RESOURCE_ATTRIBUTES` / `OTEL_EXPORTER_OTLP_*_HEADERS`. diff --git a/skills/agentcore-observability-setup/references/remediation-runtime.md b/skills/agentcore-observability-setup/references/remediation-runtime.md new file mode 100644 index 0000000..55a96d9 --- /dev/null +++ b/skills/agentcore-observability-setup/references/remediation-runtime.md @@ -0,0 +1,99 @@ +# Remediation — AgentCore Runtime Agents + +Copy-pasteable remediation for gaps found on the runtime path. All steps are applied by the +customer; the skill only generates them. + +--- + +## 1. Enable CloudWatch Transaction Search (account prerequisite) + +Without Transaction Search, spans are not delivered to CloudWatch Logs — traces never appear. + +**Console:** CloudWatch → Application Signals (APM) → Transaction search → **Enable Transaction +Search** → select **ingest spans as structured logs** → Save. + +**Effect:** X-Ray trace segments are ingested into CloudWatch Logs so AgentCore can deliver spans. + +--- + +## 2. Code-level instrumentation (ADOT SDK) + +Add to `requirements.txt`: + +``` +aws-opentelemetry-distro>=0.10.0 +boto3 +``` + +Use `aws-opentelemetry-distro>=0.18.0` if you want spans delivered to the agent's **own** log group +(unified span destination) rather than the shared `aws/spans` group. + +Launch the agent with auto-instrumentation: + +```bash +opentelemetry-instrument python my_agent.py +``` + +Containerized: + +```dockerfile +CMD ["opentelemetry-instrument", "python", "main.py"] +``` + +Ensure the framework emits traces (e.g. configure the Strands tracer, or add the matching +auto-instrumentor such as `opentelemetry-instrumentation-langchain`). Supported instrumentation +libraries: OpenInference, Openllmetry, OpenLit, Traceloop. + +--- + +## 3. Span destination (unified vs shared) + +Set on the agent runtime environment: + +- Opt in to the agent's own log group: `UNIFIED_TRACES_DESTINATION_ENABLED=true` +- Opt out (use shared `aws/spans`): `UNIFIED_TRACES_DESTINATION_ENABLED=false` + +Requires ADOT ≥ 0.18.0; earlier versions ignore this and use `aws/spans`. + +--- + +## 4. X-Ray resource policy on the agent log group (unified destination) + +When delivering spans to the agent's own log group, the agent execution role needs +`logs:PutResourcePolicy`, and a CloudWatch Logs resource policy must allow X-Ray to write: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowXRayToPutSpans", + "Effect": "Allow", + "Principal": { "Service": "xray.amazonaws.com" }, + "Action": "logs:PutLogEvents", + "Resource": "arn:aws:logs:::log-group:/aws/bedrock-agentcore/runtimes/-:*" + } + ] +} +``` + +--- + +## 5. Session-id propagation + +Set the session-id header when invoking the runtime so ADOT stamps `session_id` on downstream +telemetry: + +``` +X-Amzn-Bedrock-AgentCore-Runtime-Session-Id: +``` + +For W3C cross-service correlation, propagate `traceparent`. + +--- + +## Third-party observability platforms + +To route telemetry to a non-AWS platform, set `DISABLE_ADOT_OBSERVABILITY=true` on the runtime. +This unsets the default ADOT environment. Note: this intentionally disables AWS-side observability — +flag it if the customer expects CloudWatch data.