feat(terraform): bring-your-own Data Prepper pipeline + gp3 StorageClass

charts/observability-stack/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: observability-stack
 description: OpenTelemetry-native observability platform for microservices, web apps, and AI agents
 type: application
-version: 0.3.0
+version: 0.4.0
 appVersion: "3.7.0"
 
 home: https://github.com/opensearch-project/observability-stack

charts/observability-stack/templates/otel-collector-configmap.yaml

@@ -172,7 +172,10 @@ data:
               - set(body, ToKeyValueString(body)) where IsMap(body)
     exporters:
       debug:
-        verbosity: detailed
+        # basic logs per-batch counts only. detailed serializes every span/log/
+        # metric to stdout, which throttles the pipeline at high throughput
+        # (observed ~6x slower trace ingest).
+        verbosity: basic
       otlp/opensearch:
         endpoint: "{{ .Release.Name }}-data-prepper:21890"
         tls:

terraform/aws/observability-stack.tf

@@ -109,6 +109,61 @@ resource "kubernetes_namespace" "observability" {
   depends_on = [module.eks]
 }
 
+# EKS pre-creates only gp2 by default, but opensearch_storage_class / cortex_storage_class
+# default to gp3 (better IOPS/$ for bulk ingest). Without a matching StorageClass the PVCs
+# stay Pending and the release times out, so create it via the EBS CSI driver.
+# WaitForFirstConsumer matches gp2 so a PVC binds on the node its pod lands on.
+# Gated by create_gp3_storage_class: clusters that already ship gp3 (EKS Auto Mode,
+# a manually created class, a shared cluster) must opt out or the apply fails with
+# "storageclasses.storage.k8s.io \"gp3\" already exists".
+resource "kubernetes_storage_class" "gp3" {
+  count = var.create_gp3_storage_class ? 1 : 0
+
+  metadata {
+    name = "gp3"
+  }
+  storage_provisioner    = "ebs.csi.aws.com"
+  volume_binding_mode    = "WaitForFirstConsumer"
+  allow_volume_expansion = true
+  parameters = {
+    type = "gp3"
+  }
+
+  depends_on = [module.eks]
+}
+
+# Credentials passed into the BYO pipeline Secret below. When the corresponding
+# var is empty the chart default applies, so read both defaults from the chart's
+# values.yaml (its single source of truth) rather than re-hardcoding the literals
+# here. That way the rendered Secret and the cluster can never authenticate with
+# different credentials if the chart default changes.
+locals {
+  chart_values                  = yamldecode(file("${path.module}/../../charts/observability-stack/values.yaml"))
+  opensearch_username_effective = var.opensearch_username != "" ? var.opensearch_username : local.chart_values.opensearchUsername
+  opensearch_password_effective = var.opensearch_password != "" ? var.opensearch_password : local.chart_values.opensearchPassword
+}
+
+# Bring-your-own Data Prepper pipeline. When data_prepper_pipeline_secret_file
+# is set, create the Secret the chart's dataPrepperManageSecret=false gate expects
+# (same name the subchart mounts) and order it before the release so Data Prepper
+# finds it at boot. The chart default leaves this empty and renders its own Secret.
+resource "kubernetes_secret" "data_prepper_pipeline" {
+  count = var.data_prepper_pipeline_secret_file != "" ? 1 : 0
+
+  metadata {
+    name      = "data-prepper-pipeline"
+    namespace = kubernetes_namespace.observability.metadata[0].name
+  }
+
+  data = {
+    "pipelines.yaml" = templatefile(var.data_prepper_pipeline_secret_file, {
+      opensearch_user      = local.opensearch_username_effective
+      opensearch_password  = local.opensearch_password_effective
+      trace_flush_interval = var.data_prepper_trace_flush_interval
+    })
+  }
+}
+
 resource "helm_release" "observability_stack" {
   name      = "obs-stack"
   chart     = "${path.module}/../../charts/observability-stack"
@@ -121,9 +176,20 @@ resource "helm_release" "observability_stack" {
 
   values = concat(
     [file("${path.module}/values-eks.yaml")],
-    var.anonymous_auth ? [file("${path.module}/../../charts/observability-stack/values-anonymous-auth.yaml")] : []
+    var.anonymous_auth ? [file("${path.module}/../../charts/observability-stack/values-anonymous-auth.yaml")] : [],
+    [for f in var.extra_helm_values : file(f)]
   )
 
+  # Bring-your-own pipeline: skip the chart's managed Secret so Data Prepper
+  # mounts the one created above.
+  dynamic "set" {
+    for_each = var.data_prepper_pipeline_secret_file != "" ? [1] : []
+    content {
+      name  = "dataPrepperManageSecret"
+      value = "false"
+    }
+  }
+
   # --- TLS / Domain (conditional) ---
   dynamic "set" {
     for_each = local.enable_tls ? [1] : []
@@ -183,7 +249,14 @@ resource "helm_release" "observability_stack" {
     value = var.enable_otel_demo ? "true" : "false"
   }
 
-  # --- Custom password (conditional) ---
+  # --- Custom credentials (conditional) ---
+  dynamic "set" {
+    for_each = var.opensearch_username != "" ? [1] : []
+    content {
+      name  = "opensearchUsername"
+      value = var.opensearch_username
+    }
+  }
   dynamic "set_sensitive" {
     for_each = var.opensearch_password != "" ? [1] : []
     content {
@@ -260,5 +333,7 @@ resource "helm_release" "observability_stack" {
 
   depends_on = [
     helm_release.aws_lb_controller,
+    kubernetes_secret.data_prepper_pipeline,
+    kubernetes_storage_class.gp3,
   ]
 }

terraform/aws/variables.tf

@@ -65,8 +65,14 @@ variable "anonymous_auth" {
   default     = false
 }
 
+variable "opensearch_username" {
+  description = "OpenSearch admin username. Leave empty to use the chart default (opensearchUsername in values.yaml)."
+  type        = string
+  default     = ""
+}
+
 variable "opensearch_password" {
-  description = "OpenSearch admin password. Leave empty to use chart default."
+  description = "OpenSearch admin password. Leave empty to use the chart default (opensearchPassword in values.yaml)."
   type        = string
   default     = ""
   sensitive   = true
@@ -93,6 +99,24 @@ variable "tags" {
   }
 }
 
+variable "extra_helm_values" {
+  description = "Additional Helm values file paths layered onto the release, applied last so they win. Use for deployment-specific overrides without editing chart defaults."
+  type        = list(string)
+  default     = []
+}
+
+variable "create_gp3_storage_class" {
+  description = "Create a gp3 StorageClass via the EBS CSI driver. Set false on clusters that already provide gp3 (EKS Auto Mode, a manually created class, a shared cluster) to avoid an \"already exists\" apply error."
+  type        = bool
+  default     = true
+}
+
+variable "data_prepper_pipeline_secret_file" {
+  description = "Path to a pipelines.yaml template for a bring-your-own Data Prepper pipeline. When set, terraform creates the data-prepper-pipeline Secret from it and sets dataPrepperManageSecret=false so the chart renders no pipeline Secret. The template is rendered with templatefile and may reference opensearch_user, opensearch_password, and trace_flush_interval. Empty leaves the chart's managed pipeline in place."
+  type        = string
+  default     = ""
+}
+
 # ============================================================================
 # OpenSearch sizing
 # ============================================================================