feat(metrics): count HTTP/2 frames by type (without logging payloads)

Dockerfile

@@ -2,8 +2,31 @@ FROM debian:bullseye AS builder
 # Using Debian instead of the official Golang image because it’s based on newer OS versions
 # with newer glibc, which causes compatibility issues.
 
-RUN apt-get update && apt-get install -y \
-    curl git build-essential pkg-config libsystemd-dev
+# deb.debian.org publishes a bullseye-security index that can advertise .debs
+# already pruned from the pool after a point release, so the build fails with a
+# 404 on an exact version (seen 2026-09-05: git 1:2.30.2-1+deb11u5). It is a
+# mirror-side inconsistency, so clearing local lists and retrying do not help —
+# both were tried and both still failed on every attempt.
+#
+# snapshot.debian.org serves index and pool as a matched pair at a point in
+# time, which makes this build reproducible and immune to archive rotation.
+# Check-Valid-Until is disabled because a pinned snapshot's Release file is
+# intentionally older than apt's freshness window.
+#
+# bullseye is oldstable and its archive keeps rotating; without this pin the
+# failure recurs and blocks every build in the repo, releases included.
+ARG DEBIAN_SNAPSHOT=20260801T000000Z
+RUN set -eux; \
+    printf '%s\n' \
+      "deb http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/ bullseye main" \
+      "deb http://snapshot.debian.org/archive/debian-security/${DEBIAN_SNAPSHOT}/ bullseye-security main" \
+      "deb http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/ bullseye-updates main" \
+      > /etc/apt/sources.list; \
+    rm -rf /var/lib/apt/lists/*; \
+    apt-get -o Acquire::Check-Valid-Until=false -o Acquire::Retries=5 update; \
+    apt-get install -y --no-install-recommends -o Acquire::Retries=5 \
+        ca-certificates curl git build-essential pkg-config libsystemd-dev; \
+    rm -rf /var/lib/apt/lists/*
 
 ARG GO_VERSION=1.26.5
 RUN curl -fsSL https://go.dev/dl/go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz -o go.tar.gz && \

containers/container.go

@@ -902,6 +902,26 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R
 	return ip2fqdn
 }
 
+// payloadSizeBucket groups a delivered payload length. The boundaries are the
+// ones that matter to the frame parser: 0 and <9 produce no frame at all, and
+// 9-16 is a bare frame header with little or no payload attached — the shape a
+// reader doing io.ReadFull(header[:9]) then a separate payload read produces.
+func payloadSizeBucket(n int) string {
+	switch {
+	case n == 0:
+		return "0"
+	case n < 9:
+		return "1-8"
+	case n < 17:
+		return "9-16"
+	case n < 257:
+		return "17-256"
+	case n < 4096:
+		return "257-4095"
+	}
+	return "4096+"
+}
+
 // frameDirection labels an HTTP/2 event by which side's frames it carries.
 // Other protocols report "-" rather than inventing a direction for them.
 func frameDirection(m l7.Method) string {
@@ -1081,6 +1101,12 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6
 		if r.PayloadSize > uint64(len(r.Payload)) {
 			L7PayloadTruncatedTotal.WithLabelValues(proto, destClass).Inc()
 		}
+		// Scoped to HTTP/2: this exists to explain why external HTTP/2 events
+		// so rarely yield a parseable frame, and keeps label cardinality small.
+		if r.Protocol == l7.ProtocolHTTP2 {
+			Http2PayloadSizeTotal.WithLabelValues(
+				payloadSizeBucket(len(r.Payload)), destClass, frameDirection(r.Method)).Inc()
+		}
 	}
 
 	// Check if eBPF traces are disabled (upstream feature)
@@ -1251,6 +1277,27 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6
 		}
 		conn.http2Parser = parser // Keep reference on connection for compatibility
 		requests := parser.Parse(r.Method, r.Payload, uint64(r.Duration), r.PayloadSize > uint64(len(r.Payload)))
+
+		// HTTP/2 has the weakest detection heuristic of any protocol here — it
+		// accepts arbitrary binary as a frame roughly once every 9k buffers —
+		// and eBPF caches the verdict for the connection's lifetime, so one
+		// false positive turns every later event on that connection into
+		// garbage. trackParseFail already exists for exactly this ("eBPF
+		// protocol misidentification where weak heuristics tag a connection
+		// permanently") and is wired for Postgres, ClickHouse and Zookeeper,
+		// but never was for HTTP/2. A connection yielding no structurally valid
+		// frame is either mistagged or unrecoverable; either way, further
+		// parsing only produces HPACK noise.
+		//
+		// Empty payloads are normal and are not counted as failures.
+		if len(r.Payload) > 0 {
+			if parser.SawValidFrame() {
+				conn.parseFailCount = 0
+			} else {
+				c.trackParseFail(conn, pid, fd, r.Protocol)
+			}
+		}
+
 		activeCount := parser.ActiveRequestCount()
 		if activeCount > 0 {
 			klog.V(3).Infof("HTTP2_PARSE_RESULT: pid=%d fd=%d completed=%d active=%d",

containers/llm_metrics.go

@@ -212,6 +212,53 @@ var (
 		[]string{"stage", "destination"},
 	)
 
+	// Http2FramesTotal counts HTTP/2 frame headers the parser walks, by type.
+	//
+	// External HTTP/2 delivers ~44k client-frame events per 5 minutes but only
+	// ~71 streams, against ~161k events and ~10.8k streams internally — 86x
+	// worse. Either those events contain almost no HEADERS frames, or they are
+	// not HTTP/2 at all. Frame type distinguishes the two directly: "invalid"
+	// dominating means the bytes are not HTTP/2 and the eBPF port heuristic is
+	// over-matching; DATA/WINDOW_UPDATE dominating with no HEADERS means the
+	// request headers are being lost before the parser sees them.
+	//
+	// Deliberately structural. These events carry decrypted application
+	// traffic, so dumping payloads to diagnose this would put Authorization
+	// headers and request bodies into agent logs; frame type, and the counts
+	// alone, disclose nothing.
+	Http2FramesTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "node_agent_http2_frames_total",
+			Help: "HTTP/2 frame headers parsed, by frame type and destination class",
+		},
+		[]string{"type", "destination"},
+	)
+
+	// Http2PayloadSizeTotal buckets the delivered payload length of HTTP/2
+	// events, by destination class and frame direction.
+	//
+	// 96% of external HTTP/2 events yield no parseable frame (8,881 frames from
+	// ~221k events) against 44% internally. Parse() can only produce nothing for
+	// three reasons: an empty payload, fewer than 9 bytes (shorter than a frame
+	// header), or a first frame header that fails validation — and the third is
+	// already counted as type="invalid" in Http2FramesTotal. So the answer is in
+	// the size distribution.
+	//
+	// The "9-16" bucket is the one to watch. A correct HTTP/2 reader does
+	// io.ReadFull(header[:9]) and then reads the frame payload separately, so
+	// SSL_read returns header-sized and payload-only chunks rather than whole
+	// frames. The parser assumes each event begins on a frame boundary and
+	// contains complete frames; if external reads are predominantly 9 bytes,
+	// that assumption is the bug and the parser needs to treat the connection as
+	// a continuous byte stream instead.
+	Http2PayloadSizeTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "node_agent_http2_payload_size_total",
+			Help: "HTTP/2 event payload sizes delivered to the parser, bucketed",
+		},
+		[]string{"bucket", "destination", "direction"},
+	)
+
 	// ContainerLLMCachedTokensTotal counts input tokens served from the
 	// provider's prompt cache. Already counted in token_usage_total{type=input};
 	// this is a separate metric to make cache-hit rate computable.
@@ -281,13 +328,45 @@ func RegisterLLMMetrics(reg prometheus.Registerer) {
 		Http2ParserCapDropsTotal,
 		Http2ParserStaleReuseTotal,
 		Http2StageTotal,
+		Http2FramesTotal,
+		Http2PayloadSizeTotal,
 		ContainerLLMCachedTokensTotal,
 		ContainerLLMToolCallsTotal,
 		ContainerLLMCostUSDTotal,
 	)
 	// Hook the HTTP/2 parser's HPACK error path so we get a counter without
 	// l7 having to import prometheus.
 	l7.OnHPACKDecodeError = func() { LLMHPACKDecodeErrorsTotal.Inc() }
+	// Pre-resolve the frame counters. OnHttp2Frame fires per frame — measured
+	// around 1.6k/s — and WithLabelValues hashes the labels and takes the
+	// vector's read lock on every call. The label sets are small and fixed, so
+	// resolving them once at startup keeps that off the parser's hot path.
+	frameTypes := []string{
+		"DATA", "HEADERS", "PRIORITY", "RST_STREAM", "SETTINGS", "PUSH_PROMISE",
+		"PING", "GOAWAY", "WINDOW_UPDATE", "CONTINUATION", "extension", "invalid",
+	}
+	dests := []string{"external", "internal", "unknown"}
+	frameCounters := make(map[string]map[string]prometheus.Counter, len(frameTypes))
+	for _, ft := range frameTypes {
+		byDest := make(map[string]prometheus.Counter, len(dests))
+		for _, d := range dests {
+			byDest[d] = Http2FramesTotal.WithLabelValues(ft, d)
+		}
+		frameCounters[ft] = byDest
+	}
+	l7.OnHttp2Frame = func(frameType, dest string) {
+		if dest == "" {
+			dest = "unknown"
+		}
+		if byDest := frameCounters[frameType]; byDest != nil {
+			if c := byDest[dest]; c != nil {
+				c.Inc()
+				return
+			}
+		}
+		// Unrecognised combination: fall back rather than drop the observation.
+		Http2FramesTotal.WithLabelValues(frameType, dest).Inc()
+	}
 	l7.OnHttp2Stage = func(stage, dest string) {
 		if dest == "" {
 			dest = "unknown"

ebpftracer/ebpf/l7/l7.c

@@ -241,6 +241,35 @@ __u64 read_iovec(char *iovec, __u64 iovlen, __u64 ret, char *buf, __u64 *total_s
     return size;
 }
 
+// http2_detection_window bounds where the weak frame-shape heuristic may run.
+//
+// looks_like_http2_frame accepts arbitrary binary data as HTTP/2 roughly once
+// every 9k buffers: it requires only frame_type <= 9, a clear reserved bit, a
+// HEADERS type byte and one HPACK byte with static index 1-14. That rate is
+// survivable on its own, but conn->protocol is cached for the life of the
+// connection, so a single false positive converts every later event on that
+// connection into an HTTP/2 event permanently.
+//
+// A large binary transfer over HTTPS/1.1 (image layers, S3 objects) performs
+// tens of thousands of reads, making a false positive near-certain, which is
+// how HTTP/1.1 connections end up feeding garbage to the HPACK decoder.
+//
+// Real HTTP/2 announces itself immediately — client preface, then SETTINGS on
+// stream 0 — so the heuristic only needs to run early. Past this many bytes a
+// connection that has not already been identified is left alone. Connections
+// joined mid-stream are lost either way: their HPACK dynamic table state is
+// unrecoverable, so today they are "detected" only to produce undecodable
+// garbage.
+#define HTTP2_DETECTION_WINDOW_BYTES 65536
+
+static inline __attribute__((__always_inline__))
+int http2_detection_allowed(struct connection *conn) {
+    if (!conn) {
+        return 0;
+    }
+    return (conn->bytes_sent + conn->bytes_received) < HTTP2_DETECTION_WINDOW_BYTES;
+}
+
 static inline __attribute__((__always_inline__))
 int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size, __u64 iovlen) {
     __u64 id = bpf_get_current_pid_tgid();
@@ -365,7 +394,7 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
         // Port-based HTTP/2 hint: Try HTTP/2 detection first for HTTPS traffic (port 443/8443)
         // Most modern HTTPS traffic uses HTTP/2, and this helps detect gRPC DATA frames
         // that don't have the connection preface
-        if (conn->dport != 53 && is_likely_http2_port(conn->dport) && looks_like_http2_frame(payload, size, METHOD_HTTP2_CLIENT_FRAMES)) {
+        if (conn->dport != 53 && http2_detection_allowed(conn) && is_likely_http2_port(conn->dport) && looks_like_http2_frame(payload, size, METHOD_HTTP2_CLIENT_FRAMES)) {
             conn->protocol = PROTOCOL_HTTP2; // Cache for subsequent frames
             struct l7_event *e = reserve_l7_event();
             if (!e) { return 0; }
@@ -408,7 +437,7 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
             req->protocol = PROTOCOL_CASSANDRA;
         } else if (is_dns_request(payload, size, &k.stream_id)) {
             req->protocol = PROTOCOL_DNS;
-        } else if (looks_like_http2_frame(payload, size, METHOD_HTTP2_CLIENT_FRAMES)) {
+        } else if (http2_detection_allowed(conn) && looks_like_http2_frame(payload, size, METHOD_HTTP2_CLIENT_FRAMES)) {
             // HTTP/2 detected on non-standard port
             conn->protocol = PROTOCOL_HTTP2; // Cache for subsequent frames
             struct l7_event *e = reserve_l7_event();
@@ -614,7 +643,7 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
                 return 0;
             }
             response = 1;
-        } else if (looks_like_http2_frame(payload, ret, METHOD_HTTP2_SERVER_FRAMES)) {
+        } else if (http2_detection_allowed(conn) && looks_like_http2_frame(payload, ret, METHOD_HTTP2_SERVER_FRAMES)) {
             // HTTP/2 detected - cache protocol for subsequent frames
             conn->protocol = PROTOCOL_HTTP2;
             e->protocol = PROTOCOL_HTTP2;

ebpftracer/l7/http2.go

@@ -29,6 +29,50 @@ var OnHPACKDecodeError func()
 // the axis the failure splits on.
 var OnHttp2Stage func(stage, dest string)
 
+// OnHttp2Frame, if set, is invoked for each frame header the parser walks, and
+// once with "invalid" when a header fails the type/length sanity check.
+//
+// This answers "are these bytes actually HTTP/2, and do they contain HEADERS?"
+// without logging any payload. That distinction matters: these events carry
+// decrypted application traffic, so a raw dump would put Authorization headers
+// and request bodies into agent logs. Frame type, flags and length are
+// structural metadata and disclose nothing.
+var OnHttp2Frame func(frameType, dest string)
+
+// http2FrameTypeName keeps the metric label bounded to the ten defined frame
+// types plus "invalid"; h.Type is already range-checked by the caller.
+func http2FrameTypeName(t http2.FrameType) string {
+	switch t {
+	case http2.FrameData:
+		return "DATA"
+	case http2.FrameHeaders:
+		return "HEADERS"
+	case http2.FramePriority:
+		return "PRIORITY"
+	case http2.FrameRSTStream:
+		return "RST_STREAM"
+	case http2.FrameSettings:
+		return "SETTINGS"
+	case http2.FramePushPromise:
+		return "PUSH_PROMISE"
+	case http2.FramePing:
+		return "PING"
+	case http2.FrameGoAway:
+		return "GOAWAY"
+	case http2.FrameWindowUpdate:
+		return "WINDOW_UPDATE"
+	case http2.FrameContinuation:
+		return "CONTINUATION"
+	}
+	return "invalid"
+}
+
+func (p *Http2Parser) frame(name string) {
+	if OnHttp2Frame != nil {
+		OnHttp2Frame(name, p.DestClass)
+	}
+}
+
 func (p *Http2Parser) stage(name string) {
 	if OnHttp2Stage != nil {
 		OnHttp2Stage(name, p.DestClass)
@@ -127,6 +171,12 @@ type Http2Parser struct {
 	// DestClass labels stage counters ("external"/"internal"); set by the caller.
 	DestClass string
 
+	// sawValidFrame reports whether the most recent Parse call decoded at least
+	// one structurally valid frame header. Callers use it to detect connections
+	// the eBPF heuristic mistagged as HTTP/2: those yield nothing but invalid
+	// frames, indefinitely, because the protocol is cached per connection.
+	sawValidFrame bool
+
 	clientDecoder  *hpack.Decoder
 	serverDecoder  *hpack.Decoder
 	activeRequests map[uint32]*Http2Request
@@ -188,6 +238,12 @@ func (p *Http2Parser) resetDecoder(method Method) {
 	}
 }
 
+// SawValidFrame reports whether the last Parse call decoded at least one
+// structurally valid frame header.
+func (p *Http2Parser) SawValidFrame() bool {
+	return p.sawValidFrame
+}
+
 // ActiveRequestCount returns the number of HTTP/2 requests currently being tracked
 // (waiting for response completion)
 func (p *Http2Parser) ActiveRequestCount() int {
@@ -425,6 +481,7 @@ func (p *Http2Parser) Parse(method Method, payload []byte, kernelTime uint64, tr
 			payload = payload[l:]
 		}
 	}
+	p.sawValidFrame = false
 	if len(payload) == 0 {
 		return nil
 	}
@@ -485,10 +542,45 @@ frameLoop:
 
 		// Sanity check: HTTP/2 max frame size is 16MB (2^24-1), and frame types are 0-9
 		// If we see clearly invalid values, this isn't valid HTTP/2 - skip remaining data
-		if h.Length > 16*1024*1024 || h.Type > 9 {
-			// Invalid frame - don't save as partial, just discard
+		if h.Length > 16*1024*1024 {
+			// Length beyond the 16MB maximum: this is not a frame header.
+			// Consume the rest: leaving offset at frameStart would let the
+			// partial-frame save at the end of this function buffer the garbage
+			// and prepend it to every subsequent call, re-parsing it forever.
+			p.frame("invalid")
+			offset = len(payload)
+			break
+		}
+		// RFC 9113 4.1: an unknown frame type MUST be ignored and discarded,
+		// not treated as an error. ALTSVC (0x0a), ORIGIN (0x0c) and
+		// PRIORITY_UPDATE (0x10) are standard extensions that GitHub and Google
+		// both send. Breaking here would drop the rest of the payload, and
+		// because callers reclassify connections that yield no valid frame, a
+		// connection whose payload merely leads with an extension frame could be
+		// dropped as if it were misdetected. Skip it and keep parsing.
+		// Registered types run to 0x10 (PRIORITY_UPDATE). Anything beyond that
+		// is not a plausible extension, and treating it as one would let
+		// misdetected binary traffic masquerade as valid HTTP/2 forever.
+		if h.Type > 9 && h.Type <= 0x10 {
+			p.frame("extension")
+			p.sawValidFrame = true
+			// offset still points at the frame header here; skip header+payload.
+			if len(payload)-offset < http2FrameHeaderLength+h.Length {
+				offset = frameStart
+				break frameLoop
+			}
+			offset += http2FrameHeaderLength + h.Length
+			continue
+		}
+		if h.Type > 0x10 {
+			// No registered frame type above 0x10; consume the rest for the
+			// same reason as the oversized-length case above.
+			p.frame("invalid")
+			offset = len(payload)
 			break
 		}
+		p.frame(http2FrameTypeName(h.Type))
+		p.sawValidFrame = true
 
 		offset += http2FrameHeaderLength