From 75ae50607e36a5b2c1b6bee107606fab83377eeb Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Fri, 4 Sep 2026 18:56:30 +0530 Subject: [PATCH] feat(metrics): split L7 event counter by HTTP/2 frame direction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pipeline counters from #317 localised the failure but not its cause. External HTTP/2, 10 minutes on dev: L7 events 1,155,643 internal control: 844,250 stream_created 53 8,052 response_status 0 5,478 completed 0 5,158 That is ~22,000 external events per stream created, against ~105 internally — a 200x difference — and response_status is exactly zero. Zero matters because :status decodes from the HPACK static table (index 8), which survives a degraded decoder; the internal control produces 5,478 of them on the same code path. Server-side HEADERS are therefore never decoded externally at all. Also ruled out by that run, so they need no further attention: parser cap drops were 0, and stale fd reuse was 3 events externally in 10 minutes. Two explanations remain for the ratio, and direction separates them cleanly: server-frame events scarce -> responses never reach the parser server-frame events plentiful -> the bytes are not HTTP/2 and the eBPF port-based detection (is_likely_http2_port, 443/8443, plus a frame-shape check) is over-matching TLS traffic The second would also recast node_agent_hpack_decode_errors_total as a symptom of feeding non-HTTP/2 bytes to an HPACK decoder rather than a cause. Adds a direction label ("client"/"server", "-" where inapplicable) to node_agent_l7_events_total. Measurement only. --- containers/container.go | 14 +++++++++++++- containers/llm_metrics.go | 12 ++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/containers/container.go b/containers/container.go index 41738ebb..9de5a03d 100644 --- a/containers/container.go +++ b/containers/container.go @@ -902,6 +902,18 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R return ip2fqdn } +// frameDirection labels an HTTP/2 event by which side's frames it carries. +// Other protocols report "-" rather than inventing a direction for them. +func frameDirection(m l7.Method) string { + switch m { + case l7.MethodHttp2ClientFrames: + return "client" + case l7.MethodHttp2ServerFrames: + return "server" + } + return "-" +} + // protocolLabel renders a protocol for use as a metric label. Protocol.String() // falls back to "UNKNOWN:" for unrecognised values, which would be unbounded // cardinality on a label, so those collapse to a single bucket. @@ -1065,7 +1077,7 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6 destClass = "external" } proto := protocolLabel(r.Protocol) - L7EventsTotal.WithLabelValues(proto, destClass).Inc() + L7EventsTotal.WithLabelValues(proto, destClass, frameDirection(r.Method)).Inc() if r.PayloadSize > uint64(len(r.Payload)) { L7PayloadTruncatedTotal.WithLabelValues(proto, destClass).Inc() } diff --git a/containers/llm_metrics.go b/containers/llm_metrics.go index 52e2aab7..e1a7c5ef 100644 --- a/containers/llm_metrics.go +++ b/containers/llm_metrics.go @@ -136,12 +136,20 @@ var ( // rate(node_agent_l7_payload_truncated_total{protocol="http2",destination="external"}[5m]) // against the same labels on node_agent_l7_events_total to see what share of // external HTTP/2 traffic is arriving incomplete. + // direction is "client"/"server" for HTTP/2 (which frames the event carries) + // and "-" for protocols where the distinction does not apply. + // + // External HTTP/2 delivers ~22,000 events per stream created, against ~105 + // internally. Splitting by direction separates the two explanations for + // that: if server-frame events are scarce, responses never reach the parser; + // if they are plentiful, the bytes being fed to it are not HTTP/2 at all and + // the port-based detection heuristic is over-matching. L7EventsTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "node_agent_l7_events_total", - Help: "L7 events processed, by protocol and destination class", + Help: "L7 events processed, by protocol, destination class and frame direction", }, - []string{"protocol", "destination"}, + []string{"protocol", "destination", "direction"}, ) L7PayloadTruncatedTotal = prometheus.NewCounterVec(