From 707638e82b733c53cc13ae7ac2974666f656cf59 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Sun, 6 Sep 2026 12:38:55 +0530 Subject: [PATCH 1/2] fix(metrics): stop reclassified connections from polluting HTTP metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A connection the parser gave up on kept being reported as an HTTP request for the rest of its life. trackParseFail sets conn.protocolOverride = protocolReclassified (0xFF) once a connection has failed to parse parseFailThreshold times in a row, which is the agent concluding eBPF misidentified it. 0xFF matches none of the 14 protocol cases, so every later event fell into default:, which called observe(r.Protocol, "unknown", "", "", 0, ...) passing r.Protocol — the original, wrong protocol — rather than the override. observe() maps HTTP2 to HTTP, so a Postgres connection misdetected as HTTP/2 incremented container_http_requests_total on every subsequent event, with a literal "unknown" status and zero duration. Measured on dev before the change: 1.24M external "HTTP requests" per hour, 99.7% of them status="unknown", with 97% originating from three private IPs that are actually Postgres and an egress proxy. Genuine external HTTP traffic was outnumbered roughly 400:1 — GitHub's 2,951 requests against 1.2M of noise — which is why status codes looked unusable in aggregate. Reclassified connections now emit nothing. Giving up on a connection has to mean giving up on reporting it. Genuinely unhandled protocols still reach default: and are counted under their own protocol, which is correct there because it is not the sentinel. This also explains why traces were unaffected (1 bad row in 31,900 spans while metrics were 99.7% wrong): default: only calls observe() and never creates a span, so the pollution was confined to the metrics pipeline. No test: the containers package is excluded from CI and cannot be built on macOS. Verification is the deployed metric — container_http_requests_total {status="unknown"} should collapse from 1.24M/hr to near zero. --- containers/container.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/containers/container.go b/containers/container.go index a8b8d960..db9acb48 100644 --- a/containers/container.go +++ b/containers/container.go @@ -1476,8 +1476,26 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6 case l7.ProtocolFoundationDB: // Update stats for FoundationDB c.l7Stats.observe(r.Protocol, r.Status.String(), "", "", r.Duration, conn.DestinationKey, conn.srcWorkload, r, "") + case protocolReclassified: + // Deliberately emits nothing. Reaching here means the parser refused this + // connection's payload repeatedly and we concluded eBPF misidentified it, + // so we no longer know what protocol it is. + // + // The default branch below used to catch this, and passed r.Protocol — + // the original, wrong protocol — with a literal "unknown" status and zero + // duration. observe() maps HTTP2 to HTTP, so a Postgres connection + // misdetected as HTTP/2 kept incrementing container_http_requests_total + // on every subsequent event for the life of the connection. Measured on + // dev: 1.24M external "HTTP requests" per hour, 99.7% of them + // status="unknown", 97% originating from three private IPs that are + // actually Postgres and an egress proxy. Real external HTTP traffic was + // buried roughly 400:1. + // + // Giving up on a connection has to mean giving up on reporting it. default: - // For all other protocols, update stats + // Genuinely unhandled protocols: count them under their own protocol. + // r.Protocol is correct here precisely because this is not the + // reclassification sentinel. c.l7Stats.observe(r.Protocol, "unknown", "", "", 0, conn.DestinationKey, conn.srcWorkload, r, "") } return nil, L7RequestProcessed From 6e0b25ea495e78915748b8a731df471f4c772d13 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Sun, 6 Sep 2026 12:42:46 +0530 Subject: [PATCH 2/2] fix(memory): release parsers when a connection is reclassified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review flagged that reclassification stops parsing but does not free the parsers. Confirmed: googleHTTP2Parsers entries are only deleted once the connection is gone or the pid dies, and conn.http2Parser / postgresParser / mysqlParser live as long as the connection does. That is dead weight from the moment we reclassify — the connection will never be parsed again, yet the HTTP/2 parser keeps its HPACK decoders and partial-frame buffers, and the SQL parsers keep their prepared-statement state. It also lands on exactly the wrong connections: the ones reaching reclassification are the long-lived, high-volume Postgres and egress-proxy connections, so 'until the connection closes' can be hours. Predates this branch — the switch already skipped parsing for reclassified connections — but reclassification is the correct point to release, and this PR is what makes it unambiguous that nothing will use them again. --- containers/container.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/containers/container.go b/containers/container.go index db9acb48..23bc571f 100644 --- a/containers/container.go +++ b/containers/container.go @@ -1515,6 +1515,16 @@ func (c *Container) trackParseFail(conn *ActiveConnection, pid uint32, fd uint64 conn.parseFailCount++ if conn.parseFailCount == parseFailThreshold { conn.protocolOverride = protocolReclassified + // Release the parsers now rather than at connection close. Reclassifying + // means this connection will never be parsed again, so anything they hold + // — HPACK decoders, partial frame buffers, prepared-statement maps — is + // dead weight. gc() only reclaims parsers once the connection is gone, + // and the connections that get here are the long-lived ones (Postgres, + // egress proxies), so that could be hours. + delete(c.googleHTTP2Parsers, PidFd{Pid: pid, Fd: fd}) + conn.http2Parser = nil + conn.postgresParser = nil + conn.mysqlParser = nil klog.Warningf("reclassified connection pid=%d fd=%d from %s to unknown after %d consecutive parse failures", pid, fd, proto, conn.parseFailCount) }