Carry jumbo frames as UMEM chains instead of oversized frames

internal/dataplane/preflight.go

@@ -192,6 +192,9 @@ type PreflightInput struct {
 	MaxFrameLen int
 	NumFrames   int
 	FrameSize   int
+	// MultiBuffer is set when a packet is carried as a chain of UMEM frames
+	// rather than having to fit in one.
+	MultiBuffer bool
 	// AllLinks is every interface on the host, used to place the SSH session.
 	AllLinks []discovery.Link
 }
@@ -235,11 +238,28 @@ func RunPreflight(in PreflightInput) *Preflight {
 	}
 
 	// Frame size against the interface MTU and the UMEM frame capacity.
-	if in.MaxFrameLen > in.FrameSize {
+	//
+	// A UMEM frame cannot be grown to fit a jumbo packet: the kernel caps an
+	// aligned chunk at a page. Packets larger than a frame are chained across
+	// several instead, which is what multi-buffer mode is for. So the real
+	// ceiling is how many frames one packet may span.
+	switch frames := framesPerPacket(in.MaxFrameLen, in.FrameSize); {
+	case frames > 1 && !in.MultiBuffer:
 		add(LevelFatal, "packets are larger than an AF_XDP frame",
-			fmt.Sprintf("the largest packet is %d bytes but each UMEM frame holds %d.",
-				in.MaxFrameLen, in.FrameSize),
+			fmt.Sprintf("the largest packet is %d bytes but each UMEM frame holds %d, and this "+
+				"run is not chaining frames.", in.MaxFrameLen, in.FrameSize),
 			"Use a smaller --packet-size, or a capture with smaller packets.")
+	case frames > maxTxSegs:
+		add(LevelFatal, "packets are too large to chain",
+			fmt.Sprintf("the largest packet is %d bytes, which needs %d frames of %d, more than "+
+				"the %d a single packet may span.", in.MaxFrameLen, frames, in.FrameSize, maxTxSegs),
+			"Use a smaller --packet-size, or a capture with smaller packets.")
+	case frames > 1:
+		add(LevelInfo, "jumbo frames are chained across buffers",
+			fmt.Sprintf("the largest packet is %d bytes and each UMEM frame holds %d, so a packet "+
+				"spans up to %d frames.", in.MaxFrameLen, in.FrameSize, frames),
+			"Chaining needs the driver to accept an XDP_USE_SG bind. Where it will not do that in "+
+				"zero-copy mode the run falls back to copy mode, which the startup line reports.")
 	}
 	// The MTU bounds what may be *sent*. A run that transmits nothing is not
 	// constrained by it — it just has to have frames big enough to receive

internal/dataplane/preflight_test.go

@@ -148,6 +148,45 @@ func TestFrameSizeAndMTUChecks(t *testing.T) {
 		t.Error("the frame-capacity check did not fire")
 	}
 
+	// ...unless the frames are being chained, which is the whole point of
+	// multi-buffer mode. Then it is worth mentioning, not worth refusing.
+	p = RunPreflight(input(func(in *PreflightInput) {
+		in.Res.Link.MTU = 9200 // a jumbo packet needs a jumbo link
+		in.MaxFrameLen = 9014
+		in.FrameSize = 4096
+		in.MultiBuffer = true
+	}))
+	if !p.OK() {
+		t.Fatalf("a chained jumbo packet must be allowed: %v", p.Err())
+	}
+	if find(p, "larger than an AF_XDP frame") != nil {
+		t.Error("the frame-capacity check must not fire when frames are chained")
+	}
+	c := find(p, "chained across buffers")
+	if c == nil {
+		t.Fatal("chaining should be reported")
+	}
+	if c.Level != LevelInfo {
+		t.Errorf("chaining is informational, got level %v", c.Level)
+	}
+	if !strings.Contains(c.Detail, "3 frames") {
+		t.Errorf("the detail should say how many frames a packet spans: %s", c.Detail)
+	}
+
+	// There is still a ceiling: a packet may only span so many frames.
+	p = RunPreflight(input(func(in *PreflightInput) {
+		in.Res.Link.MTU = 1 << 20 // isolate the chain limit from the MTU check
+		in.MaxFrameLen = (maxTxSegs + 1) * 4096
+		in.FrameSize = 4096
+		in.MultiBuffer = true
+	}))
+	if p.OK() {
+		t.Fatal("a packet needing more frames than one may span must be refused")
+	}
+	if find(p, "too large to chain") == nil {
+		t.Error("the chain-length check did not fire")
+	}
+
 	// A packet larger than the MTU is refused, with the interface's own number
 	// in the message.
 	p = RunPreflight(input(func(in *PreflightInput) {
@@ -157,7 +196,7 @@ func TestFrameSizeAndMTUChecks(t *testing.T) {
 	if p.OK() {
 		t.Fatal("a packet larger than the MTU must be refused")
 	}
-	c := find(p, "MTU")
+	c = find(p, "MTU")
 	if c == nil {
 		t.Fatal("the MTU check did not fire")
 	}

internal/dataplane/runner.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"os"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -25,6 +26,14 @@ const (
 	// small enough that a rate change or a stop is noticed promptly.
 	txBatch = 256
 
+	// jumboTxBatch is txBatch for the multi-buffer path, which stages packets
+	// in a buffer of batch x maxFrame bytes per queue. At a 9018-byte MTU the
+	// full txBatch would be 2.3 MiB a queue, so use a smaller batch: jumbo
+	// packet rates are two orders of magnitude below minimum-size rates, and
+	// even one queue saturating 100G only needs a few tens of thousands of
+	// batches a second.
+	jumboTxBatch = 32
+
 	// rxPollTimeout bounds a blocking receive so cancellation is seen quickly.
 	rxPollTimeout = 200 * time.Millisecond
 
@@ -52,6 +61,10 @@ type Info struct {
 	Filter    string
 	FrameSize int
 	NumFrames int
+	// MultiBuffer is true when packets are chained across several UMEM frames,
+	// which is how jumbo frames are carried. Worth showing: it is also why a
+	// jumbo run may report copy rather than zero-copy.
+	MultiBuffer bool
 	// Tuning is what the library did to the interface's NAPI settings to make
 	// the receive path keep up, e.g. "defer=2 flush=200ms", or "untuned". These
 	// are host settings the library restores on close; worth showing.
@@ -77,6 +90,9 @@ func (i Info) String() string {
 		zc = "zero-copy"
 	}
 	s := fmt.Sprintf("%s: %d queue(s), %s, %s XDP", i.Interface, i.Queues, zc, i.XDPMode)
+	if i.MultiBuffer {
+		s += ", multi-buffer"
+	}
 	if i.Driver != "" {
 		s += ", driver " + i.Driver
 	}
@@ -110,6 +126,9 @@ type Runner struct {
 	numFrames int
 	frameSize int
 	maxFrame  int
+	// multiBuffer is set when the largest packet does not fit one UMEM frame
+	// and has to be chained across several. See multiBufferFor.
+	multiBuffer bool
 
 	// frames is the loaded PCAP, nil for generated traffic.
 	frames generator.FrameSource
@@ -209,6 +228,7 @@ func New(cfg *config.Config, res *discovery.Resolved, opts Options) (*Runner, er
 		r.maxFrame = mtu + 18
 	}
 	r.frameSize = orDefault(opts.FrameSize, frameSizeFor(r.maxFrame, res.Link.Driver))
+	r.multiBuffer = multiBufferFor(r.maxFrame, r.frameSize)
 
 	r.limiter = rate.New(cfg.PPS, cfg.BPS, rate.WithBatch(txBatch))
 	statsOpts := []stats.Option{}
@@ -225,6 +245,7 @@ func New(cfg *config.Config, res *discovery.Resolved, opts Options) (*Runner, er
 		NumFrames:   r.numFrames,
 		Pattern:     string(cfg.Mode),
 		PacketSizes: sizes,
+		MultiBuffer: r.multiBuffer,
 	}
 	return r, nil
 }
@@ -236,30 +257,61 @@ func orDefault(v, def int) int {
 	return def
 }
 
-// frameSizeFor picks a UMEM frame size that holds the largest packet. 2048 is
-// the library default and covers everything up to a standard Ethernet frame;
-// jumbo traffic needs page-sized frames, which are also what zero-copy wants
-// on drivers that require them.
+// frameSizeFor picks a UMEM frame size. 2048 is the library default and covers
+// everything up to a standard Ethernet frame; anything larger gets a full page.
+//
+// A page is the ceiling, not a preference. An aligned-chunk UMEM must satisfy
+// XDP_UMEM_MIN_CHUNK_SIZE (2048) <= chunk_size <= PAGE_SIZE, and go-afxdp never
+// sets XDP_UMEM_UNALIGNED_CHUNK_FLAG, so asking for more makes XDP_UMEM_REG
+// fail with a bare EINVAL. Frames therefore do not grow to fit a jumbo packet.
+// Packets bigger than one frame span several, which is what multiBufferFor
+// below decides.
 //
 // AWS ENA's zero-copy datapath needs page-sized (4096) frames; with the default
 // 2048 the bind silently falls back to native copy. So floor at 4096 on ena,
 // which keeps our own UMEM accounting (memlock preflight, banner) consistent
 // with what the driver actually binds. Scoped to ena; other drivers keep the
 // smaller, more cache-friendly frames.
 func frameSizeFor(maxFrame int, driver string) int {
-	size := 32768
-	for _, n := range []int{2048, 4096, 8192, 16384} {
-		if maxFrame <= n {
-			size = n
-			break
-		}
-	}
-	if driver == "ena" && size < 4096 {
+	size := 2048
+	if maxFrame > size || driver == "ena" {
 		size = 4096
 	}
+	// Every Linux architecture pages at 4 KiB or more, so this never bites in
+	// practice. It is here so the kernel's ceiling is expressed in the code
+	// rather than assumed.
+	if page := os.Getpagesize(); size > page {
+		size = page
+	}
 	return size
 }
 
+// multiBufferFor reports whether packets have to span several UMEM frames.
+//
+// This is the jumbo path: the socket binds with XDP_USE_SG and the XDP program
+// loads with BPF_F_XDP_HAS_FRAGS, so a packet arrives as a chain of descriptors
+// instead of being dropped for not fitting. It costs zero-copy on any device
+// reporting xdp-zc-max-segs = 1, and the transmit side has to build through a
+// staging buffer, so it stays off unless the traffic actually needs it.
+func multiBufferFor(maxFrame, frameSize int) bool {
+	return maxFrame > frameSize
+}
+
+// maxTxSegs is how many UMEM frames one transmitted packet may span. It mirrors
+// the limit go-afxdp enforces, which in turn comes from the kernel building an
+// skb of at most CONFIG_MAX_SKB_FRAGS + 1 buffers. Kept here so preflight can
+// reject an impossible size with an explanation instead of letting the transmit
+// loop fail on every batch.
+const maxTxSegs = 18
+
+// framesPerPacket is how many UMEM frames the largest packet occupies.
+func framesPerPacket(maxFrame, frameSize int) int {
+	if maxFrame <= 0 || frameSize <= 0 {
+		return 1
+	}
+	return (maxFrame + frameSize - 1) / frameSize
+}
+
 // buildGenerators makes one generator per queue.
 func (r *Runner) buildGenerators() ([]generator.Generator, error) {
 	var srcMAC, dstMAC [6]byte
@@ -294,6 +346,7 @@ func (r *Runner) Preflight(src discovery.Source) *Preflight {
 		MaxFrameLen: r.maxFrame,
 		NumFrames:   r.numFrames,
 		FrameSize:   r.frameSize,
+		MultiBuffer: r.multiBuffer,
 		AllLinks:    links,
 	})
 }
@@ -499,12 +552,13 @@ func (r *Runner) Run(ctx context.Context) error {
 func (r *Runner) fleetKey() fleetKey {
 	o := r.umemOptions()
 	return fleetKey{
-		iface:     r.res.Link.Name,
-		queues:    r.queues,
-		filter:    r.plan.Summary,
-		numFrames: o.NumFrames,
-		frameSize: o.FrameSize,
-		receives:  r.plan.Receives(),
+		iface:       r.res.Link.Name,
+		queues:      r.queues,
+		filter:      r.plan.Summary,
+		numFrames:   o.NumFrames,
+		frameSize:   o.FrameSize,
+		receives:    r.plan.Receives(),
+		multiBuffer: r.multiBuffer,
 	}
 }
 
@@ -576,6 +630,12 @@ func (r *Runner) attach() error {
 			// of parking, burning cores while forwarding nothing.
 			afxdp.WithNeedWakeup(),
 		}
+		if r.multiBuffer {
+			// Packets bigger than a frame have to chain across several. This
+			// also lets the program attach at a jumbo MTU on drivers that
+			// otherwise cap XDP to a single buffer.
+			opts = append(opts, afxdp.WithMultiBuffer())
+		}
 		if r.plan.KeepManagement {
 			// Spare ARP, ND, SSH and DNS from the match-all redirect so the box
 			// stays reachable while everything else is captured.

internal/dataplane/runner_test.go

@@ -1,27 +1,92 @@
 package dataplane
 
-import "testing"
+import (
+	"os"
+	"testing"
+)
 
-// frameSizeFor picks the UMEM frame size. The one subtlety is AWS ENA, whose
-// zero-copy bind needs page-sized (4096) frames: a standard frame must be
-// floored to 4096 there, while every other driver keeps the smaller 2048.
-func TestFrameSizeForENA(t *testing.T) {
+// frameSizeFor picks the UMEM frame size. Two things matter. A frame may never
+// exceed a page, because an aligned-chunk UMEM larger than PAGE_SIZE makes
+// XDP_UMEM_REG fail with a bare EINVAL. So jumbo traffic gets a page and
+// chains, it does not get a bigger frame. And AWS ENA's zero-copy bind needs
+// page-sized frames, so a standard frame is floored to 4096 there while every
+// other driver keeps the smaller 2048.
+func TestFrameSizeFor(t *testing.T) {
 	tests := []struct {
 		driver   string
 		maxFrame int
 		want     int
 	}{
-		{"ena", 1518, 4096},   // the fix: a standard frame is floored to a page on ena
+		{"ena", 1518, 4096},   // a standard frame is floored to a page on ena
 		{"ixgbe", 1518, 2048}, // other drivers keep the smaller frame
 		{"", 1518, 2048},      // unknown driver behaves like non-ena
 		{"ena", 3018, 4096},   // already 4096, the floor is a no-op
-		{"ena", 9018, 16384},  // jumbo is unaffected by the floor
-		{"mlx5_core", 9018, 16384},
-		{"ena", 64, 4096}, // a tiny frame still gets a page on ena
+		{"ena", 64, 4096},     // a tiny frame still gets a page on ena
+
+		// Above 2048 every driver takes a page, and stops there. These are the
+		// sizes that used to ask for 8192 or 16384 and get EINVAL.
+		{"ixgbe", 2049, 4096},
+		{"ixgbe", 4097, 4096},
+		{"ena", 9018, 4096},
+		{"mlx5_core", 9018, 4096},
+		{"mlx5_core", 16384, 4096},
 	}
 	for _, tt := range tests {
-		if got := frameSizeFor(tt.maxFrame, tt.driver); got != tt.want {
+		got := frameSizeFor(tt.maxFrame, tt.driver)
+		if got != tt.want {
 			t.Errorf("frameSizeFor(%d, %q) = %d, want %d", tt.maxFrame, tt.driver, got, tt.want)
 		}
+		if page := os.Getpagesize(); got > page {
+			t.Errorf("frameSizeFor(%d, %q) = %d, above the %d-byte page the kernel allows",
+				tt.maxFrame, tt.driver, got, page)
+		}
+	}
+}
+
+// Chaining is what carries a packet that no longer fits one frame, so it must
+// turn on exactly when that happens and stay off otherwise: it costs zero-copy
+// on drivers that will not accept an XDP_USE_SG bind.
+func TestMultiBufferFor(t *testing.T) {
+	tests := []struct {
+		maxFrame  int
+		frameSize int
+		want      bool
+	}{
+		{60, 2048, false},   // a 64-byte frame
+		{1514, 2048, false}, // a standard frame
+		{2048, 2048, false}, // exactly a frame still fits
+		{2049, 4096, false}, // ...and gets a bigger frame rather than chaining
+		{4096, 4096, false}, // exactly a page fits
+		{4097, 4096, true},  // one byte over is where chaining starts
+		{9014, 4096, true},  // a jumbo frame
+	}
+	for _, tt := range tests {
+		if got := multiBufferFor(tt.maxFrame, tt.frameSize); got != tt.want {
+			t.Errorf("multiBufferFor(%d, %d) = %v, want %v",
+				tt.maxFrame, tt.frameSize, got, tt.want)
+		}
+	}
+}
+
+// framesPerPacket feeds preflight's chain-length check, so the boundaries have
+// to be exact rather than approximately right.
+func TestFramesPerPacket(t *testing.T) {
+	tests := []struct {
+		maxFrame  int
+		frameSize int
+		want      int
+	}{
+		{0, 4096, 1},    // degenerate input never reports zero frames
+		{60, 4096, 1},   //
+		{4096, 4096, 1}, // exactly one frame
+		{4097, 4096, 2}, // one byte over needs a second
+		{9014, 4096, 3}, // a jumbo frame spans three
+		{8192, 4096, 2},
+	}
+	for _, tt := range tests {
+		if got := framesPerPacket(tt.maxFrame, tt.frameSize); got != tt.want {
+			t.Errorf("framesPerPacket(%d, %d) = %d, want %d",
+				tt.maxFrame, tt.frameSize, got, tt.want)
+		}
 	}
 }

internal/dataplane/session.go

@@ -27,6 +27,11 @@ type fleetKey struct {
 	// receives changes the frame split between the transmit and receive pools
 	// and the ring depths, so it is part of the bind too.
 	receives bool
+	// multiBuffer is a bind flag (XDP_USE_SG) and a program flag
+	// (BPF_F_XDP_HAS_FRAGS), so it cannot be changed on an attached fleet. It
+	// needs its own field because frameSize does not imply it: at 4096-byte
+	// frames a 4100-byte run chains nothing and a 9000-byte run chains.
+	multiBuffer bool
 }
 
 // Session keeps an AF_XDP fleet attached across several runs.