From bbe591d2c4512c7dfaebf3a14e108c9af653b781 Mon Sep 17 00:00:00 2001 From: Andree Toonk Date: Mon, 10 Aug 2026 19:07:04 +0000 Subject: [PATCH] Carry jumbo frames as UMEM chains instead of oversized frames Any packet over 4100 bytes failed to start with a bare "XDP_UMEM_REG: invalid argument". frameSizeFor grew the UMEM frame to fit the largest packet, picking 8192 or 16384, but an aligned-chunk UMEM must satisfy 2048 <= chunk_size <= PAGE_SIZE and go-afxdp never sets XDP_UMEM_UNALIGNED_CHUNK_FLAG, so the kernel refused it. This is not architecture-specific, and it needed no unusual configuration: EC2 ships ENA at MTU 9001, and a receive-only run sizes frames as mtu + 18, so plain "--mode receive" on a stock instance was enough to hit it. Frames now stop at a page and packets larger than one span several, which is what go-afxdp's WithMultiBuffer has provided since v0.7.0. It engages on its own when the largest frame exceeds the frame size, so normal traffic keeps the single-frame path and its zero-copy generation untouched. Receive moves to ReceivePackets so a chain counts as one packet at its full length, with the protocol read from the fragment that carries the headers. Transmit keeps SendFunc, which writes straight into the UMEM frame but cannot chain; the jumbo path stages into a preallocated arena and lets SendBatch split it. Preflight reports chaining rather than refusing it, and now bounds the real limit, how many frames one packet may span. fleetKey gains the flag too: it is fixed at bind time, and the frame size does not imply it, since at 4096-byte frames a 4100-byte run chains nothing while a 9000-byte run chains. Verified on c7gn.xlarge (aarch64, AL2023, ENA 2.17.2g, MTU 9001), where the old binary reproduced the reported failure exactly: 9000-byte frames now run at 40 Gbit/s and 399,970 of 399,970 packets arrive. 64-byte and 1518-byte rates are unchanged. Closes #1 --- internal/dataplane/preflight.go | 26 ++++- internal/dataplane/preflight_test.go | 41 +++++++- internal/dataplane/runner.go | 96 +++++++++++++++---- internal/dataplane/runner_test.go | 85 +++++++++++++++-- internal/dataplane/session.go | 5 + internal/dataplane/worker.go | 138 ++++++++++++++++++++++----- internal/pcapfile/pcapfile.go | 6 +- internal/tui/dashboard.go | 5 + 8 files changed, 345 insertions(+), 57 deletions(-) diff --git a/internal/dataplane/preflight.go b/internal/dataplane/preflight.go index 591210f..7e4a480 100644 --- a/internal/dataplane/preflight.go +++ b/internal/dataplane/preflight.go @@ -192,6 +192,9 @@ type PreflightInput struct { MaxFrameLen int NumFrames int FrameSize int + // MultiBuffer is set when a packet is carried as a chain of UMEM frames + // rather than having to fit in one. + MultiBuffer bool // AllLinks is every interface on the host, used to place the SSH session. AllLinks []discovery.Link } @@ -235,11 +238,28 @@ func RunPreflight(in PreflightInput) *Preflight { } // Frame size against the interface MTU and the UMEM frame capacity. - if in.MaxFrameLen > in.FrameSize { + // + // A UMEM frame cannot be grown to fit a jumbo packet: the kernel caps an + // aligned chunk at a page. Packets larger than a frame are chained across + // several instead, which is what multi-buffer mode is for. So the real + // ceiling is how many frames one packet may span. + switch frames := framesPerPacket(in.MaxFrameLen, in.FrameSize); { + case frames > 1 && !in.MultiBuffer: add(LevelFatal, "packets are larger than an AF_XDP frame", - fmt.Sprintf("the largest packet is %d bytes but each UMEM frame holds %d.", - in.MaxFrameLen, in.FrameSize), + fmt.Sprintf("the largest packet is %d bytes but each UMEM frame holds %d, and this "+ + "run is not chaining frames.", in.MaxFrameLen, in.FrameSize), "Use a smaller --packet-size, or a capture with smaller packets.") + case frames > maxTxSegs: + add(LevelFatal, "packets are too large to chain", + fmt.Sprintf("the largest packet is %d bytes, which needs %d frames of %d, more than "+ + "the %d a single packet may span.", in.MaxFrameLen, frames, in.FrameSize, maxTxSegs), + "Use a smaller --packet-size, or a capture with smaller packets.") + case frames > 1: + add(LevelInfo, "jumbo frames are chained across buffers", + fmt.Sprintf("the largest packet is %d bytes and each UMEM frame holds %d, so a packet "+ + "spans up to %d frames.", in.MaxFrameLen, in.FrameSize, frames), + "Chaining needs the driver to accept an XDP_USE_SG bind. Where it will not do that in "+ + "zero-copy mode the run falls back to copy mode, which the startup line reports.") } // The MTU bounds what may be *sent*. A run that transmits nothing is not // constrained by it — it just has to have frames big enough to receive diff --git a/internal/dataplane/preflight_test.go b/internal/dataplane/preflight_test.go index 88da4d8..e8560a6 100644 --- a/internal/dataplane/preflight_test.go +++ b/internal/dataplane/preflight_test.go @@ -148,6 +148,45 @@ func TestFrameSizeAndMTUChecks(t *testing.T) { t.Error("the frame-capacity check did not fire") } + // ...unless the frames are being chained, which is the whole point of + // multi-buffer mode. Then it is worth mentioning, not worth refusing. + p = RunPreflight(input(func(in *PreflightInput) { + in.Res.Link.MTU = 9200 // a jumbo packet needs a jumbo link + in.MaxFrameLen = 9014 + in.FrameSize = 4096 + in.MultiBuffer = true + })) + if !p.OK() { + t.Fatalf("a chained jumbo packet must be allowed: %v", p.Err()) + } + if find(p, "larger than an AF_XDP frame") != nil { + t.Error("the frame-capacity check must not fire when frames are chained") + } + c := find(p, "chained across buffers") + if c == nil { + t.Fatal("chaining should be reported") + } + if c.Level != LevelInfo { + t.Errorf("chaining is informational, got level %v", c.Level) + } + if !strings.Contains(c.Detail, "3 frames") { + t.Errorf("the detail should say how many frames a packet spans: %s", c.Detail) + } + + // There is still a ceiling: a packet may only span so many frames. + p = RunPreflight(input(func(in *PreflightInput) { + in.Res.Link.MTU = 1 << 20 // isolate the chain limit from the MTU check + in.MaxFrameLen = (maxTxSegs + 1) * 4096 + in.FrameSize = 4096 + in.MultiBuffer = true + })) + if p.OK() { + t.Fatal("a packet needing more frames than one may span must be refused") + } + if find(p, "too large to chain") == nil { + t.Error("the chain-length check did not fire") + } + // A packet larger than the MTU is refused, with the interface's own number // in the message. p = RunPreflight(input(func(in *PreflightInput) { @@ -157,7 +196,7 @@ func TestFrameSizeAndMTUChecks(t *testing.T) { if p.OK() { t.Fatal("a packet larger than the MTU must be refused") } - c := find(p, "MTU") + c = find(p, "MTU") if c == nil { t.Fatal("the MTU check did not fire") } diff --git a/internal/dataplane/runner.go b/internal/dataplane/runner.go index 99dac4b..9e794a5 100644 --- a/internal/dataplane/runner.go +++ b/internal/dataplane/runner.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "os" "sync" "sync/atomic" "time" @@ -25,6 +26,14 @@ const ( // small enough that a rate change or a stop is noticed promptly. txBatch = 256 + // jumboTxBatch is txBatch for the multi-buffer path, which stages packets + // in a buffer of batch x maxFrame bytes per queue. At a 9018-byte MTU the + // full txBatch would be 2.3 MiB a queue, so use a smaller batch: jumbo + // packet rates are two orders of magnitude below minimum-size rates, and + // even one queue saturating 100G only needs a few tens of thousands of + // batches a second. + jumboTxBatch = 32 + // rxPollTimeout bounds a blocking receive so cancellation is seen quickly. rxPollTimeout = 200 * time.Millisecond @@ -52,6 +61,10 @@ type Info struct { Filter string FrameSize int NumFrames int + // MultiBuffer is true when packets are chained across several UMEM frames, + // which is how jumbo frames are carried. Worth showing: it is also why a + // jumbo run may report copy rather than zero-copy. + MultiBuffer bool // Tuning is what the library did to the interface's NAPI settings to make // the receive path keep up, e.g. "defer=2 flush=200ms", or "untuned". These // are host settings the library restores on close; worth showing. @@ -77,6 +90,9 @@ func (i Info) String() string { zc = "zero-copy" } s := fmt.Sprintf("%s: %d queue(s), %s, %s XDP", i.Interface, i.Queues, zc, i.XDPMode) + if i.MultiBuffer { + s += ", multi-buffer" + } if i.Driver != "" { s += ", driver " + i.Driver } @@ -110,6 +126,9 @@ type Runner struct { numFrames int frameSize int maxFrame int + // multiBuffer is set when the largest packet does not fit one UMEM frame + // and has to be chained across several. See multiBufferFor. + multiBuffer bool // frames is the loaded PCAP, nil for generated traffic. frames generator.FrameSource @@ -209,6 +228,7 @@ func New(cfg *config.Config, res *discovery.Resolved, opts Options) (*Runner, er r.maxFrame = mtu + 18 } r.frameSize = orDefault(opts.FrameSize, frameSizeFor(r.maxFrame, res.Link.Driver)) + r.multiBuffer = multiBufferFor(r.maxFrame, r.frameSize) r.limiter = rate.New(cfg.PPS, cfg.BPS, rate.WithBatch(txBatch)) statsOpts := []stats.Option{} @@ -225,6 +245,7 @@ func New(cfg *config.Config, res *discovery.Resolved, opts Options) (*Runner, er NumFrames: r.numFrames, Pattern: string(cfg.Mode), PacketSizes: sizes, + MultiBuffer: r.multiBuffer, } return r, nil } @@ -236,10 +257,15 @@ func orDefault(v, def int) int { return def } -// frameSizeFor picks a UMEM frame size that holds the largest packet. 2048 is -// the library default and covers everything up to a standard Ethernet frame; -// jumbo traffic needs page-sized frames, which are also what zero-copy wants -// on drivers that require them. +// frameSizeFor picks a UMEM frame size. 2048 is the library default and covers +// everything up to a standard Ethernet frame; anything larger gets a full page. +// +// A page is the ceiling, not a preference. An aligned-chunk UMEM must satisfy +// XDP_UMEM_MIN_CHUNK_SIZE (2048) <= chunk_size <= PAGE_SIZE, and go-afxdp never +// sets XDP_UMEM_UNALIGNED_CHUNK_FLAG, so asking for more makes XDP_UMEM_REG +// fail with a bare EINVAL. Frames therefore do not grow to fit a jumbo packet. +// Packets bigger than one frame span several, which is what multiBufferFor +// below decides. // // AWS ENA's zero-copy datapath needs page-sized (4096) frames; with the default // 2048 the bind silently falls back to native copy. So floor at 4096 on ena, @@ -247,19 +273,45 @@ func orDefault(v, def int) int { // with what the driver actually binds. Scoped to ena; other drivers keep the // smaller, more cache-friendly frames. func frameSizeFor(maxFrame int, driver string) int { - size := 32768 - for _, n := range []int{2048, 4096, 8192, 16384} { - if maxFrame <= n { - size = n - break - } - } - if driver == "ena" && size < 4096 { + size := 2048 + if maxFrame > size || driver == "ena" { size = 4096 } + // Every Linux architecture pages at 4 KiB or more, so this never bites in + // practice. It is here so the kernel's ceiling is expressed in the code + // rather than assumed. + if page := os.Getpagesize(); size > page { + size = page + } return size } +// multiBufferFor reports whether packets have to span several UMEM frames. +// +// This is the jumbo path: the socket binds with XDP_USE_SG and the XDP program +// loads with BPF_F_XDP_HAS_FRAGS, so a packet arrives as a chain of descriptors +// instead of being dropped for not fitting. It costs zero-copy on any device +// reporting xdp-zc-max-segs = 1, and the transmit side has to build through a +// staging buffer, so it stays off unless the traffic actually needs it. +func multiBufferFor(maxFrame, frameSize int) bool { + return maxFrame > frameSize +} + +// maxTxSegs is how many UMEM frames one transmitted packet may span. It mirrors +// the limit go-afxdp enforces, which in turn comes from the kernel building an +// skb of at most CONFIG_MAX_SKB_FRAGS + 1 buffers. Kept here so preflight can +// reject an impossible size with an explanation instead of letting the transmit +// loop fail on every batch. +const maxTxSegs = 18 + +// framesPerPacket is how many UMEM frames the largest packet occupies. +func framesPerPacket(maxFrame, frameSize int) int { + if maxFrame <= 0 || frameSize <= 0 { + return 1 + } + return (maxFrame + frameSize - 1) / frameSize +} + // buildGenerators makes one generator per queue. func (r *Runner) buildGenerators() ([]generator.Generator, error) { var srcMAC, dstMAC [6]byte @@ -294,6 +346,7 @@ func (r *Runner) Preflight(src discovery.Source) *Preflight { MaxFrameLen: r.maxFrame, NumFrames: r.numFrames, FrameSize: r.frameSize, + MultiBuffer: r.multiBuffer, AllLinks: links, }) } @@ -499,12 +552,13 @@ func (r *Runner) Run(ctx context.Context) error { func (r *Runner) fleetKey() fleetKey { o := r.umemOptions() return fleetKey{ - iface: r.res.Link.Name, - queues: r.queues, - filter: r.plan.Summary, - numFrames: o.NumFrames, - frameSize: o.FrameSize, - receives: r.plan.Receives(), + iface: r.res.Link.Name, + queues: r.queues, + filter: r.plan.Summary, + numFrames: o.NumFrames, + frameSize: o.FrameSize, + receives: r.plan.Receives(), + multiBuffer: r.multiBuffer, } } @@ -576,6 +630,12 @@ func (r *Runner) attach() error { // of parking, burning cores while forwarding nothing. afxdp.WithNeedWakeup(), } + if r.multiBuffer { + // Packets bigger than a frame have to chain across several. This + // also lets the program attach at a jumbo MTU on drivers that + // otherwise cap XDP to a single buffer. + opts = append(opts, afxdp.WithMultiBuffer()) + } if r.plan.KeepManagement { // Spare ARP, ND, SSH and DNS from the match-all redirect so the box // stays reachable while everything else is captured. diff --git a/internal/dataplane/runner_test.go b/internal/dataplane/runner_test.go index 03d8eb4..46b4198 100644 --- a/internal/dataplane/runner_test.go +++ b/internal/dataplane/runner_test.go @@ -1,27 +1,92 @@ package dataplane -import "testing" +import ( + "os" + "testing" +) -// frameSizeFor picks the UMEM frame size. The one subtlety is AWS ENA, whose -// zero-copy bind needs page-sized (4096) frames: a standard frame must be -// floored to 4096 there, while every other driver keeps the smaller 2048. -func TestFrameSizeForENA(t *testing.T) { +// frameSizeFor picks the UMEM frame size. Two things matter. A frame may never +// exceed a page, because an aligned-chunk UMEM larger than PAGE_SIZE makes +// XDP_UMEM_REG fail with a bare EINVAL. So jumbo traffic gets a page and +// chains, it does not get a bigger frame. And AWS ENA's zero-copy bind needs +// page-sized frames, so a standard frame is floored to 4096 there while every +// other driver keeps the smaller 2048. +func TestFrameSizeFor(t *testing.T) { tests := []struct { driver string maxFrame int want int }{ - {"ena", 1518, 4096}, // the fix: a standard frame is floored to a page on ena + {"ena", 1518, 4096}, // a standard frame is floored to a page on ena {"ixgbe", 1518, 2048}, // other drivers keep the smaller frame {"", 1518, 2048}, // unknown driver behaves like non-ena {"ena", 3018, 4096}, // already 4096, the floor is a no-op - {"ena", 9018, 16384}, // jumbo is unaffected by the floor - {"mlx5_core", 9018, 16384}, - {"ena", 64, 4096}, // a tiny frame still gets a page on ena + {"ena", 64, 4096}, // a tiny frame still gets a page on ena + + // Above 2048 every driver takes a page, and stops there. These are the + // sizes that used to ask for 8192 or 16384 and get EINVAL. + {"ixgbe", 2049, 4096}, + {"ixgbe", 4097, 4096}, + {"ena", 9018, 4096}, + {"mlx5_core", 9018, 4096}, + {"mlx5_core", 16384, 4096}, } for _, tt := range tests { - if got := frameSizeFor(tt.maxFrame, tt.driver); got != tt.want { + got := frameSizeFor(tt.maxFrame, tt.driver) + if got != tt.want { t.Errorf("frameSizeFor(%d, %q) = %d, want %d", tt.maxFrame, tt.driver, got, tt.want) } + if page := os.Getpagesize(); got > page { + t.Errorf("frameSizeFor(%d, %q) = %d, above the %d-byte page the kernel allows", + tt.maxFrame, tt.driver, got, page) + } + } +} + +// Chaining is what carries a packet that no longer fits one frame, so it must +// turn on exactly when that happens and stay off otherwise: it costs zero-copy +// on drivers that will not accept an XDP_USE_SG bind. +func TestMultiBufferFor(t *testing.T) { + tests := []struct { + maxFrame int + frameSize int + want bool + }{ + {60, 2048, false}, // a 64-byte frame + {1514, 2048, false}, // a standard frame + {2048, 2048, false}, // exactly a frame still fits + {2049, 4096, false}, // ...and gets a bigger frame rather than chaining + {4096, 4096, false}, // exactly a page fits + {4097, 4096, true}, // one byte over is where chaining starts + {9014, 4096, true}, // a jumbo frame + } + for _, tt := range tests { + if got := multiBufferFor(tt.maxFrame, tt.frameSize); got != tt.want { + t.Errorf("multiBufferFor(%d, %d) = %v, want %v", + tt.maxFrame, tt.frameSize, got, tt.want) + } + } +} + +// framesPerPacket feeds preflight's chain-length check, so the boundaries have +// to be exact rather than approximately right. +func TestFramesPerPacket(t *testing.T) { + tests := []struct { + maxFrame int + frameSize int + want int + }{ + {0, 4096, 1}, // degenerate input never reports zero frames + {60, 4096, 1}, // + {4096, 4096, 1}, // exactly one frame + {4097, 4096, 2}, // one byte over needs a second + {9014, 4096, 3}, // a jumbo frame spans three + {8192, 4096, 2}, + } + for _, tt := range tests { + if got := framesPerPacket(tt.maxFrame, tt.frameSize); got != tt.want { + t.Errorf("framesPerPacket(%d, %d) = %d, want %d", + tt.maxFrame, tt.frameSize, got, tt.want) + } } } diff --git a/internal/dataplane/session.go b/internal/dataplane/session.go index f5fe8c2..bd9a25e 100644 --- a/internal/dataplane/session.go +++ b/internal/dataplane/session.go @@ -27,6 +27,11 @@ type fleetKey struct { // receives changes the frame split between the transmit and receive pools // and the ring depths, so it is part of the bind too. receives bool + // multiBuffer is a bind flag (XDP_USE_SG) and a program flag + // (BPF_F_XDP_HAS_FRAGS), so it cannot be changed on an attached fleet. It + // needs its own field because frameSize does not imply it: at 4096-byte + // frames a 4100-byte run chains nothing and a 9000-byte run chains. + multiBuffer bool } // Session keeps an AF_XDP fleet attached across several runs. diff --git a/internal/dataplane/worker.go b/internal/dataplane/worker.go index 7739cd9..a34650f 100644 --- a/internal/dataplane/worker.go +++ b/internal/dataplane/worker.go @@ -2,6 +2,7 @@ package dataplane import ( "context" + "errors" "time" afxdp "github.com/atoonk/go-afxdp" @@ -18,6 +19,11 @@ const fcsLen = 4 // preamble + 1-byte start-frame delimiter + 12-byte interframe gap. const wireOverhead = 20 +// errNothingBuilt is the jumbo sender's stand-in for the error SendFunc raises +// when its build callback fails: the generator had nothing to give, so the +// batch is empty. It is only ever seen alongside a non-zero build-error count. +var errNothingBuilt = errors.New("generator produced no packets") + // pacingFloor is the smallest wait a PCAP replay in original-timing mode will // actually sleep. Timer granularity on Linux is tens of microseconds, so // shorter gaps are accumulated and paid off together. @@ -39,7 +45,7 @@ func (r *Runner) txLoop(ctx context.Context, queue int, xsk *afxdp.Socket, gen g finite, isFinite := gen.(generator.Finite) avgWire := gen.AvgWireBytes() - // Per-batch accumulators, declared once so the closure below captures + // Per-batch accumulators, declared once so the closures below capture // stack slots rather than allocating each time round. var ( bytes uint64 @@ -50,12 +56,26 @@ func (r *Runner) txLoop(ctx context.Context, queue int, xsk *afxdp.Socket, gen g owed time.Duration ) + // count folds one built packet into the accumulators. The FCS the NIC will + // append is included, so byte totals and average frame sizes are in the + // same units as --packet-size. + count := func(n int, class stats.Class) { + bytes += uint64(n + fcsLen) + clsPkts[class]++ + clsBytes[class] += uint64(n + fcsLen) + } + + // send builds and queues up to want packets, returning how many the kernel + // took. Which implementation runs is decided once, here, rather than per + // batch. + batchCap, send := r.sender(xsk, gen, count, &buildErrs) + for { if ctx.Err() != nil { return } - want := txBatch + want := batchCap if isFinite { switch left := finite.Remaining(); { case left == 0: @@ -98,22 +118,7 @@ func (r *Runner) txLoop(ctx context.Context, queue int, xsk *afxdp.Socket, gen g clsBytes = [3]uint64{} buildErrs = 0 - sent, err := xsk.SendFunc(grant.Packets, func(_ int, frame []byte) int { - n, class := gen.Next(frame) - if n <= 0 { - // A generator with nothing left. Returning 0 would put an - // empty frame on the wire, so report it as a build failure: - // SendFunc abandons the whole batch unqueued. - buildErrs++ - return -1 - } - // Count the FCS the NIC will append, so byte totals and average - // frame sizes are in the same units as --packet-size. - bytes += uint64(n + fcsLen) - clsPkts[class]++ - clsBytes[class] += uint64(n + fcsLen) - return n - }) + sent, err := send(grant.Packets) switch { case err != nil && closedSocket(err): @@ -148,6 +153,83 @@ func (r *Runner) txLoop(ctx context.Context, queue int, xsk *afxdp.Socket, gen g } } +// sender picks how this run puts packets on the wire and returns the batch +// size that goes with it. count is called for every packet the kernel accepts; +// buildErrs is incremented when the generator has nothing left to give. +// +// The two differ in where the packet is written. Normally the generator +// serialises straight into the UMEM frame and nothing is copied. That is not +// expressible for a packet larger than a frame: SendFunc hands out exactly one +// frame and never sets XDP_PKT_CONTD, so it cannot chain. The jumbo path +// stages the packet in ordinary memory and lets SendBatch split it across +// frames, paying one copy per packet. That is affordable precisely because a +// packet needing chaining is at least a frame long, so the packet rate is low. +func (r *Runner) sender( + xsk *afxdp.Socket, + gen generator.Generator, + count func(int, stats.Class), + buildErrs *int, +) (int, func(want int) (int, error)) { + if !r.multiBuffer { + return txBatch, func(want int) (int, error) { + return xsk.SendFunc(want, func(_ int, frame []byte) int { + n, class := gen.Next(frame) + if n <= 0 { + // A generator with nothing left. Returning 0 would put an + // empty frame on the wire, so report it as a build failure: + // SendFunc abandons the whole batch unqueued. + *buildErrs++ + return -1 + } + count(n, class) + return n + }) + } + } + + // Allocated once per queue, so the steady state still allocates nothing. + var ( + arena = make([]byte, jumboTxBatch*r.maxFrame) + payloads = make([][]byte, jumboTxBatch) + pktLen [jumboTxBatch]int + pktClass [jumboTxBatch]stats.Class + ) + return jumboTxBatch, func(want int) (int, error) { + if want > jumboTxBatch { + want = jumboTxBatch + } + built := 0 + for i := range want { + buf := arena[i*r.maxFrame : (i+1)*r.maxFrame] + n, class := gen.Next(buf) + if n <= 0 { + *buildErrs++ + break + } + payloads[built] = buf[:n] + pktLen[built], pktClass[built] = n, class + built++ + } + if built == 0 { + // Nothing could be built. Report it the way SendFunc does when its + // callback fails, so the caller's exhaustion handling is the same + // on both paths rather than spinning on an empty batch. + return 0, errNothingBuilt + } + sent, err := xsk.SendBatch(payloads[:built]) + if err != nil { + return 0, err + } + // SendBatch takes whole packets and may take fewer than offered when + // the ring is short of room, so count what went rather than what was + // built. + for i := range sent { + count(pktLen[i], pktClass[i]) + } + return sent, nil + } +} + // rxLoop is one queue's receive loop, running only when a receive mode is // enabled. It owns this socket's receive side exclusively. // @@ -178,12 +260,22 @@ func (r *Runner) rxLoop(ctx context.Context, queue int, xsk *afxdp.Socket) { continue } - descs := xsk.Receive(n) - for _, d := range descs { - frame := xsk.GetFrame(d) - ctr.AddRx(uint64(len(frame)+fcsLen), generator.Classify(frame)) + // ReceivePackets rather than Receive, always. Receive hands back one + // descriptor per *frame*, so a chained jumbo packet would be counted as + // several undersized ones with headers on only the first. Grouping is + // free when nothing chains: with multi-buffer off every packet has + // exactly one fragment. The two must not be mixed on one socket either, + // since ReceivePackets carries partial-chain state between calls. + pkts := xsk.ReceivePackets(n) + for _, p := range pkts { + if len(p) == 0 { + continue + } + // Length is the whole packet, and only the first fragment carries + // the headers Classify reads. + ctr.AddRx(uint64(p.Len()+fcsLen), generator.Classify(xsk.GetFrame(p[0]))) } - xsk.Recycle(descs) + xsk.RecyclePackets(pkts) } } diff --git a/internal/pcapfile/pcapfile.go b/internal/pcapfile/pcapfile.go index ea9647a..cdceeb2 100644 --- a/internal/pcapfile/pcapfile.go +++ b/internal/pcapfile/pcapfile.go @@ -27,8 +27,10 @@ const ( // MinFrame is the smallest thing that can be an Ethernet frame: two MAC // addresses and an EtherType. MinFrame = 14 - // MaxFrame is the largest frame that can be transmitted. It matches the - // biggest UMEM frame Wireblast will configure. + // MaxFrame is the largest frame that can be transmitted. A UMEM frame is + // at most a page, so anything above that is chained across several; this + // stays comfortably inside how many frames one packet may span, and + // preflight rejects a capture that does not. MaxFrame = 16384 // MaxPackets bounds how many frames a capture may hold, so pointing // Wireblast at a capture with millions of tiny records fails politely diff --git a/internal/tui/dashboard.go b/internal/tui/dashboard.go index 1ca30e2..0cee6dc 100644 --- a/internal/tui/dashboard.go +++ b/internal/tui/dashboard.go @@ -165,6 +165,11 @@ func (m model) dashHeader(s *stats.Snapshot) string { } iface := fmt.Sprintf("%s · %d queue(s) · %s XDP · %s", driverOr(i.Driver), i.Queues, i.XDPMode, zc) + if i.MultiBuffer { + // Worth showing next to the copy/zero-copy word, since chaining is + // usually the reason a jumbo run reports copy. + iface += " · multi-buffer" + } if i.Tuning != "" && i.Tuning != "untuned" { iface += " · napi " + i.Tuning }