From f391f490ff9ec98054cb1149e65bf3e92fcf107c Mon Sep 17 00:00:00 2001 From: Rucha0901 Date: Mon, 3 Aug 2026 23:08:04 +0530 Subject: [PATCH] feat(network): support multi-queue TAP devices Introduce multi-queue TAP device support for high-throughput container networking scenarios when using QEMU and Cloud Hypervisor. - Add urunc.io/net-queues annotation and net_queues config parsing - Update createTapDevice to set multiqueue flags and multi-FD setup - Propagate queue parameters to QEMU and Cloud Hypervisor CLI builders Fixes: #893 Signed-off-by: Rucha0901 --- pkg/network/network.go | 32 +++++++----- pkg/network/network_dynamic.go | 4 +- pkg/network/network_static.go | 4 +- pkg/unikontainers/config.go | 19 +++++++ .../hypervisors/cloud_hypervisor.go | 6 ++- .../hypervisors/cloud_hypervisor_test.go | 52 +++++++++++++++++++ pkg/unikontainers/hypervisors/qemu.go | 14 +++-- pkg/unikontainers/hypervisors/qemu_test.go | 15 ++++++ pkg/unikontainers/types/types.go | 2 + pkg/unikontainers/unikontainers.go | 25 ++++++++- pkg/unikontainers/urunc_config.go | 5 ++ pkg/unikontainers/urunc_config_test.go | 2 + 12 files changed, 159 insertions(+), 21 deletions(-) create mode 100644 pkg/unikontainers/hypervisors/cloud_hypervisor_test.go diff --git a/pkg/network/network.go b/pkg/network/network.go index cbc73c707..64c5d24d6 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -38,7 +38,7 @@ type UnikernelNetworkInfo struct { EthDevice Interface } type Manager interface { - NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) + NetworkSetup(uid uint32, gid uint32, queues int) (*UnikernelNetworkInfo, error) } type Interface struct { @@ -79,7 +79,18 @@ func getTapIndex() (int, error) { return tapCount, nil } -func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.Link, error) { +func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32, queues int) (netlink.Link, error) { + if queues <= 0 { + queues = 1 + } + + flags := netlink.TUNTAP_VNET_HDR + if queues > 1 { + flags |= netlink.TUNTAP_MULTI_QUEUE + } else { + flags |= netlink.TUNTAP_ONE_QUEUE + } + tapLinkAttrs := netlink.NewLinkAttrs() tapLinkAttrs.Name = name tapLink := &netlink.Tuntap{ @@ -88,12 +99,9 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L // We want a tap device (L2) as opposed to a tun (L3) Mode: netlink.TUNTAP_MODE_TAP, - // Firecracker does not support multiqueue tap devices at this time: - // https://github.com/firecracker-microvm/firecracker/issues/750 - Queues: 1, + Queues: queues, - Flags: netlink.TUNTAP_ONE_QUEUE | // single queue tap device - netlink.TUNTAP_VNET_HDR, // parse vnet headers added by the vm's virtio_net implementation + Flags: flags, } err := netlink.LinkAdd(tapLink) @@ -222,12 +230,12 @@ func addRedirectFilter(source netlink.Link, target netlink.Link) error { }) } -func networkSetup(tapName string, ipAddress string, redirectLink netlink.Link, addTCRules bool, uid uint32, gid uint32) (netlink.Link, error) { - netlog.Debugf("starting for tapName=%s ipAddress=%s redirectLink=%s addTCRules=%v", - tapName, ipAddress, redirectLink.Attrs().Name, addTCRules) +func networkSetup(tapName string, ipAddress string, redirectLink netlink.Link, addTCRules bool, uid uint32, gid uint32, queues int) (netlink.Link, error) { + netlog.Debugf("starting for tapName=%s ipAddress=%s redirectLink=%s addTCRules=%v queues=%d", + tapName, ipAddress, redirectLink.Attrs().Name, addTCRules, queues) // Create TAP - netlog.Debugf("creating tap device %s (mtu=%d)", tapName, redirectLink.Attrs().MTU) - newTapDevice, err := createTapDevice(tapName, redirectLink.Attrs().MTU, uid, gid) + netlog.Debugf("creating tap device %s (mtu=%d, queues=%d)", tapName, redirectLink.Attrs().MTU, queues) + newTapDevice, err := createTapDevice(tapName, redirectLink.Attrs().MTU, uid, gid, queues) if err != nil { return nil, fmt.Errorf("createTapDevice(%s) failed: %w", tapName, err) } diff --git a/pkg/network/network_dynamic.go b/pkg/network/network_dynamic.go index 1d6a2237b..2bdc39be7 100644 --- a/pkg/network/network_dynamic.go +++ b/pkg/network/network_dynamic.go @@ -32,7 +32,7 @@ type DynamicNetwork struct { // FIXME: CUrrently only one tap device per netns can provide functional networking. We need to find a proper way to handle networking // for multiple unikernels in the same pod/network namespace. // See: https://github.com/urunc-dev/urunc/issues/13 -func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { +func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32, queues int) (*UnikernelNetworkInfo, error) { tapIndex, err := getTapIndex() if err != nil { return nil, fmt.Errorf("getTapIndex failed: %w", err) @@ -50,7 +50,7 @@ func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkI newTapName := strings.ReplaceAll(DefaultTap, "X", strconv.Itoa(tapIndex)) netlog.Debugf("creating tap device %s", newTapName) - newTapDevice, err := networkSetup(newTapName, "", redirectLink, true, uid, gid) + newTapDevice, err := networkSetup(newTapName, "", redirectLink, true, uid, gid, queues) if err != nil { return nil, fmt.Errorf("networkSetup(%s) failed: %w", newTapName, err) } diff --git a/pkg/network/network_static.go b/pkg/network/network_static.go index fa46608cc..617909780 100644 --- a/pkg/network/network_static.go +++ b/pkg/network/network_static.go @@ -88,7 +88,7 @@ func setNATRule(iface string, sourceIP string) error { return nil } -func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { +func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32, queues int) (*UnikernelNetworkInfo, error) { newTapName := strings.ReplaceAll(DefaultTap, "X", "0") addTCRules := false redirectLink, err := discoverContainerIface() @@ -96,7 +96,7 @@ func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkIn netlog.Errorf("failed to find container interface, (unikernel may have been spawned using ctr): %v", err) return nil, err } - newTapDevice, err := networkSetup(newTapName, StaticIPAddr, redirectLink, addTCRules, uid, gid) + newTapDevice, err := networkSetup(newTapName, StaticIPAddr, redirectLink, addTCRules, uid, gid, queues) if err != nil { return nil, err } diff --git a/pkg/unikontainers/config.go b/pkg/unikontainers/config.go index 14a4cb1a7..2728aa7a2 100644 --- a/pkg/unikontainers/config.go +++ b/pkg/unikontainers/config.go @@ -46,6 +46,8 @@ const ( annotBlockMntPoint = "com.urunc.unikernel.blkMntPoint" annotMountRootfs = "com.urunc.unikernel.mountRootfs" annotNetDev = "com.urunc.unikernel.solo5NetDev" + annotNetQueues = "urunc.io/net-queues" + annotNetQueuesAlt = "com.urunc.unikernel.netQueues" ) // A UnikernelConfig struct holds the info provided by bima image on how to execute our unikernel @@ -60,6 +62,7 @@ type UnikernelConfig struct { BlkMntPoint string `json:"com.urunc.unikernel.blkMntPoint,omitempty"` MountRootfs string `json:"com.urunc.unikernel.mountRootfs"` NetDev string `json:"com.urunc.unikernel.solo5NetDev,omitempty"` + NetQueues string `json:"urunc.io/net-queues,omitempty"` } // validate checks if the mandatory configuration fields are present. @@ -122,6 +125,10 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig { blkMntPoint := spec.Annotations[annotBlockMntPoint] MountRootfs := spec.Annotations[annotMountRootfs] netDev := spec.Annotations[annotNetDev] + netQueues := spec.Annotations[annotNetQueues] + if netQueues == "" { + netQueues = spec.Annotations[annotNetQueuesAlt] + } uniklog.WithFields(logrus.Fields{ "unikernelType": unikernelType, "unikernelVersion": unikernelVersion, @@ -133,6 +140,7 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig { "blkMntPoint": blkMntPoint, "mountRootfs": MountRootfs, "netDev": netDev, + "netQueues": netQueues, }).WithField("source", "spec").Debug("urunc annotations") return &UnikernelConfig{ @@ -146,6 +154,7 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig { BlkMntPoint: blkMntPoint, MountRootfs: MountRootfs, NetDev: netDev, + NetQueues: netQueues, } } @@ -186,6 +195,7 @@ func getConfigFromJSON(jsonFilePath string) (*UnikernelConfig, error) { "blkMntPoint": tryDecode(conf.BlkMntPoint), "mountRootfs": tryDecode(conf.MountRootfs), "netDev": tryDecode(conf.NetDev), + "netQueues": tryDecode(conf.NetQueues), }).WithField("source", uruncJSONFilename).Debug("urunc annotations") return &conf, nil @@ -262,6 +272,12 @@ func (c *UnikernelConfig) decode() error { } c.NetDev = string(decoded) + if c.NetQueues != "" { + if decoded, err := base64.StdEncoding.DecodeString(c.NetQueues); err == nil { + c.NetQueues = string(decoded) + } + } + return nil } @@ -298,6 +314,9 @@ func (c *UnikernelConfig) Map() map[string]string { if c.NetDev != "" { myMap[annotNetDev] = c.NetDev } + if c.NetQueues != "" { + myMap[annotNetQueues] = c.NetQueues + } return myMap } diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor.go b/pkg/unikontainers/hypervisors/cloud_hypervisor.go index 606a3c02e..d5ecf238f 100644 --- a/pkg/unikontainers/hypervisors/cloud_hypervisor.go +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor.go @@ -100,7 +100,11 @@ func (ch *CloudHypervisor) BuildExecCmd(args types.ExecArgs, ukernel types.Unike netCli := ukernel.MonitorNetCli(args.Net.TapDev, args.Net.MAC) if netCli == "" { // Default network configuration for Cloud Hypervisor - exArgs = append(exArgs, "--net", fmt.Sprintf("tap=%s,mac=%s,mtu=%d", args.Net.TapDev, args.Net.MAC, args.Net.MTU)) + netStr := fmt.Sprintf("tap=%s,mac=%s,mtu=%d", args.Net.TapDev, args.Net.MAC, args.Net.MTU) + if args.Net.Queues > 1 { + netStr += fmt.Sprintf(",num_queues=%d", args.Net.Queues) + } + exArgs = append(exArgs, "--net", netStr) } else { exArgs = append(exArgs, strings.Split(strings.TrimSpace(netCli), " ")...) } diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor_test.go b/pkg/unikontainers/hypervisors/cloud_hypervisor_test.go new file mode 100644 index 000000000..276e0df94 --- /dev/null +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor_test.go @@ -0,0 +1,52 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +func TestCloudHypervisorMultiQueueNet(t *testing.T) { + ch := &CloudHypervisor{binaryPath: "/usr/bin/cloud-hypervisor", binary: CloudHypervisorBinary} + args := types.ExecArgs{ + UnikernelPath: "/path/to/kernel", + Net: types.NetDevParams{ + TapDev: "tap0", + MAC: "52:54:00:12:34:56", + MTU: 1500, + Queues: 4, + }, + } + + execCmd, err := ch.BuildExecCmd(args, &fakeUnikernel{}) + assert.NoError(t, err) + + foundNet := false + for i, arg := range execCmd { + if arg == "--net" && i+1 < len(execCmd) { + netVal := execCmd[i+1] + assert.Contains(t, netVal, "tap=tap0") + assert.Contains(t, netVal, "mac=52:54:00:12:34:56") + assert.Contains(t, netVal, "mtu=1500") + assert.Contains(t, netVal, "num_queues=4") + foundNet = true + break + } + } + assert.True(t, foundNet, "--net argument should be present") +} diff --git a/pkg/unikontainers/hypervisors/qemu.go b/pkg/unikontainers/hypervisors/qemu.go index 1ac77f870..ed57aa20f 100644 --- a/pkg/unikontainers/hypervisors/qemu.go +++ b/pkg/unikontainers/hypervisors/qemu.go @@ -98,10 +98,13 @@ func (q *Qemu) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel) ([]str if netcli == "" { netcli += " -netdev tap,id=net0,script=no,downscript=no,ifname=" netcli += args.Net.TapDev + if args.Net.Queues > 1 { + netcli += fmt.Sprintf(",queues=%d", args.Net.Queues) + } if q.vhost { netcli += ",vhost=on" } - netcli += fmt.Sprintf(" %s,host_mtu=%d,mac=%s", getVirtioNetArg(), args.Net.MTU, args.Net.MAC) + netcli += fmt.Sprintf(" %s,host_mtu=%d,mac=%s", getVirtioNetArg(args.Net.Queues), args.Net.MTU, args.Net.MAC) } cmdString += netcli } else { @@ -152,10 +155,15 @@ func (q *Qemu) PreExec(_ types.ExecArgs) error { return nil } -func getVirtioNetArg() string { +func getVirtioNetArg(queues int) string { devType := "virtio-net-pci" if runtime.GOARCH == "arm64" { devType = "virtio-net-device" } - return "-device " + devType + ",netdev=net0" + arg := "-device " + devType + ",netdev=net0" + if queues > 1 { + vectors := 2*queues + 2 + arg += fmt.Sprintf(",mq=on,vectors=%d", vectors) + } + return arg } diff --git a/pkg/unikontainers/hypervisors/qemu_test.go b/pkg/unikontainers/hypervisors/qemu_test.go index fff45244c..74ab80267 100644 --- a/pkg/unikontainers/hypervisors/qemu_test.go +++ b/pkg/unikontainers/hypervisors/qemu_test.go @@ -146,6 +146,21 @@ func TestQemuBuildExecCmd(t *testing.T) { }, mustNotContain: []string{"-nic none", "vhost=on"}, }, + { + name: "multiqueue tap renders queues parameter and mq virtio-net flags", + args: types.ExecArgs{ + UnikernelPath: testKernelPath, + Command: testCommand, + Net: types.NetDevParams{TapDev: "tap0", MAC: "52:54:00:12:34:56", MTU: 1500, Queues: 4}, + }, + unikernel: &fakeUnikernel{}, + mustContain: []string{ + "-netdev tap", + "ifname=tap0,queues=4", + "mq=on", + "vectors=10", + }, + }, { name: "vhost on emits vhost=on", vhost: true, diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index bbfde55cd..8e3622beb 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -51,6 +51,7 @@ type NetDevParams struct { MAC string // The MAC address of the guest network device TapDev string // The tap device name MTU int // The MTU value of the tap device + Queues int // The queue count of the tap device } type BlockDevParams struct { @@ -140,4 +141,5 @@ type MonitorConfig struct { BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + NetQueues int `toml:"net_queues,omitempty"` // Optional: network queue count for multi-queue TAP } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index b218c4cea..39d5021f4 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" "runtime" + "strconv" "strings" "sync" "syscall" @@ -255,6 +256,25 @@ func (u *Unikontainer) SetRunningState() error { return u.saveContainerState() } +func (u *Unikontainer) getNetworkQueues() int { + queuesStr := u.State.Annotations[annotNetQueues] + if queuesStr == "" { + queuesStr = u.State.Annotations[annotNetQueuesAlt] + } + if queuesStr != "" { + if q, err := strconv.Atoi(queuesStr); err == nil && q > 0 { + return q + } + } + if u.UruncCfg != nil { + vmmType := u.State.Annotations[annotHypervisor] + if monCfg, exists := u.UruncCfg.Monitors[vmmType]; exists && monCfg.NetQueues > 0 { + return monCfg.NetQueues + } + } + return 1 +} + func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { networkType := u.getNetworkType() uniklog.WithField("network type", networkType).Debug("Retrieved network type") @@ -264,7 +284,10 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { return netArgs, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) } - networkInfo, err := netManager.NetworkSetup(u.Spec.Process.User.UID, u.Spec.Process.User.GID) + queues := u.getNetworkQueues() + netArgs.Queues = queues + + networkInfo, err := netManager.NetworkSetup(u.Spec.Process.User.UID, u.Spec.Process.User.GID, queues) if err != nil { // TODO: Handle this case better. We do not need to show an error // since there was no network in the container. Therefore, we diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 22573f43c..bd41a5ab1 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -145,6 +145,7 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"net_queues"] = strconv.Itoa(hvCfg.NetQueues) } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -198,6 +199,10 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } else { hvCfg.Vhost = boolVal } + case "net_queues": + if intVal, err := strconv.Atoi(val); err == nil && intVal > 0 { + hvCfg.NetQueues = intVal + } } cfg.Monitors[hv] = hvCfg } diff --git a/pkg/unikontainers/urunc_config_test.go b/pkg/unikontainers/urunc_config_test.go index a51a3eb96..f719c1088 100644 --- a/pkg/unikontainers/urunc_config_test.go +++ b/pkg/unikontainers/urunc_config_test.go @@ -60,6 +60,7 @@ func TestUruncConfigFromMap(t *testing.T) { testQemuBinaryKey: testQemuBinaryPath, testQemuDataKey: testQemuDataPath, testQemuVhostKey: "true", + "urunc_config.monitors.qemu.net_queues": "4", } config := UruncConfigFromMap(cfgMap) @@ -72,6 +73,7 @@ func TestUruncConfigFromMap(t *testing.T) { assert.Equal(t, testQemuBinaryPath, qemuConfig.BinaryPath) assert.Equal(t, testQemuDataPath, qemuConfig.DataPath) assert.True(t, qemuConfig.Vhost) + assert.Equal(t, 4, qemuConfig.NetQueues) }) t.Run("multiple monitors", func(t *testing.T) {