feat(network): support multi-queue TAP devices

pkg/network/network.go

@@ -38,7 +38,7 @@ type UnikernelNetworkInfo struct {
 	EthDevice Interface
 }
 type Manager interface {
-	NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error)
+	NetworkSetup(uid uint32, gid uint32, queues int) (*UnikernelNetworkInfo, error)
 }
 
 type Interface struct {
@@ -79,7 +79,18 @@ func getTapIndex() (int, error) {
 	return tapCount, nil
 }
 
-func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.Link, error) {
+func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32, queues int) (netlink.Link, error) {
+	if queues <= 0 {
+		queues = 1
+	}
+
+	flags := netlink.TUNTAP_VNET_HDR
+	if queues > 1 {
+		flags |= netlink.TUNTAP_MULTI_QUEUE
+	} else {
+		flags |= netlink.TUNTAP_ONE_QUEUE
+	}
+
 	tapLinkAttrs := netlink.NewLinkAttrs()
 	tapLinkAttrs.Name = name
 	tapLink := &netlink.Tuntap{
@@ -88,12 +99,9 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L
 		// We want a tap device (L2) as opposed to a tun (L3)
 		Mode: netlink.TUNTAP_MODE_TAP,
 
-		// Firecracker does not support multiqueue tap devices at this time:
-		// https://github.com/firecracker-microvm/firecracker/issues/750
-		Queues: 1,
+		Queues: queues,
 
-		Flags: netlink.TUNTAP_ONE_QUEUE | // single queue tap device
-			netlink.TUNTAP_VNET_HDR, // parse vnet headers added by the vm's virtio_net implementation
+		Flags: flags,
 	}
 
 	err := netlink.LinkAdd(tapLink)
@@ -222,12 +230,12 @@ func addRedirectFilter(source netlink.Link, target netlink.Link) error {
 	})
 }
 
-func networkSetup(tapName string, ipAddress string, redirectLink netlink.Link, addTCRules bool, uid uint32, gid uint32) (netlink.Link, error) {
-	netlog.Debugf("starting for tapName=%s ipAddress=%s redirectLink=%s addTCRules=%v",
-		tapName, ipAddress, redirectLink.Attrs().Name, addTCRules)
+func networkSetup(tapName string, ipAddress string, redirectLink netlink.Link, addTCRules bool, uid uint32, gid uint32, queues int) (netlink.Link, error) {
+	netlog.Debugf("starting for tapName=%s ipAddress=%s redirectLink=%s addTCRules=%v queues=%d",
+		tapName, ipAddress, redirectLink.Attrs().Name, addTCRules, queues)
 	// Create TAP
-	netlog.Debugf("creating tap device %s (mtu=%d)", tapName, redirectLink.Attrs().MTU)
-	newTapDevice, err := createTapDevice(tapName, redirectLink.Attrs().MTU, uid, gid)
+	netlog.Debugf("creating tap device %s (mtu=%d, queues=%d)", tapName, redirectLink.Attrs().MTU, queues)
+	newTapDevice, err := createTapDevice(tapName, redirectLink.Attrs().MTU, uid, gid, queues)
 	if err != nil {
 		return nil, fmt.Errorf("createTapDevice(%s) failed: %w", tapName, err)
 	}

pkg/network/network_dynamic.go

@@ -32,7 +32,7 @@ type DynamicNetwork struct {
 // FIXME: CUrrently only one tap device per netns can provide functional networking. We need to find a proper way to handle networking
 // for multiple unikernels in the same pod/network namespace.
 // See: https://github.com/urunc-dev/urunc/issues/13
-func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) {
+func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32, queues int) (*UnikernelNetworkInfo, error) {
 	tapIndex, err := getTapIndex()
 	if err != nil {
 		return nil, fmt.Errorf("getTapIndex failed: %w", err)
@@ -50,7 +50,7 @@ func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkI
 	newTapName := strings.ReplaceAll(DefaultTap, "X", strconv.Itoa(tapIndex))
 	netlog.Debugf("creating tap device %s", newTapName)
 
-	newTapDevice, err := networkSetup(newTapName, "", redirectLink, true, uid, gid)
+	newTapDevice, err := networkSetup(newTapName, "", redirectLink, true, uid, gid, queues)
 	if err != nil {
 		return nil, fmt.Errorf("networkSetup(%s) failed: %w", newTapName, err)
 	}

pkg/network/network_static.go

@@ -88,15 +88,15 @@ func setNATRule(iface string, sourceIP string) error {
 	return nil
 }
 
-func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) {
+func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32, queues int) (*UnikernelNetworkInfo, error) {
 	newTapName := strings.ReplaceAll(DefaultTap, "X", "0")
 	addTCRules := false
 	redirectLink, err := discoverContainerIface()
 	if err != nil {
 		netlog.Errorf("failed to find container interface, (unikernel may have been spawned using ctr): %v", err)
 		return nil, err
 	}
-	newTapDevice, err := networkSetup(newTapName, StaticIPAddr, redirectLink, addTCRules, uid, gid)
+	newTapDevice, err := networkSetup(newTapName, StaticIPAddr, redirectLink, addTCRules, uid, gid, queues)
 	if err != nil {
 		return nil, err
 	}

pkg/unikontainers/config.go

@@ -46,6 +46,8 @@ const (
 	annotBlockMntPoint = "com.urunc.unikernel.blkMntPoint"
 	annotMountRootfs   = "com.urunc.unikernel.mountRootfs"
 	annotNetDev        = "com.urunc.unikernel.solo5NetDev"
+	annotNetQueues     = "urunc.io/net-queues"
+	annotNetQueuesAlt  = "com.urunc.unikernel.netQueues"
 )
 
 // A UnikernelConfig struct holds the info provided by bima image on how to execute our unikernel
@@ -60,6 +62,7 @@ type UnikernelConfig struct {
 	BlkMntPoint      string `json:"com.urunc.unikernel.blkMntPoint,omitempty"`
 	MountRootfs      string `json:"com.urunc.unikernel.mountRootfs"`
 	NetDev           string `json:"com.urunc.unikernel.solo5NetDev,omitempty"`
+	NetQueues        string `json:"urunc.io/net-queues,omitempty"`
 }
 
 // validate checks if the mandatory configuration fields are present.
@@ -122,6 +125,10 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig {
 	blkMntPoint := spec.Annotations[annotBlockMntPoint]
 	MountRootfs := spec.Annotations[annotMountRootfs]
 	netDev := spec.Annotations[annotNetDev]
+	netQueues := spec.Annotations[annotNetQueues]
+	if netQueues == "" {
+		netQueues = spec.Annotations[annotNetQueuesAlt]
+	}
 	uniklog.WithFields(logrus.Fields{
 		"unikernelType":    unikernelType,
 		"unikernelVersion": unikernelVersion,
@@ -133,6 +140,7 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig {
 		"blkMntPoint":      blkMntPoint,
 		"mountRootfs":      MountRootfs,
 		"netDev":           netDev,
+		"netQueues":        netQueues,
 	}).WithField("source", "spec").Debug("urunc annotations")
 
 	return &UnikernelConfig{
@@ -146,6 +154,7 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig {
 		BlkMntPoint:      blkMntPoint,
 		MountRootfs:      MountRootfs,
 		NetDev:           netDev,
+		NetQueues:        netQueues,
 	}
 }
 
@@ -186,6 +195,7 @@ func getConfigFromJSON(jsonFilePath string) (*UnikernelConfig, error) {
 		"blkMntPoint":      tryDecode(conf.BlkMntPoint),
 		"mountRootfs":      tryDecode(conf.MountRootfs),
 		"netDev":           tryDecode(conf.NetDev),
+		"netQueues":        tryDecode(conf.NetQueues),
 	}).WithField("source", uruncJSONFilename).Debug("urunc annotations")
 
 	return &conf, nil
@@ -262,6 +272,12 @@ func (c *UnikernelConfig) decode() error {
 	}
 	c.NetDev = string(decoded)
 
+	if c.NetQueues != "" {
+		if decoded, err := base64.StdEncoding.DecodeString(c.NetQueues); err == nil {
+			c.NetQueues = string(decoded)
+		}
+	}
+
 	return nil
 }
 
@@ -298,6 +314,9 @@ func (c *UnikernelConfig) Map() map[string]string {
 	if c.NetDev != "" {
 		myMap[annotNetDev] = c.NetDev
 	}
+	if c.NetQueues != "" {
+		myMap[annotNetQueues] = c.NetQueues
+	}
 
 	return myMap
 }

pkg/unikontainers/hypervisors/cloud_hypervisor.go

@@ -100,7 +100,11 @@ func (ch *CloudHypervisor) BuildExecCmd(args types.ExecArgs, ukernel types.Unike
 		netCli := ukernel.MonitorNetCli(args.Net.TapDev, args.Net.MAC)
 		if netCli == "" {
 			// Default network configuration for Cloud Hypervisor
-			exArgs = append(exArgs, "--net", fmt.Sprintf("tap=%s,mac=%s,mtu=%d", args.Net.TapDev, args.Net.MAC, args.Net.MTU))
+			netStr := fmt.Sprintf("tap=%s,mac=%s,mtu=%d", args.Net.TapDev, args.Net.MAC, args.Net.MTU)
+			if args.Net.Queues > 1 {
+				netStr += fmt.Sprintf(",num_queues=%d", args.Net.Queues)
+			}
+			exArgs = append(exArgs, "--net", netStr)
 		} else {
 			exArgs = append(exArgs, strings.Split(strings.TrimSpace(netCli), " ")...)
 		}

pkg/unikontainers/hypervisors/cloud_hypervisor_test.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2023-2026, Nubificus LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hypervisors
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/urunc-dev/urunc/pkg/unikontainers/types"
+)
+
+func TestCloudHypervisorMultiQueueNet(t *testing.T) {
+	ch := &CloudHypervisor{binaryPath: "/usr/bin/cloud-hypervisor", binary: CloudHypervisorBinary}
+	args := types.ExecArgs{
+		UnikernelPath: "/path/to/kernel",
+		Net: types.NetDevParams{
+			TapDev: "tap0",
+			MAC:    "52:54:00:12:34:56",
+			MTU:    1500,
+			Queues: 4,
+		},
+	}
+
+	execCmd, err := ch.BuildExecCmd(args, &fakeUnikernel{})
+	assert.NoError(t, err)
+
+	foundNet := false
+	for i, arg := range execCmd {
+		if arg == "--net" && i+1 < len(execCmd) {
+			netVal := execCmd[i+1]
+			assert.Contains(t, netVal, "tap=tap0")
+			assert.Contains(t, netVal, "mac=52:54:00:12:34:56")
+			assert.Contains(t, netVal, "mtu=1500")
+			assert.Contains(t, netVal, "num_queues=4")
+			foundNet = true
+			break
+		}
+	}
+	assert.True(t, foundNet, "--net argument should be present")
+}

pkg/unikontainers/hypervisors/qemu.go

@@ -98,10 +98,13 @@ func (q *Qemu) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel) ([]str
 		if netcli == "" {
 			netcli += " -netdev tap,id=net0,script=no,downscript=no,ifname="
 			netcli += args.Net.TapDev
+			if args.Net.Queues > 1 {
+				netcli += fmt.Sprintf(",queues=%d", args.Net.Queues)
+			}
 			if q.vhost {
 				netcli += ",vhost=on"
 			}
-			netcli += fmt.Sprintf(" %s,host_mtu=%d,mac=%s", getVirtioNetArg(), args.Net.MTU, args.Net.MAC)
+			netcli += fmt.Sprintf(" %s,host_mtu=%d,mac=%s", getVirtioNetArg(args.Net.Queues), args.Net.MTU, args.Net.MAC)
 		}
 		cmdString += netcli
 	} else {
@@ -152,10 +155,15 @@ func (q *Qemu) PreExec(_ types.ExecArgs) error {
 	return nil
 }
 
-func getVirtioNetArg() string {
+func getVirtioNetArg(queues int) string {
 	devType := "virtio-net-pci"
 	if runtime.GOARCH == "arm64" {
 		devType = "virtio-net-device"
 	}
-	return "-device " + devType + ",netdev=net0"
+	arg := "-device " + devType + ",netdev=net0"
+	if queues > 1 {
+		vectors := 2*queues + 2
+		arg += fmt.Sprintf(",mq=on,vectors=%d", vectors)
+	}
+	return arg
 }

pkg/unikontainers/hypervisors/qemu_test.go

@@ -146,6 +146,21 @@ func TestQemuBuildExecCmd(t *testing.T) {
 			},
 			mustNotContain: []string{"-nic none", "vhost=on"},
 		},
+		{
+			name: "multiqueue tap renders queues parameter and mq virtio-net flags",
+			args: types.ExecArgs{
+				UnikernelPath: testKernelPath,
+				Command:       testCommand,
+				Net:           types.NetDevParams{TapDev: "tap0", MAC: "52:54:00:12:34:56", MTU: 1500, Queues: 4},
+			},
+			unikernel: &fakeUnikernel{},
+			mustContain: []string{
+				"-netdev tap",
+				"ifname=tap0,queues=4",
+				"mq=on",
+				"vectors=10",
+			},
+		},
 		{
 			name:  "vhost on emits vhost=on",
 			vhost: true,

pkg/unikontainers/types/types.go

@@ -51,6 +51,7 @@ type NetDevParams struct {
 	MAC     string // The MAC address of the guest network device
 	TapDev  string // The tap device name
 	MTU     int    // The MTU value of the tap device
+	Queues  int    // The queue count of the tap device
 }
 
 type BlockDevParams struct {
@@ -140,4 +141,5 @@ type MonitorConfig struct {
 	BinaryPath      string `toml:"path,omitempty"`      // Optional path to the hypervisor binary
 	DataPath        string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff)
 	Vhost           bool   `toml:"vhost,omitempty"`     // Optional: enable vhost for network performance optimization
+	NetQueues       int    `toml:"net_queues,omitempty"` // Optional: network queue count for multi-queue TAP
 }

pkg/unikontainers/unikontainers.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -255,6 +256,25 @@ func (u *Unikontainer) SetRunningState() error {
 	return u.saveContainerState()
 }
 
+func (u *Unikontainer) getNetworkQueues() int {
+	queuesStr := u.State.Annotations[annotNetQueues]
+	if queuesStr == "" {
+		queuesStr = u.State.Annotations[annotNetQueuesAlt]
+	}
+	if queuesStr != "" {
+		if q, err := strconv.Atoi(queuesStr); err == nil && q > 0 {
+			return q
+		}
+	}
+	if u.UruncCfg != nil {
+		vmmType := u.State.Annotations[annotHypervisor]
+		if monCfg, exists := u.UruncCfg.Monitors[vmmType]; exists && monCfg.NetQueues > 0 {
+			return monCfg.NetQueues
+		}
+	}
+	return 1
+}
+
 func (u *Unikontainer) SetupNet() (types.NetDevParams, error) {
 	networkType := u.getNetworkType()
 	uniklog.WithField("network type", networkType).Debug("Retrieved network type")
@@ -264,7 +284,10 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) {
 		return netArgs, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err)
 	}
 
-	networkInfo, err := netManager.NetworkSetup(u.Spec.Process.User.UID, u.Spec.Process.User.GID)
+	queues := u.getNetworkQueues()
+	netArgs.Queues = queues
+
+	networkInfo, err := netManager.NetworkSetup(u.Spec.Process.User.UID, u.Spec.Process.User.GID, queues)
 	if err != nil {
 		// TODO: Handle this case better. We do not need to show an error
 		// since there was no network in the container. Therefore, we

pkg/unikontainers/urunc_config.go

@@ -145,6 +145,7 @@ func (p *UruncConfig) Map() map[string]string {
 		cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath
 		cfgMap[prefix+"data_path"] = hvCfg.DataPath
 		cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost)
+		cfgMap[prefix+"net_queues"] = strconv.Itoa(hvCfg.NetQueues)
 	}
 	for eb, ebCfg := range p.ExtraBins {
 		prefix := "urunc_config.extra_binaries." + eb + "."
@@ -198,6 +199,10 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig {
 			} else {
 				hvCfg.Vhost = boolVal
 			}
+		case "net_queues":
+			if intVal, err := strconv.Atoi(val); err == nil && intVal > 0 {
+				hvCfg.NetQueues = intVal
+			}
 		}
 		cfg.Monitors[hv] = hvCfg
 	}

pkg/unikontainers/urunc_config_test.go

@@ -60,6 +60,7 @@ func TestUruncConfigFromMap(t *testing.T) {
 			testQemuBinaryKey: testQemuBinaryPath,
 			testQemuDataKey:   testQemuDataPath,
 			testQemuVhostKey:  "true",
+			"urunc_config.monitors.qemu.net_queues": "4",
 		}
 
 		config := UruncConfigFromMap(cfgMap)
@@ -72,6 +73,7 @@ func TestUruncConfigFromMap(t *testing.T) {
 		assert.Equal(t, testQemuBinaryPath, qemuConfig.BinaryPath)
 		assert.Equal(t, testQemuDataPath, qemuConfig.DataPath)
 		assert.True(t, qemuConfig.Vhost)
+		assert.Equal(t, 4, qemuConfig.NetQueues)
 	})
 
 	t.Run("multiple monitors", func(t *testing.T) {