diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index 3ec464f6d9b89c..6b36dc557d0503 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -10,6 +10,7 @@ #ifndef __ASSEMBLY__ +#include #include #include #include @@ -65,6 +66,18 @@ void mk_park_cpu(void); /* Park an offlined pool CPU (host park area, or instance context) */ void mk_pool_park_cpu(void); +struct mk_instance; +struct mk_pci_host_bridge; + +#ifdef CONFIG_MULTIKERNEL +void __init x86_multikernel_pci_platform_init(void); +int mk_arch_snapshot_pci_host_bridges(const struct mk_instance *instance, + struct mk_pci_host_bridge *bridges, + size_t capacity); +#else +static inline void x86_multikernel_pci_platform_init(void) { } +#endif + #endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_MULTIKERNEL_H */ diff --git a/arch/x86/include/asm/pci_x86.h b/arch/x86/include/asm/pci_x86.h index 70533fdcbf02c9..668bfead038643 100644 --- a/arch/x86/include/asm/pci_x86.h +++ b/arch/x86/include/asm/pci_x86.h @@ -190,6 +190,10 @@ extern struct pci_mmcfg_region *__init pci_mmconfig_add(int segment, int start, extern struct list_head pci_mmcfg_list; +typedef int (*pci_mmcfg_region_cb)(const struct pci_mmcfg_region *region, + void *data); +int pci_mmcfg_walk_regions(pci_mmcfg_region_cb callback, void *data); + #define PCI_MMCFG_BUS_OFFSET(bus) ((bus) << 20) /* diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index bd4aa1c37e1c3d..4136eed85f563c 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -133,6 +133,7 @@ void __init x86_early_init_platform_quirks(void) x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; break; case X86_SUBARCH_MULTIKERNEL: + x86_multikernel_pci_platform_init(); x86_platform.legacy.devices.pnpbios = 0; x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; x86_platform.legacy.rtc = 0; @@ -171,7 +172,9 @@ void __init x86_early_init_platform_quirks(void) * the PIT - which belongs to the host - and then request * legacy IRQ0, which can never reach an instance CPU that * has neither a PIC nor an IO-APIC. Ticks come from the - * local APIC timer via setup_percpu_clockev() instead. + * local APIC timer initialized by setup_percpu_clockev(). + * Keeping global_clock_event unset bypasses LAPIC timer + * verification, whose fallback path requires legacy IRQ0. */ x86_init.timers.timer_init = x86_init_noop; x86_init.timers.wallclock_init = x86_init_noop; diff --git a/arch/x86/multikernel/Makefile b/arch/x86/multikernel/Makefile index 331bd895af1f7b..f4d399154f4ce6 100644 --- a/arch/x86/multikernel/Makefile +++ b/arch/x86/multikernel/Makefile @@ -3,4 +3,4 @@ # Makefile for multikernel spawn support # -obj-y += spawn.o direct_boot.o head_64.o +obj-y += spawn.o pci.o direct_boot.o head_64.o diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c new file mode 100644 index 00000000000000..bb4f97fe039c7c --- /dev/null +++ b/arch/x86/multikernel/pci.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * x86 PCI support for multikernel spawn kernels. + * + * This file owns the x86-specific transport of PCI ECAM windows. Generic + * MMCONFIG code only provides an iterator over its region list. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifdef CONFIG_PCI_MMCONFIG +struct mk_mmcfg_snapshot { + const struct mk_instance *instance; + struct mk_pci_host_bridge *bridges; + size_t capacity; + size_t count; +}; + +static struct pci_ops mk_pci_native_ops; +static bool mk_pci_roots_ready; + +static bool mk_mmcfg_snapshot_contains(const struct mk_mmcfg_snapshot *snapshot, + u16 segment, u8 bus) +{ + size_t index; + + for (index = 0; index < snapshot->count; index++) { + if (snapshot->bridges[index].segment == segment && + snapshot->bridges[index].bus_start == bus) + return true; + } + + return false; +} + +static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, + u32 *value) +{ + u32 identity; + u32 mask; + + if (where < PCI_VENDOR_ID || where + size > PCI_COMMAND) + return false; + + identity = vendor | (u32)device << 16; + mask = size == sizeof(identity) ? ~0U : (1U << (size * 8)) - 1; + *value = (identity >> (where * 8)) & mask; + return true; +} + +static int mk_pci_read(struct pci_bus *bus, unsigned int devfn, int where, + int size, u32 *value) +{ + u16 vendor, device; + + if (!mk_pci_get_assigned_identity(bus, devfn, &vendor, &device)) { + *value = ~0U; + return PCIBIOS_DEVICE_NOT_FOUND; + } + if (mk_pci_identity_read(vendor, device, where, size, value)) + return PCIBIOS_SUCCESSFUL; + + return mk_pci_native_ops.read(bus, devfn, where, size, value); +} + +static int mk_pci_write(struct pci_bus *bus, unsigned int devfn, int where, + int size, u32 value) +{ + if (!mk_pci_get_assigned_identity(bus, devfn, NULL, NULL)) + return PCIBIOS_DEVICE_NOT_FOUND; + + return mk_pci_native_ops.write(bus, devfn, where, size, value); +} + +static int mk_mmcfg_snapshot_region(const struct pci_mmcfg_region *region, + void *data) +{ + struct mk_mmcfg_snapshot *snapshot = data; + const struct mk_pci_device *device; + + list_for_each_entry(device, &snapshot->instance->pci_devices, list) { + if (device->domain != region->segment || + device->bus < region->start_bus || + device->bus > region->end_bus) + continue; + if (mk_mmcfg_snapshot_contains(snapshot, device->domain, + device->bus)) + continue; + if (snapshot->count == snapshot->capacity) + return -ENOSPC; + + snapshot->bridges[snapshot->count].segment = region->segment; + snapshot->bridges[snapshot->count].bus_start = device->bus; + snapshot->bridges[snapshot->count].bus_end = device->bus; + snapshot->bridges[snapshot->count].ecam_base = region->address; + pr_info("Multikernel publishing synthetic PCI root %04x:%02x ECAM base %#llx\n", + region->segment, device->bus, + (unsigned long long)region->address); + snapshot->count++; + } + + return 0; +} + +int mk_arch_snapshot_pci_host_bridges(const struct mk_instance *instance, + struct mk_pci_host_bridge *bridges, + size_t capacity) +{ + const struct mk_pci_device *device; + struct mk_mmcfg_snapshot snapshot = { + .instance = instance, + .bridges = bridges, + .capacity = capacity, + }; + int ret; + + if (!instance || !bridges || !capacity || + !instance->pci_devices_valid || !instance->pci_device_count) + return 0; + + ret = pci_mmcfg_walk_regions(mk_mmcfg_snapshot_region, &snapshot); + if (ret) + return ret; + + list_for_each_entry(device, &instance->pci_devices, list) { + if (mk_mmcfg_snapshot_contains(&snapshot, device->domain, + device->bus)) + continue; + pr_err("Multikernel has no ECAM window for assigned PCI bus %04x:%02x\n", + device->domain, device->bus); + return -ENOENT; + } + + return snapshot.count; +} + +static int __init x86_multikernel_pci_arch_init(void) +{ + const struct mk_pci_host_bridge *bridge; + int bridge_count = 0; + + if (!root_instance || !root_instance->pci_host_bridges_valid || + !root_instance->pci_host_bridge_count) { + pr_err("Multikernel has no restored PCI host bridge metadata\n"); + return 0; + } + if (!list_empty(&pci_mmcfg_list)) { + pr_err("Multikernel PCI host bridge list was not empty before restore\n"); + return 0; + } + + list_for_each_entry(bridge, &root_instance->pci_host_bridges, list) { + if (bridge->bus_start != bridge->bus_end) { + pr_err("Multikernel PCI root %04x:[%02x-%02x] exposes shared bridge topology\n", + bridge->segment, bridge->bus_start, + bridge->bus_end); + return 0; + } + bridge_count++; + } + if (bridge_count != root_instance->pci_host_bridge_count) { + pr_err("Multikernel PCI root count mismatch: expected %d, restored %d\n", + root_instance->pci_host_bridge_count, bridge_count); + return 0; + } + + list_for_each_entry(bridge, &root_instance->pci_host_bridges, list) { + if (!pci_mmconfig_add(bridge->segment, bridge->bus_start, + bridge->bus_end, bridge->ecam_base)) { + pr_err("Multikernel failed to register synthetic PCI root %04x:%02x\n", + bridge->segment, bridge->bus_start); + return 0; + } + pr_notice("Multikernel restored synthetic PCI root %04x:%02x ECAM base %#llx\n", + bridge->segment, bridge->bus_start, + (unsigned long long)bridge->ecam_base); + } + if (!pci_mmcfg_arch_init()) { + pr_err("Multikernel failed to map restored PCI ECAM windows\n"); + return 0; + } + + raw_pci_ops = &pci_mmcfg; + raw_pci_ext_ops = &pci_mmcfg; + mk_pci_native_ops = pci_root_ops; + pci_root_ops.read = mk_pci_read; + pci_root_ops.write = mk_pci_write; + mk_pci_roots_ready = true; + pr_notice("Multikernel selected ECAM for PCI config access\n"); + + return 0; +} + +static int __init mk_pci_scan_root(const struct mk_pci_host_bridge *bridge) +{ + struct pci_sysdata *sd; + struct pci_bus *bus; + LIST_HEAD(resources); + + if (bridge->segment && !pci_domains_supported) { + pr_err("Multikernel cannot scan PCI root %04x:%02x without domain support\n", + bridge->segment, bridge->bus_start); + return -EOPNOTSUPP; + } + if (pci_find_bus(bridge->segment, bridge->bus_start)) { + pr_err("Multikernel PCI root %04x:%02x already exists\n", + bridge->segment, bridge->bus_start); + return -EEXIST; + } + + sd = kzalloc(sizeof(*sd), GFP_KERNEL); + if (!sd) + return -ENOMEM; + sd->domain = bridge->segment; + sd->node = x86_pci_root_bus_node(bridge->bus_start); + x86_pci_root_bus_resources(bridge->bus_start, &resources); + bus = pci_scan_root_bus(NULL, bridge->bus_start, &pci_root_ops, sd, + &resources); + if (!bus) { + pci_free_resource_list(&resources); + kfree(sd); + return -ENOMEM; + } + pci_bus_add_devices(bus); + pr_notice("Multikernel scanned synthetic PCI root %04x:%02x\n", + bridge->segment, bridge->bus_start); + return 0; +} + +static int __init x86_multikernel_pci_init(void) +{ + const struct mk_pci_host_bridge *bridge; + int ret; + + if (!root_instance) + panic("Multikernel lost restored instance metadata"); + if (!root_instance->pci_device_count) + return 0; + if (!root_instance->pci_devices_valid || !mk_pci_roots_ready) + panic("Multikernel synthetic PCI roots are unavailable"); + + list_for_each_entry(bridge, &root_instance->pci_host_bridges, list) { + ret = mk_pci_scan_root(bridge); + if (ret) + panic("Multikernel failed to scan synthetic PCI root %04x:%02x: %d", + bridge->segment, bridge->bus_start, ret); + } + + /* Suppress legacy bus 0 probing after every assigned root is present. */ + return 0; +} +#else +int mk_arch_snapshot_pci_host_bridges(const struct mk_instance *instance, + struct mk_pci_host_bridge *bridges, + size_t capacity) +{ + return 0; +} + +static int __init x86_multikernel_pci_arch_init(void) +{ + pr_err("Multikernel PCI requires CONFIG_PCI_MMCONFIG\n"); + return 0; +} + +static int __init x86_multikernel_pci_init(void) +{ + return 0; +} +#endif + +void __init x86_multikernel_pci_platform_init(void) +{ + pci_probe = PCI_PROBE_MMCONF | PCI_PROBE_NOEARLY; + x86_init.pci.arch_init = x86_multikernel_pci_arch_init; + x86_init.pci.init = x86_multikernel_pci_init; +} diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c index 1f45223259204d..c22666789167ca 100644 --- a/arch/x86/pci/mmconfig-shared.c +++ b/arch/x86/pci/mmconfig-shared.c @@ -35,6 +35,25 @@ static DEFINE_MUTEX(pci_mmcfg_lock); LIST_HEAD(pci_mmcfg_list); +int pci_mmcfg_walk_regions(pci_mmcfg_region_cb callback, void *data) +{ + const struct pci_mmcfg_region *region; + int ret = 0; + + if (!callback) + return -EINVAL; + + mutex_lock(&pci_mmcfg_lock); + list_for_each_entry(region, &pci_mmcfg_list, list) { + ret = callback(region, data); + if (ret) + break; + } + mutex_unlock(&pci_mmcfg_lock); + + return ret; +} + static void __init pci_mmconfig_remove(struct pci_mmcfg_region *cfg) { if (cfg->res.parent) diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index eed1f98d556f9a..41183aed8f5d94 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -22,9 +22,6 @@ #include #include #include -#ifdef CONFIG_MULTIKERNEL -#include -#endif #include "pci.h" #define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */ @@ -2625,14 +2622,6 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn) struct pci_dev *dev; u32 l; - /* - * For multikernel spawns, check if we should even probe this location - * BEFORE any config space access. This prevents hardware conflicts - * when the host kernel is also using PCI devices. - */ - if (IS_ENABLED(CONFIG_MULTIKERNEL) && !mk_pci_should_probe(bus, devfn)) - return NULL; - /* * Create pwrctrl device (if required) for the PCI device to handle the * power state. If the pwrctrl device is created, then skip scanning diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index ec99f0ffecf326..7ff2fd2455a771 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -10,10 +10,13 @@ #include #include #include +#include #include #include #include #include +struct pci_bus; +struct pci_ops; /** * Physical CPU identifiers @@ -399,6 +402,12 @@ struct mk_memory_region { * Represents a single PCI device that should be accessible to an instance. * Format: vendor:device@domain:bus:slot.func */ +#define MK_PCI_RESOURCE_COUNT 6 +struct mk_pci_resource { + u64 start; + u64 end; + u64 flags; +}; struct mk_pci_device { char name[64]; /* Device name from DTB (e.g., "enp9s0_dev") */ u16 vendor; /* PCI vendor ID */ @@ -407,9 +416,27 @@ struct mk_pci_device { u8 bus; /* PCI bus number */ u8 slot; /* PCI slot number */ u8 func; /* PCI function number */ + struct mk_pci_resource resources[MK_PCI_RESOURCE_COUNT]; + bool resources_valid; struct list_head list; /* Link to device list */ }; +/** + * PCI host bridge configuration-space descriptor + * + * Describes an ECAM window for a PCI segment and inclusive bus range. This is + * platform metadata shared with an instance, not an assignable device. + */ +struct mk_pci_host_bridge { + u16 segment; + u8 bus_start; + u8 bus_end; + u64 ecam_base; + struct list_head list; +}; + +#define MK_MAX_PCI_HOST_BRIDGES 16 + /** * Platform device specification * @@ -450,13 +477,18 @@ struct mk_dt_config { int pci_device_count; /* Number of PCI devices */ bool pci_devices_valid; /* Whether PCI device list is valid */ + /* PCI host bridge metadata */ + struct list_head pci_host_bridges; + int pci_host_bridge_count; + bool pci_host_bridges_valid; + /* Platform device resources */ struct list_head platform_devices; /* List of struct mk_platform_device */ int platform_device_count; /* Number of platform devices */ bool platform_devices_valid; /* Whether platform device list is valid */ /* Extensibility: Reserved fields for future use */ - u32 reserved[7]; /* Reduced due to added fields */ + u32 reserved[4]; /* Raw device tree data */ void *dtb_data; @@ -473,6 +505,8 @@ struct mk_instance { int id; /* Kernel-assigned instance ID */ char *name; /* User-provided instance name */ enum mk_instance_state state; /* Current state */ + /* Serializes memory topology changes with PCI assignment leases. */ + struct mutex resource_mutex; /* Resource management - list of reserved memory regions */ struct list_head memory_regions; /* List of struct mk_memory_region */ @@ -488,6 +522,13 @@ struct mk_instance { struct list_head pci_devices; /* List of struct mk_pci_device */ int pci_device_count; /* Number of PCI devices */ bool pci_devices_valid; /* Whether PCI device list is valid */ + /* Host-only live PCI assignment leases (private elements). */ + struct list_head pci_assignments; + + /* PCI host bridge metadata (descriptive, never transferred) */ + struct list_head pci_host_bridges; + int pci_host_bridge_count; + bool pci_host_bridges_valid; /* Platform device resources */ struct list_head platform_devices; /* List of struct mk_platform_device */ @@ -687,14 +728,14 @@ void mk_instance_set_state(struct mk_instance *instance, void __noreturn mk_halt_to_pool(void); /** - * mk_instance_reserve_resources() - Reserve CPU and memory resources for instance - * @instance: Instance to reserve resources for - * @config: Device tree configuration with memory size and CPU assignment + * mk_instance_reserve_resources() - Atomically reserve instance resources + * @instance: Empty instance to populate + * @config: Parsed memory, CPU, PCI, and platform resource configuration * - * Allocates the specified memory size from the multikernel pool, creates - * memory regions, and copies CPU assignment. + * Reserves every configured resource class or returns all resources acquired + * by the attempt. A failure never leaves a partially populated instance. * - * Returns 0 on success, negative error code on failure. + * Returns: 0 on success, negative error code on failure */ int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config); @@ -869,17 +910,20 @@ void __init mk_register_cpus_from_kho(void); */ /** - * mk_pci_should_probe() - Check if PCI probing should occur at a location + * mk_pci_get_assigned_identity() - Check and identify an assigned PCI function * @bus: PCI bus * @devfn: PCI device/function number + * @vendor: optional assigned vendor ID output + * @device: optional assigned device ID output * - * Called BEFORE any PCI config space reads to determine if probing - * should proceed. This prevents config space accesses to devices - * that are not in the whitelist, avoiding hardware conflicts on bare metal. + * Synthetic roots make assigned functions directly discoverable. Only exact + * assignment metadata matches may access config space; physical bridges and + * all other functions remain inaccessible. * - * Returns: true if probing should proceed, false to skip entirely + * Returns: true for an exact assignment metadata match, false otherwise */ -bool mk_pci_should_probe(struct pci_bus *bus, int devfn); +bool mk_pci_get_assigned_identity(struct pci_bus *bus, int devfn, + u16 *vendor, u16 *device); /** * mk_platform_device_allowed() - Check if a platform device is allowed by DTB allowlist diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index 1b0191beffa79e..44cf0541c594c5 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -23,4 +23,7 @@ config MULTIKERNEL Requires KEXEC_HANDOVER for preserving device trees and instance state across kernel boundaries. + SR-IOV VF assignment additionally requires PCI_IOV and an active + hardware IOMMU with isolated interrupt delivery. Assignment fails + closed when those isolation facilities are unavailable. If unsure, say N. diff --git a/kernel/multikernel/Makefile b/kernel/multikernel/Makefile index c5c0b15d3adc4c..9439d80f93cdb0 100644 --- a/kernel/multikernel/Makefile +++ b/kernel/multikernel/Makefile @@ -3,7 +3,7 @@ # Makefile for multikernel support # -obj-y += core.o cpuset.o mem.o kernfs.o dts.o kho.o ipi.o messaging.o overlay.o hotplug.o baseline.o +obj-y += core.o cpuset.o mem.o kernfs.o dts.o kho.o pci.o ipi.o messaging.o overlay.o hotplug.o baseline.o # DMA-BUF heap for multikernel memory allocation obj-$(CONFIG_DMABUF_HEAPS) += dma_heap.o diff --git a/kernel/multikernel/baseline.c b/kernel/multikernel/baseline.c index 9de5c9db29af53..c71c1be085da89 100644 --- a/kernel/multikernel/baseline.c +++ b/kernel/multikernel/baseline.c @@ -175,6 +175,9 @@ static void mk_baseline_clear_resources(struct mk_instance *instance) } instance->pci_device_count = 0; instance->pci_devices_valid = false; + mk_pci_host_bridges_free(&instance->pci_host_bridges, + &instance->pci_host_bridge_count, + &instance->pci_host_bridges_valid); list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { list_del(&plat_dev->list); @@ -491,54 +494,54 @@ static int mk_baseline_initialize_devices(const struct mk_instance *instance) { struct mk_pci_device *pci_dev; struct pci_dev *dev; - int failed = 0, unbound = 0; + int failed = 0; + int available = 0; - if (instance->pci_device_count == 0) { - pr_debug("No PCI devices in baseline to unbind\n"); + if (!instance->pci_device_count) { + pr_debug("No PCI devices in the multikernel pool\n"); return 0; } - pr_info("Unbinding %d PCI devices for multikernel pool\n", + pr_info("Validating %d PCI devices for the multikernel pool\n", instance->pci_device_count); + pci_lock_rescan_remove(); list_for_each_entry(pci_dev, &instance->pci_devices, list) { - dev = pci_get_domain_bus_and_slot(pci_dev->domain, pci_dev->bus, - PCI_DEVFN(pci_dev->slot, pci_dev->func)); + dev = pci_get_domain_bus_and_slot(pci_dev->domain, + pci_dev->bus, + PCI_DEVFN(pci_dev->slot, + pci_dev->func)); if (!dev) { pr_warn("PCI device %04x:%04x@%04x:%02x:%02x.%x not found in system\n", - pci_dev->vendor, pci_dev->device, pci_dev->domain, - pci_dev->bus, pci_dev->slot, pci_dev->func); + pci_dev->vendor, pci_dev->device, + pci_dev->domain, pci_dev->bus, + pci_dev->slot, pci_dev->func); failed++; continue; } - if (!dev->driver) { - pr_debug("PCI device %04x:%04x@%04x:%02x:%02x.%x already unbound\n", - pci_dev->vendor, pci_dev->device, pci_dev->domain, - pci_dev->bus, pci_dev->slot, pci_dev->func); - pci_dev_put(dev); - unbound++; - continue; + if (dev->vendor != pci_dev->vendor || + dev->device != pci_dev->device || + pci_dev_is_disconnected(dev) || + !pci_device_is_present(dev)) { + pr_warn("PCI device %04x:%04x@%04x:%02x:%02x.%x is not available\n", + pci_dev->vendor, pci_dev->device, + pci_dev->domain, pci_dev->bus, + pci_dev->slot, pci_dev->func); + failed++; + } else { + available++; } - - const char *driver_name = dev->driver->name; - - device_release_driver(&dev->dev); - - pr_info("Unbound PCI device %04x:%04x@%04x:%02x:%02x.%x (was: %s) for multikernel pool\n", - pci_dev->vendor, pci_dev->device, pci_dev->domain, - pci_dev->bus, pci_dev->slot, pci_dev->func, - driver_name); - pci_dev_put(dev); - unbound++; } + pci_unlock_rescan_remove(); - if (failed > 0) { - pr_warn("Failed to find %d PCI devices in system\n", failed); - } + if (failed) + pr_warn("%d PCI devices in the multikernel pool are unavailable\n", + failed); - pr_info("Successfully unbound %d PCI devices for multikernel pool\n", unbound); + pr_info("Validated %d PCI devices; host drivers remain bound until assignment\n", + available); return 0; } @@ -594,6 +597,15 @@ int mk_baseline_validate_and_initialize(const void *fdt, size_t fdt_size) return ret; } + ret = mk_dt_parse_pci_host_bridges(fdt, resources_node, + &root_instance->pci_host_bridges, + &root_instance->pci_host_bridge_count, + &root_instance->pci_host_bridges_valid); + if (ret) { + pr_err("Failed to parse baseline PCI host bridges: %d\n", ret); + return ret; + } + ret = mk_baseline_parse_devices(fdt, resources_node, root_instance); if (ret) { pr_err("Failed to parse baseline devices: %d\n", ret); diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index ca5096e49073a7..89acb73f5c6578 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -32,16 +32,20 @@ static void mk_instance_return_all_cpus(struct mk_instance *instance) mk_instance_return_cpus(instance, instance->cpus); } -static void mk_instance_return_pci_devices(struct mk_instance *instance) +static int mk_instance_return_pci_devices(struct mk_instance *instance) { struct mk_pci_device *pci_dev, *pci_tmp; int returned_count = 0; + int ret; - if (!instance || !instance->pci_devices_valid) - return; + if (!instance || instance == root_instance || instance->id == 0) + return 0; - if (instance == root_instance || instance->id == 0) - return; + ret = mk_pci_release_assignments(instance); + if (ret) + return ret; + if (!instance->pci_devices_valid) + return 0; if (!root_instance) { pr_warn("Cannot return PCI devices from instance %d (%s): no root instance\n", @@ -49,52 +53,37 @@ static void mk_instance_return_pci_devices(struct mk_instance *instance) goto cleanup; } - list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, list) { - struct mk_pci_device *root_dev; - - root_dev = kzalloc(sizeof(*root_dev), GFP_KERNEL); - if (!root_dev) { - pr_warn("Failed to allocate PCI device entry for root instance\n"); - continue; - } - - *root_dev = *pci_dev; - INIT_LIST_HEAD(&root_dev->list); - - list_add_tail(&root_dev->list, &root_instance->pci_devices); + list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, + list) { + list_move_tail(&pci_dev->list, &root_instance->pci_devices); root_instance->pci_device_count++; root_instance->pci_devices_valid = true; - - pr_debug("Returned PCI device %04x:%02x:%02x.%d from instance %d to root\n", - root_dev->domain, root_dev->bus, root_dev->slot, - root_dev->func, instance->id); - returned_count++; } - if (returned_count > 0) { + if (returned_count) pr_info("Returned %d PCI devices from instance %d (%s) to root instance\n", returned_count, instance->id, instance->name); - } cleanup: - list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, list) { + list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, + list) { list_del(&pci_dev->list); kfree(pci_dev); } instance->pci_device_count = 0; instance->pci_devices_valid = false; + return 0; } - static void mk_instance_return_platform_devices(struct mk_instance *instance) { - struct mk_platform_device *plat_dev, *plat_tmp; - int returned_count = 0; + struct mk_platform_device *device, *tmp; + int returned = 0; - if (!instance || !instance->platform_devices_valid) + if (!instance || instance == root_instance || instance->id == 0) return; - - if (instance == root_instance || instance->id == 0) + if (!instance->platform_devices_valid && + list_empty(&instance->platform_devices)) return; if (!root_instance) { @@ -103,60 +92,63 @@ static void mk_instance_return_platform_devices(struct mk_instance *instance) goto cleanup; } - list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { - struct mk_platform_device *root_dev; - - root_dev = kzalloc(sizeof(*root_dev), GFP_KERNEL); - if (!root_dev) { - pr_warn("Failed to allocate platform device entry for root instance\n"); - continue; - } - - *root_dev = *plat_dev; - INIT_LIST_HEAD(&root_dev->list); - - list_add_tail(&root_dev->list, &root_instance->platform_devices); + list_for_each_entry_safe(device, tmp, &instance->platform_devices, + list) { + list_move_tail(&device->list, + &root_instance->platform_devices); root_instance->platform_device_count++; root_instance->platform_devices_valid = true; - - pr_debug("Returned platform device '%s' from instance %d to root\n", - root_dev->name, instance->id); - - returned_count++; + returned++; } - if (returned_count > 0) { + if (returned) pr_info("Returned %d platform devices from instance %d (%s) to root instance\n", - returned_count, instance->id, instance->name); - } + returned, instance->id, instance->name); cleanup: - list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { - list_del(&plat_dev->list); - kfree(plat_dev); + list_for_each_entry_safe(device, tmp, &instance->platform_devices, + list) { + list_del(&device->list); + kfree(device); } instance->platform_device_count = 0; instance->platform_devices_valid = false; } -static void mk_instance_release(struct kref *kref) +int mk_instance_release_resources(struct mk_instance *instance) { - struct mk_instance *instance = container_of(kref, struct mk_instance, refcount); + int ret; - pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", - instance->id, instance->name); + if (!instance || instance == root_instance || instance->id == 0) + return 0; - mk_instance_return_all_cpus(instance); - mk_instance_return_pci_devices(instance); + ret = mk_instance_return_pci_devices(instance); + if (ret) + return ret; mk_instance_return_platform_devices(instance); + mk_pci_host_bridges_free(&instance->pci_host_bridges, + &instance->pci_host_bridge_count, + &instance->pci_host_bridges_valid); + mk_instance_return_all_cpus(instance); mk_instance_free_memory(instance); + return 0; +} + +static void mk_instance_release(struct kref *kref) +{ + struct mk_instance *instance = + container_of(kref, struct mk_instance, refcount); + int ret; + pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", + instance->id, instance->name); + ret = mk_instance_release_resources(instance); + WARN_ON_ONCE(ret); mk_cpu_set_free(instance->cpus); kfree(instance->dtb_data); kfree(instance->name); kfree(instance); } - /** * Instance reference counting */ @@ -418,9 +410,9 @@ static int mk_instance_reserve_cpus(struct mk_instance *instance, const struct mk_dt_config *config) { if (!config->cpus) { - pr_warn("No CPU configuration for instance %d (%s)\n", - instance->id, instance->name); - return 0; + pr_err("No CPU configuration for instance %d (%s)\n", + instance->id, instance->name); + return -ENOMEM; } return mk_instance_transfer_cpus(instance, config->cpus); @@ -430,91 +422,43 @@ static int mk_instance_transfer_pci_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count) { - struct mk_pci_device *req_dev, *root_dev, *tmp; - int transferred = 0; - int not_found = 0; - bool found; - if (!root_instance || !root_instance->pci_devices_valid) { pr_err("No root instance or PCI devices not initialized\n"); return -EINVAL; } - if (requested_count == 0 || list_empty(requested_devices)) { + if (!requested_count || list_empty(requested_devices)) { pr_info("No PCI devices requested for instance %d (%s)\n", instance->id, instance->name); instance->pci_devices_valid = true; return 0; } - list_for_each_entry(req_dev, requested_devices, list) { - found = false; - list_for_each_entry(root_dev, &root_instance->pci_devices, list) { - if (root_dev->vendor == req_dev->vendor && - root_dev->device == req_dev->device && - root_dev->domain == req_dev->domain && - root_dev->bus == req_dev->bus && - root_dev->slot == req_dev->slot && - root_dev->func == req_dev->func) { - found = true; - break; - } - } - if (!found) { - pr_err("PCI device %04x:%04x@%04x:%02x:%02x.%x not available in root pool\n", - req_dev->vendor, req_dev->device, req_dev->domain, - req_dev->bus, req_dev->slot, req_dev->func); - not_found++; - } - } - - if (not_found > 0) { - pr_err("Instance %d (%s): %d PCI devices not available\n", - instance->id, instance->name, not_found); - return -ENOENT; - } - - list_for_each_entry(req_dev, requested_devices, list) { - list_for_each_entry_safe(root_dev, tmp, &root_instance->pci_devices, list) { - if (root_dev->vendor == req_dev->vendor && - root_dev->device == req_dev->device && - root_dev->domain == req_dev->domain && - root_dev->bus == req_dev->bus && - root_dev->slot == req_dev->slot && - root_dev->func == req_dev->func) { - - list_del(&root_dev->list); - list_add_tail(&root_dev->list, &instance->pci_devices); - root_instance->pci_device_count--; - instance->pci_device_count++; - transferred++; - - pr_debug("Transferred PCI device %04x:%04x@%04x:%02x:%02x.%x to instance %d\n", - root_dev->vendor, root_dev->device, root_dev->domain, - root_dev->bus, root_dev->slot, root_dev->func, - instance->id); - break; - } - } - } - - instance->pci_devices_valid = true; - pr_info("Transferred %d PCI devices from root to instance %d (%s), root pool remaining: %d devices\n", - transferred, instance->id, instance->name, root_instance->pci_device_count); - - return 0; + return mk_pci_assign_devices(instance, requested_devices, + requested_count); } - static int mk_instance_reserve_pci_devices(struct mk_instance *instance, const struct mk_dt_config *config) { - if (!config->pci_devices_valid || config->pci_device_count == 0) { + if (!config->pci_devices_valid) { + if (config->pci_device_count || + !list_empty(&config->pci_devices)) + return -EINVAL; + instance->pci_devices_valid = true; + return 0; + } + + if (!config->pci_device_count) { + if (!list_empty(&config->pci_devices)) + return -EINVAL; instance->pci_devices_valid = true; instance->pci_device_count = 0; pr_debug("No PCI devices to reserve for instance %d (%s)\n", instance->id, instance->name); return 0; } + if (list_empty(&config->pci_devices)) + return -EINVAL; return mk_instance_transfer_pci_devices(instance, &config->pci_devices, @@ -522,86 +466,111 @@ static int mk_instance_reserve_pci_devices(struct mk_instance *instance, } static int mk_instance_transfer_platform_devices(struct mk_instance *instance, - const struct list_head *requested_devices, - int requested_count) + const struct list_head *requested_devices, + int requested_count) { - struct mk_platform_device *req_dev, *root_dev, *tmp; + struct mk_platform_device *requested, *other, *root_device; + int actual_count = 0; int transferred = 0; - int not_found = 0; - bool found; if (!root_instance || !root_instance->platform_devices_valid) { pr_err("No root instance or platform devices not initialized\n"); return -EINVAL; } + if (requested_count <= 0 || list_empty(requested_devices)) + return -EINVAL; - if (requested_count == 0 || list_empty(requested_devices)) { - pr_info("No platform devices requested for instance %d (%s)\n", - instance->id, instance->name); - instance->platform_devices_valid = true; - return 0; - } + list_for_each_entry(requested, requested_devices, list) { + actual_count++; + list_for_each_entry(other, requested_devices, list) { + if (other == requested) + break; + if (!strcmp(other->name, requested->name)) { + pr_err("Platform device %s is requested more than once\n", + requested->name); + return -EINVAL; + } + } - list_for_each_entry(req_dev, requested_devices, list) { - found = false; - list_for_each_entry(root_dev, &root_instance->platform_devices, list) { - if (strcmp(root_dev->name, req_dev->name) == 0) { - found = true; + root_device = NULL; + list_for_each_entry(other, &root_instance->platform_devices, + list) { + if (!strcmp(other->name, requested->name)) { + root_device = other; break; } } - if (!found) { - pr_err("Platform device '%s' not available in root pool\n", - req_dev->name); - not_found++; + if (!root_device) { + pr_err("Platform device %s not available in root pool\n", + requested->name); + return -ENOENT; } } - if (not_found > 0) { - pr_err("Instance %d (%s): %d platform devices not available\n", - instance->id, instance->name, not_found); - return -ENOENT; + if (actual_count != requested_count) { + pr_err("Platform device count mismatch: metadata=%d list=%d\n", + requested_count, actual_count); + return -EINVAL; } - list_for_each_entry(req_dev, requested_devices, list) { - list_for_each_entry_safe(root_dev, tmp, &root_instance->platform_devices, list) { - if (strcmp(root_dev->name, req_dev->name) == 0) { - list_del(&root_dev->list); - list_add_tail(&root_dev->list, &instance->platform_devices); - root_instance->platform_device_count--; - instance->platform_device_count++; - transferred++; - - pr_debug("Transferred platform device '%s' to instance %d\n", - root_dev->name, instance->id); + list_for_each_entry(requested, requested_devices, list) { + root_device = NULL; + list_for_each_entry(other, &root_instance->platform_devices, + list) { + if (!strcmp(other->name, requested->name)) { + root_device = other; break; } } + if (!root_device) + goto rollback; + + list_move_tail(&root_device->list, + &instance->platform_devices); + root_instance->platform_device_count--; + instance->platform_device_count++; + transferred++; } instance->platform_devices_valid = true; - pr_info("Transferred %d platform devices from root to instance %d (%s), root pool remaining: %d devices\n", - transferred, instance->id, instance->name, root_instance->platform_device_count); - + pr_info("Transferred %d platform devices from root to instance %d (%s)\n", + transferred, instance->id, instance->name); return 0; + +rollback: + pr_err("Platform inventory changed during reservation for instance %d\n", + instance->id); + mk_instance_return_platform_devices(instance); + return -EIO; } static int mk_instance_reserve_platform_devices(struct mk_instance *instance, - const struct mk_dt_config *config) + const struct mk_dt_config *config) { - if (!config->platform_devices_valid || config->platform_device_count == 0) { + if (!config->platform_devices_valid) { + if (config->platform_device_count || + !list_empty(&config->platform_devices)) + return -EINVAL; + instance->platform_devices_valid = true; + return 0; + } + + if (!config->platform_device_count) { + if (!list_empty(&config->platform_devices)) + return -EINVAL; instance->platform_devices_valid = true; instance->platform_device_count = 0; pr_debug("No platform devices to reserve for instance %d (%s)\n", instance->id, instance->name); return 0; } + if (list_empty(&config->platform_devices)) + return -EINVAL; return mk_instance_transfer_platform_devices(instance, &config->platform_devices, config->platform_device_count); } - /** * mk_instance_add_pci_device - Add a single PCI device to an instance * @instance: Target instance @@ -617,37 +586,14 @@ static int mk_instance_reserve_platform_devices(struct mk_instance *instance, int mk_instance_add_pci_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { - struct mk_pci_device *root_dev, *tmp; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - - if (!root_instance || !root_instance->pci_devices_valid) { - pr_err("No root instance or PCI devices not initialized\n"); - return -EINVAL; - } - - list_for_each_entry_safe(root_dev, tmp, &root_instance->pci_devices, list) { - if (root_dev->domain == domain && - root_dev->bus == bus && - root_dev->slot == slot && - root_dev->func == func) { - - list_del(&root_dev->list); - list_add_tail(&root_dev->list, &instance->pci_devices); - root_instance->pci_device_count--; - instance->pci_device_count++; - instance->pci_devices_valid = true; - - pr_info("Transferred PCI device %04x:%04x@%04x:%02x:%02x.%x to instance %d\n", - root_dev->vendor, root_dev->device, domain, bus, slot, func, - instance->id); - return 0; - } - } + int ret; - pr_err("PCI device %04x:%02x:%02x.%x not found in root pool\n", - domain, bus, slot, func); - return -ENOENT; + ret = mk_pci_assign_device(instance, domain, bus, devfn); + if (!ret) + pr_info("Leased PCI VF %04x:%02x:%02x.%x to instance %d\n", + domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), + instance->id); + return ret; } /** @@ -658,63 +604,22 @@ int mk_instance_add_pci_device(struct mk_instance *instance, * @devfn: PCI device and function (combined) * * Returns a single PCI device from the specified instance back to root instance. - * Used for dynamic PCI device hotplug from non-running instances. + * Dynamic assignment changes are accepted only while the instance is ready. * * Returns: 0 on success, negative error code on failure */ int mk_instance_remove_pci_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { - struct mk_pci_device *inst_dev, *tmp; - struct mk_pci_device *root_dev; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - - if (!instance->pci_devices_valid) { - pr_err("Instance %d PCI devices not initialized\n", instance->id); - return -EINVAL; - } - - if (!root_instance) { - pr_err("Cannot return PCI device: no root instance\n"); - return -EINVAL; - } - - list_for_each_entry_safe(inst_dev, tmp, &instance->pci_devices, list) { - if (inst_dev->domain == domain && - inst_dev->bus == bus && - inst_dev->slot == slot && - inst_dev->func == func) { - - root_dev = kzalloc(sizeof(*root_dev), GFP_KERNEL); - if (!root_dev) { - pr_err("Failed to allocate PCI device entry for root instance\n"); - return -ENOMEM; - } - - *root_dev = *inst_dev; - INIT_LIST_HEAD(&root_dev->list); - - list_add_tail(&root_dev->list, &root_instance->pci_devices); - root_instance->pci_device_count++; - root_instance->pci_devices_valid = true; - - list_del(&inst_dev->list); - kfree(inst_dev); - instance->pci_device_count--; - - pr_info("Returned PCI device %04x:%04x@%04x:%02x:%02x.%x from instance %d to root\n", - root_dev->vendor, root_dev->device, domain, bus, slot, func, - instance->id); - return 0; - } - } + int ret; - pr_err("PCI device %04x:%02x:%02x.%x not found in instance %d\n", - domain, bus, slot, func, instance->id); - return -ENOENT; + ret = mk_pci_unassign_device(instance, domain, bus, devfn); + if (!ret) + pr_info("Released PCI VF %04x:%02x:%02x.%x from instance %d\n", + domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), + instance->id); + return ret; } - /** * Memory management functions for instances */ @@ -905,67 +810,106 @@ void mk_instance_free_memory(struct mk_instance *instance) instance->id, instance->name); } +static bool mk_instance_resources_empty(const struct mk_instance *instance) +{ + return list_empty(&instance->memory_regions) && + !instance->instance_pool && !instance->region_count && + mk_cpu_set_empty(instance->cpus) && + list_empty(&instance->pci_devices) && + list_empty(&instance->pci_assignments) && + !instance->pci_device_count && + list_empty(&instance->pci_host_bridges) && + !instance->pci_host_bridge_count && + list_empty(&instance->platform_devices) && + !instance->platform_device_count; +} + /** - * mk_instance_reserve_resources() - Reserve memory and CPU resources for an instance + * mk_instance_reserve_resources() - Atomically reserve instance resources * @instance: Instance to reserve resources for - * @config: Device tree configuration with memory regions and CPU assignment + * @config: Parsed resource configuration * - * Reserves all memory regions specified in the device tree configuration, - * makes them children of the main multikernel_res, and copies CPU assignment. + * Each resource class is acquired only after the preceding class succeeds. + * Any error returns every acquired resource to the root instance in reverse + * order, so callers never observe a partially populated instance. * - * Returns 0 on success, negative error code on failure. + * Returns: 0 on success, negative error code on failure */ int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config) { + const char *failed_resource; + int release_ret; int ret; - if (!config || !instance) { + if (!config || !instance || !instance->cpus) { pr_err("Invalid parameters to mk_instance_reserve_resources\n"); return -EINVAL; } + if (!mk_instance_resources_empty(instance)) { + pr_err("Instance %d (%s) already owns resources\n", + instance->id, instance->name); + return -EBUSY; + } - /* Free any existing memory regions first */ - mk_instance_free_memory(instance); - - /* Reserve memory regions */ + failed_resource = "memory"; ret = mk_instance_reserve_memory(instance, config); - if (ret) { - pr_err("Failed to reserve memory regions for instance %d (%s): %d\n", - instance->id, instance->name, ret); - return ret; - } + if (ret) + goto rollback; - /* Reserve CPU resources */ + failed_resource = "CPU"; ret = mk_instance_reserve_cpus(instance, config); - if (ret) { - pr_err("Failed to reserve CPU resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for CPU reservation failure */ - pr_warn("Continuing without CPU assignment\n"); - } - - /* Reserve PCI device resources */ - ret = mk_instance_reserve_pci_devices(instance, config); - if (ret) { - pr_err("Failed to reserve PCI device resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for PCI reservation failure */ - pr_warn("Continuing without PCI device assignment\n"); + if (ret) + goto rollback; + + failed_resource = "PCI host bridge metadata"; + if ((!config->pci_host_bridges_valid && + (config->pci_host_bridge_count || + !list_empty(&config->pci_host_bridges))) || + (config->pci_host_bridges_valid && + !config->pci_host_bridge_count && + !list_empty(&config->pci_host_bridges))) { + ret = -EINVAL; + } else if (config->pci_host_bridges_valid && + config->pci_host_bridge_count > 0) { + ret = mk_pci_host_bridges_clone(&instance->pci_host_bridges, + &instance->pci_host_bridge_count, + &instance->pci_host_bridges_valid, + &config->pci_host_bridges, + config->pci_host_bridge_count, true); + } else if (root_instance) { + ret = mk_pci_host_bridges_clone(&instance->pci_host_bridges, + &instance->pci_host_bridge_count, + &instance->pci_host_bridges_valid, + &root_instance->pci_host_bridges, + root_instance->pci_host_bridge_count, + root_instance->pci_host_bridges_valid); + } else { + ret = -EINVAL; } + if (ret) + goto rollback; - /* Reserve platform device resources */ + failed_resource = "platform device"; ret = mk_instance_reserve_platform_devices(instance, config); - if (ret) { - pr_err("Failed to reserve platform device resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for platform reservation failure */ - pr_warn("Continuing without platform device assignment\n"); - } + if (ret) + goto rollback; + + failed_resource = "PCI device"; + ret = mk_instance_reserve_pci_devices(instance, config); + if (ret) + goto rollback; return 0; -} +rollback: + pr_err("Failed to reserve %s resources for instance %d (%s): %d\n", + failed_resource, instance->id, instance->name, ret); + release_ret = mk_instance_release_resources(instance); + if (release_ret) + return release_ret; + return ret; +} /** * Per-instance memory pool management */ @@ -1327,8 +1271,8 @@ int multikernel_halt_by_id(int mk_id) } /** - * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI - * @mk_id: Instance ID to halt + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop * * Forces a spawn kernel's CPUs to stop by queuing a shutdown message in the * IPI ring buffer and sending NMIs directly to each CPU. The NMI handler @@ -1339,41 +1283,35 @@ int multikernel_halt_by_id(int mk_id) * * Returns: 0 on success, negative error code on failure */ -int multikernel_force_halt_by_id(int mk_id) +int mk_instance_force_halt(struct mk_instance *instance) { - struct mk_instance *instance; struct mk_shutdown_payload payload; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); if (!instance) - return -ENOENT; - + return -EINVAL; if (instance->state != MK_STATE_ACTIVE) { pr_err("Instance %d not active (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } if (mk_cpu_set_empty(instance->cpus)) { - pr_err("Instance %d has no CPUs assigned\n", mk_id); - mk_instance_put(instance); + pr_err("Instance %d has no CPUs assigned\n", instance->id); return -EINVAL; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); - - /* Queue shutdown message - NMI handler will check for this */ + pr_info("Force halting multikernel instance %d via NMI\n", instance->id); payload.flags = MK_SHUTDOWN_IMMEDIATE; - payload.sender_instance_id = root_instance->id; - ret = mk_send_message(mk_id, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, + payload.sender_instance_id = root_instance ? root_instance->id : 0; + ret = mk_send_message(instance->id, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, &payload, sizeof(payload)); if (ret < 0) - pr_err("Failed to queue shutdown message: %d (sending NMI anyway)\n", ret); + pr_err("Failed to queue shutdown message: %d (sending NMI anyway)\n", + ret); /* Send NMI to each CPU in the instance */ mk_cpu_set_for_each(i, phys_cpu, instance->cpus) { @@ -1381,17 +1319,34 @@ int multikernel_force_halt_by_id(int mk_id) cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); - + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); mk_instance_set_state(instance, MK_STATE_LOADED); - mk_instance_put(instance); return 0; } +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + ret = mk_instance_force_halt(instance); + mk_instance_put(instance); + return ret; +} static int __init multikernel_init(void) { int ret; + ret = mk_pci_lease_system_init(); + if (ret) { + pr_err("Failed to initialize PCI assignment leases: %d\n", ret); + return ret; + } + /* Register NMI handler for forcible shutdown */ ret = mk_register_stop_nmi_handler(); if (ret < 0) { @@ -1402,6 +1357,7 @@ static int __init multikernel_init(void) ret = mk_messaging_init(); if (ret < 0) { pr_err("Failed to initialize multikernel messaging: %d\n", ret); + mk_pci_lease_system_cleanup(); return ret; } @@ -1409,6 +1365,7 @@ static int __init multikernel_init(void) if (ret < 0) { pr_err("Failed to register system message handler: %d\n", ret); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } @@ -1417,6 +1374,7 @@ static int __init multikernel_init(void) pr_err("Failed to initialize multikernel hotplug: %d\n", ret); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } @@ -1426,6 +1384,7 @@ static int __init multikernel_init(void) mk_hotplug_cleanup(); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } diff --git a/kernel/multikernel/dts.c b/kernel/multikernel/dts.c index ebe650d9119769..a09e59ff82658a 100644 --- a/kernel/multikernel/dts.c +++ b/kernel/multikernel/dts.c @@ -17,13 +17,23 @@ #include #include #include +#include #include #include #include #include +#include #include "internal.h" +int __weak +mk_arch_snapshot_pci_host_bridges(const struct mk_instance *instance, + struct mk_pci_host_bridge *bridges, + size_t capacity) +{ + return 0; +} + static const void *mk_dt_get_base_fdt(void) { if (!root_instance || !root_instance->dtb_data) { @@ -55,6 +65,9 @@ void mk_dt_config_init(struct mk_dt_config *config) INIT_LIST_HEAD(&config->pci_devices); config->pci_device_count = 0; config->pci_devices_valid = true; + INIT_LIST_HEAD(&config->pci_host_bridges); + config->pci_host_bridge_count = 0; + config->pci_host_bridges_valid = false; INIT_LIST_HEAD(&config->platform_devices); config->platform_device_count = 0; @@ -64,6 +77,7 @@ void mk_dt_config_init(struct mk_dt_config *config) void mk_dt_config_free(struct mk_dt_config *config) { struct mk_pci_device *pci_dev, *tmp_pci; + struct mk_pci_host_bridge *host_bridge, *tmp_bridge; struct mk_platform_device *plat_dev, *tmp_plat; if (!config) @@ -82,6 +96,16 @@ void mk_dt_config_free(struct mk_dt_config *config) config->pci_devices_valid = false; } + if (config->pci_host_bridges_valid) { + list_for_each_entry_safe(host_bridge, tmp_bridge, + &config->pci_host_bridges, list) { + list_del(&host_bridge->list); + kfree(host_bridge); + } + config->pci_host_bridge_count = 0; + config->pci_host_bridges_valid = false; + } + /* Free platform device list */ if (config->platform_devices_valid) { list_for_each_entry_safe(plat_dev, tmp_plat, &config->platform_devices, list) { @@ -98,6 +122,138 @@ void mk_dt_config_free(struct mk_dt_config *config) /* Note: We don't free dtb_data here as it's managed by the caller */ } +void mk_pci_host_bridges_free(struct list_head *bridges, int *count, + bool *valid) +{ + struct mk_pci_host_bridge *bridge, *tmp; + + list_for_each_entry_safe(bridge, tmp, bridges, list) { + list_del(&bridge->list); + kfree(bridge); + } + *count = 0; + *valid = false; +} + +int mk_pci_host_bridges_clone(struct list_head *dst, int *dst_count, + bool *dst_valid, const struct list_head *src, + int src_count, bool src_valid) +{ + const struct mk_pci_host_bridge *src_bridge; + struct mk_pci_host_bridge *dst_bridge; + int ret = -EINVAL; + + mk_pci_host_bridges_free(dst, dst_count, dst_valid); + if (src_count < 0) + return -EINVAL; + if (!src_valid) { + if (src_count || !list_empty(src)) + return -EINVAL; + return 0; + } + if (!src_count) { + if (!list_empty(src)) + return -EINVAL; + return 0; + } + if (list_empty(src)) + return -EINVAL; + + *dst_valid = true; + list_for_each_entry(src_bridge, src, list) { + dst_bridge = kmemdup(src_bridge, sizeof(*dst_bridge), + GFP_KERNEL); + if (!dst_bridge) { + ret = -ENOMEM; + goto error; + } + INIT_LIST_HEAD(&dst_bridge->list); + list_add_tail(&dst_bridge->list, dst); + (*dst_count)++; + } + + if (*dst_count != src_count) + goto error; + return 0; + +error: + mk_pci_host_bridges_free(dst, dst_count, dst_valid); + return ret; +} +int mk_dt_parse_pci_host_bridges(const void *fdt, int resources_node, + struct list_head *bridges, int *count, + bool *valid) +{ + struct mk_pci_host_bridge *bridge, *existing; + const fdt32_t *segment_prop, *bus_range; + const fdt64_t *ecam_prop; + int bridges_node, bridge_node, len, ret = -EINVAL; + u32 segment, bus_start, bus_end; + u64 ecam_base; + + bridges_node = fdt_subnode_offset(fdt, resources_node, + "pci-host-bridges"); + if (bridges_node < 0) + return 0; + + *valid = true; + fdt_for_each_subnode(bridge_node, fdt, bridges_node) { + segment_prop = fdt_getprop(fdt, bridge_node, "segment", &len); + if (!segment_prop || len != sizeof(*segment_prop)) + goto invalid; + segment = fdt32_to_cpu(*segment_prop); + + bus_range = fdt_getprop(fdt, bridge_node, "bus-range", &len); + if (!bus_range || len != 2 * sizeof(*bus_range)) + goto invalid; + bus_start = fdt32_to_cpu(bus_range[0]); + bus_end = fdt32_to_cpu(bus_range[1]); + + ecam_prop = fdt_getprop(fdt, bridge_node, "ecam-base", &len); + if (!ecam_prop || len != sizeof(*ecam_prop)) + goto invalid; + ecam_base = fdt64_to_cpu(*ecam_prop); + + if (segment > U16_MAX || bus_start > U8_MAX || bus_end > U8_MAX || + bus_start > bus_end || !ecam_base || !IS_ALIGNED(ecam_base, SZ_1M)) + goto invalid; + + list_for_each_entry(existing, bridges, list) { + if (existing->segment == segment && + bus_start <= existing->bus_end && + bus_end >= existing->bus_start) { + pr_err("Overlapping PCI host bridge bus ranges in segment %04x\n", + segment); + goto error; + } + } + + bridge = kzalloc(sizeof(*bridge), GFP_KERNEL); + if (!bridge) { + ret = -ENOMEM; + goto error; + } + bridge->segment = segment; + bridge->bus_start = bus_start; + bridge->bus_end = bus_end; + bridge->ecam_base = ecam_base; + list_add_tail(&bridge->list, bridges); + (*count)++; + pr_info("Added PCI host bridge: segment %04x [bus %02x-%02x] ECAM %#llx\n", + bridge->segment, bridge->bus_start, bridge->bus_end, + (unsigned long long)bridge->ecam_base); + } + + return 0; + +invalid: + pr_err("Invalid PCI host bridge metadata in node '%s'\n", + fdt_get_name(fdt, bridge_node, NULL)); +error: + mk_pci_host_bridges_free(bridges, count, valid); + return ret; +} + /** * Function prototypes */ @@ -209,10 +365,11 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, { const char *pci_id_str; const fdt32_t *vendor_prop, *device_prop; + const fdt64_t *resources_prop; struct mk_pci_device *pci_dev; unsigned int domain, bus, slot, func; const char *node_name; - int len; + int len, i; node_name = fdt_get_name(source_fdt, dev_node, NULL); @@ -249,12 +406,38 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, return -ENOMEM; } + strscpy(pci_dev->name, device_name, sizeof(pci_dev->name)); pci_dev->vendor = (u16)fdt32_to_cpu(*vendor_prop); pci_dev->device = (u16)fdt32_to_cpu(*device_prop); pci_dev->domain = (u16)domain; pci_dev->bus = (u8)bus; pci_dev->slot = (u8)slot; pci_dev->func = (u8)func; + resources_prop = fdt_getprop(source_fdt, dev_node, "bar-resources", &len); + if (resources_prop) { + if (len != MK_PCI_RESOURCE_COUNT * 3 * sizeof(*resources_prop)) { + pr_err("Invalid bar-resources in device '%s'\n", device_name); + kfree(pci_dev); + return -EINVAL; + } + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + u64 start = fdt64_to_cpu(resources_prop[i * 3]); + u64 end = fdt64_to_cpu(resources_prop[i * 3 + 1]); + u64 flags = fdt64_to_cpu(resources_prop[i * 3 + 2]); + + if ((start || end) && + (end < start || !(flags & (IORESOURCE_IO | IORESOURCE_MEM)))) { + pr_err("Invalid PCI BAR %d range in device '%s'\n", + i, device_name); + kfree(pci_dev); + return -EINVAL; + } + pci_dev->resources[i].start = start; + pci_dev->resources[i].end = end; + pci_dev->resources[i].flags = flags; + } + pci_dev->resources_valid = true; + } list_add_tail(&pci_dev->list, &config->pci_devices); config->pci_device_count++; @@ -558,6 +741,16 @@ int mk_dt_parse(const void *dtb_data, size_t dtb_size, return ret; } + ret = mk_dt_parse_pci_host_bridges(fdt, resources_node, + &config->pci_host_bridges, + &config->pci_host_bridge_count, + &config->pci_host_bridges_valid); + if (ret) { + pr_err("Failed to parse PCI host bridge metadata: %d\n", ret); + mk_dt_config_free(config); + return ret; + } + ret = mk_dt_parse_devices(fdt, resources_node, config); if (ret) { pr_err("Failed to parse device resources: %d\n", ret); @@ -565,9 +758,10 @@ int mk_dt_parse(const void *dtb_data, size_t dtb_size, return ret; } - pr_info("Successfully parsed multikernel device tree with %zu bytes memory, %u CPUs, %d PCI devices, and %d platform devices\n", + pr_info("Successfully parsed multikernel device tree with %zu bytes memory, %d CPUs, %d PCI host bridges, %d PCI devices, and %d platform devices\n", config->memory_size, mk_cpu_set_count(config->cpus), - config->pci_device_count, config->platform_device_count); + config->pci_host_bridge_count, config->pci_device_count, + config->platform_device_count); return 0; } @@ -608,6 +802,17 @@ int mk_dt_parse_resources(const void *fdt, int resources_node, return ret; } + ret = mk_dt_parse_pci_host_bridges(fdt, resources_node, + &config->pci_host_bridges, + &config->pci_host_bridge_count, + &config->pci_host_bridges_valid); + if (ret) { + pr_err("Failed to parse PCI host bridge metadata for '%s': %d\n", + instance_name, ret); + mk_dt_config_free(config); + return ret; + } + ret = mk_dt_parse_devices(fdt, resources_node, config); if (ret) { pr_err("Failed to parse device resources for '%s': %d\n", instance_name, ret); @@ -615,10 +820,11 @@ int mk_dt_parse_resources(const void *fdt, int resources_node, return ret; } - pr_info("Successfully parsed instance '%s': %zu bytes memory, %u CPUs, %d PCI devices, %d platform devices\n", + pr_info("Successfully parsed instance '%s': %zu bytes memory, %d CPUs, %d PCI host bridges, %d PCI devices, %d platform devices\n", instance_name, config->memory_size, mk_cpu_set_count(config->cpus), - config->pci_device_count, config->platform_device_count); + config->pci_host_bridge_count, config->pci_device_count, + config->platform_device_count); return 0; } @@ -817,6 +1023,7 @@ int mk_dt_get_property_size(const void *dtb_data, size_t dtb_size, void mk_dt_print_config(const struct mk_dt_config *config) { struct mk_pci_device *pci_dev; + struct mk_pci_host_bridge *host_bridge; struct mk_platform_device *plat_dev; if (!config) { @@ -847,6 +1054,17 @@ void mk_dt_print_config(const struct mk_dt_config *config) pr_info(" CPU assignment: unavailable (allocation failed)\n"); } + if (config->pci_host_bridges_valid) { + pr_info(" PCI host bridges: %d\n", config->pci_host_bridge_count); + list_for_each_entry(host_bridge, &config->pci_host_bridges, list) + pr_info(" - segment %04x [bus %02x-%02x] ECAM %#llx\n", + host_bridge->segment, host_bridge->bus_start, + host_bridge->bus_end, + (unsigned long long)host_bridge->ecam_base); + } else { + pr_info(" PCI host bridges: none specified\n"); + } + if (config->pci_devices_valid) { if (config->pci_device_count == 0) { pr_info(" PCI devices: none specified\n"); @@ -881,6 +1099,73 @@ void mk_dt_print_config(const struct mk_dt_config *config) pr_info(" DTB: %zu bytes\n", config->dtb_size); } +static int mk_dt_emit_pci_host_bridge(void *fdt, + const struct mk_pci_host_bridge *bridge) +{ + char node_name[32]; + fdt32_t bus_range[2]; + int ret; + + snprintf(node_name, sizeof(node_name), "host@%04x,%02x", + bridge->segment, bridge->bus_start); + ret = fdt_begin_node(fdt, node_name); + if (ret) + return ret; + ret = fdt_property_u32(fdt, "segment", bridge->segment); + if (ret) + return ret; + bus_range[0] = cpu_to_fdt32(bridge->bus_start); + bus_range[1] = cpu_to_fdt32(bridge->bus_end); + ret = fdt_property(fdt, "bus-range", bus_range, sizeof(bus_range)); + if (ret) + return ret; + ret = fdt_property_u64(fdt, "ecam-base", bridge->ecam_base); + if (ret) + return ret; + return fdt_end_node(fdt); +} + +static int mk_dt_emit_pci_host_bridges(void *fdt, + const struct mk_instance *instance) +{ + struct mk_pci_host_bridge discovered[MK_MAX_PCI_HOST_BRIDGES]; + const struct mk_pci_host_bridge *bridge; + int discovered_count, index, ret; + + if (!instance->pci_devices_valid) + return -EINVAL; + if (!instance->pci_device_count) + return 0; + + discovered_count = + mk_arch_snapshot_pci_host_bridges(instance, discovered, + ARRAY_SIZE(discovered)); + if (discovered_count < 0) + return discovered_count; + if (!discovered_count && + (!instance->pci_host_bridges_valid || + !instance->pci_host_bridge_count)) + return 0; + + ret = fdt_begin_node(fdt, "pci-host-bridges"); + if (ret) + return ret; + if (discovered_count) { + for (index = 0; index < discovered_count; index++) { + ret = mk_dt_emit_pci_host_bridge(fdt, &discovered[index]); + if (ret) + return ret; + } + } else { + list_for_each_entry(bridge, &instance->pci_host_bridges, list) { + ret = mk_dt_emit_pci_host_bridge(fdt, bridge); + if (ret) + return ret; + } + } + return fdt_end_node(fdt); +} + /** * mk_dt_generate_instance_dtb() - Generate instance DTB from kernel data structures * @instance: Instance with transferred resources (CPUs, memory, devices) @@ -974,6 +1259,10 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, if (ret) goto err_free; } + ret = mk_dt_emit_pci_host_bridges(fdt, instance); + if (ret) + goto err_free; + if ((instance->pci_devices_valid && instance->pci_device_count > 0) || (instance->platform_devices_valid && instance->platform_device_count > 0)) { ret = fdt_begin_node(fdt, "devices"); @@ -985,6 +1274,43 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, list_for_each_entry(pci_dev, &instance->pci_devices, list) { char node_name[64]; char pci_id_str[32]; + fdt64_t resources[MK_PCI_RESOURCE_COUNT * 3]; + struct pci_dev *live_dev = NULL; + unsigned int devfn; + int i; + + if (!pci_dev->resources_valid) { + devfn = PCI_DEVFN(pci_dev->slot, + pci_dev->func); + live_dev = + pci_get_domain_bus_and_slot(pci_dev->domain, + pci_dev->bus, devfn); + if (!live_dev) { + pr_err("PCI device %04x:%02x:%02x.%x disappeared before resource snapshot\n", + pci_dev->domain, pci_dev->bus, + pci_dev->slot, pci_dev->func); + ret = -ENODEV; + goto err_free; + } + } + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + u64 start, end, flags; + + if (live_dev) { + start = pci_resource_start(live_dev, i); + end = pci_resource_end(live_dev, i); + flags = pci_resource_flags(live_dev, i); + } else { + start = pci_dev->resources[i].start; + end = pci_dev->resources[i].end; + flags = pci_dev->resources[i].flags; + } + resources[i * 3] = cpu_to_fdt64(start); + resources[i * 3 + 1] = cpu_to_fdt64(end); + resources[i * 3 + 2] = cpu_to_fdt64(flags); + } + if (live_dev) + pci_dev_put(live_dev); snprintf(node_name, sizeof(node_name), "%s", pci_dev->name[0] ? pci_dev->name : "unnamed_pci"); @@ -1006,6 +1332,11 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, ret = fdt_property_u32(fdt, "device-id", pci_dev->device); if (ret) goto err_free; + ret = fdt_property(fdt, "bar-resources", resources, + sizeof(resources)); + if (ret) + goto err_free; + ret = fdt_end_node(fdt); if (ret) goto err_free; } diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 40068f45c0a67f..ecd1333de5de2d 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -1164,6 +1164,21 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl return ret; } +static int mk_memory_change_allowed(struct mk_instance *instance) +{ + bool iommu_active; + + mutex_lock(&instance->resource_mutex); + iommu_active = mk_pci_iommu_lease_active_locked(instance); + mutex_unlock(&instance->resource_mutex); + if (!iommu_active) + return 0; + + pr_err("Cannot change memory for instance %d while an IOMMU lease is active\n", + instance->id); + return -EBUSY; +} + /** * mk_send_mem_add - Add memory to instance * @instance_id: Target instance ID @@ -1200,6 +1215,11 @@ int mk_send_mem_add(int instance_id, u64 start_pfn, u64 nr_pages, target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; + ret = mk_memory_change_allowed(target_instance); + if (ret) { + mk_instance_put(target_instance); + return ret; + } /* For non-running instances, allocate memory from pool and add to instance */ if (target_instance->state != MK_STATE_ACTIVE) { @@ -1266,6 +1286,11 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; + ret = mk_memory_change_allowed(target_instance); + if (ret) { + mk_instance_put(target_instance); + return ret; + } /* For non-running instances, just remove the memory region from the instance */ if (target_instance->state != MK_STATE_ACTIVE) { @@ -1305,117 +1330,66 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) } /** - * mk_send_device_add - Add PCI device to instance and wait for completion + * mk_send_device_add - Assign a PCI device to an instance * @instance_id: Target instance ID * @domain: PCI domain * @bus: PCI bus * @devfn: PCI device and function (combined) - * @driver_override: Target driver name for binding (can be NULL) - * @flags: Additional flags + * @driver_override: Target driver name for a root-kernel add + * @flags: Additional root-kernel add flags * - * For local instance, executes addition synchronously. - * For remote instance, sends IPI and waits for ACK response. - * For instances that are not yet running (MK_STATE_READY/LOADED), - * adds device to instance's device list. + * Remote assignment changes are permitted only while the target instance is + * ready. Active instances must be stopped and returned to ready state first. * * Returns: 0 on success, negative error code on failure */ int mk_send_device_add(int instance_id, u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { - struct mk_device_resource_payload payload = { - .domain = domain, - .bus = bus, - .devfn = devfn, - .flags = flags, - .sender_instance_id = root_instance->id - }; - struct mk_pending_msg *pending; struct mk_instance *target_instance; int ret; - u32 resource_id; - - if (driver_override) - strscpy(payload.driver_override, driver_override, sizeof(payload.driver_override)); - else - payload.driver_override[0] = '\0'; - - resource_id = (domain << 16) | (bus << 8) | devfn; + if (!root_instance) + return -ENODEV; if (instance_id == root_instance->id) - return mk_do_device_add(domain, bus, devfn, driver_override, flags); + return mk_do_device_add(domain, bus, devfn, driver_override, + flags); target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; - if (target_instance->state != MK_STATE_ACTIVE) { - ret = mk_instance_add_pci_device(target_instance, domain, bus, devfn); - goto out; - } - - pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_DEVICE_ADD, resource_id); - if (!pending) { - ret = -ENOMEM; - goto out; - } - - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_DEVICE_ADD, - &payload, sizeof(payload)); - if (ret < 0) { - mk_msg_pending_wait(pending, 0); - goto out; - } - - ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) - goto out; - - ret = mk_instance_add_pci_device(target_instance, domain, bus, devfn); - if (ret < 0) { - pr_warn("Device added to target but failed to update tracking: %d\n", ret); + if (target_instance->state != MK_STATE_READY) { + pr_err("PCI assignment changes require instance %d to be ready\n", + instance_id); + ret = -EBUSY; + } else { + ret = mk_instance_add_pci_device(target_instance, domain, bus, + devfn); } - - pr_info("Multikernel hotplug: Device %04x:%02x:%02x.%x successfully added to instance %d\n", - domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), instance_id); - - ret = 0; -out: mk_instance_put(target_instance); return ret; } /** - * mk_send_device_remove - Remove PCI device from instance and wait for completion + * mk_send_device_remove - Release a PCI device from an instance * @instance_id: Target instance ID * @domain: PCI domain * @bus: PCI bus * @devfn: PCI device and function (combined) * - * For local instance, executes removal synchronously. - * For remote instance, sends IPI and waits for ACK response. - * For instances that are not yet running (MK_STATE_READY/LOADED), - * removes device from instance's device list. + * Remote assignment changes are permitted only while the target instance is + * ready. Active instances must be stopped and returned to ready state first. * * Returns: 0 on success, negative error code on failure */ int mk_send_device_remove(int instance_id, u16 domain, u8 bus, u8 devfn) { - struct mk_device_resource_payload payload = { - .domain = domain, - .bus = bus, - .devfn = devfn, - .flags = 0, - .sender_instance_id = root_instance->id - }; - struct mk_pending_msg *pending; struct mk_instance *target_instance; int ret; - u32 resource_id; - - payload.driver_override[0] = '\0'; - resource_id = (domain << 16) | (bus << 8) | devfn; + if (!root_instance) + return -ENODEV; if (instance_id == root_instance->id) return mk_do_device_remove(domain, bus, devfn); @@ -1423,38 +1397,14 @@ int mk_send_device_remove(int instance_id, u16 domain, u8 bus, u8 devfn) if (!target_instance) return -ENODEV; - if (target_instance->state != MK_STATE_ACTIVE) { - ret = mk_instance_remove_pci_device(target_instance, domain, bus, devfn); - goto out; - } - - pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_DEVICE_REMOVE, resource_id); - if (!pending) { - ret = -ENOMEM; - goto out; - } - - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_DEVICE_REMOVE, - &payload, sizeof(payload)); - if (ret < 0) { - mk_msg_pending_wait(pending, 0); - goto out; - } - - ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) - goto out; - - ret = mk_instance_remove_pci_device(target_instance, domain, bus, devfn); - if (ret < 0) { - pr_warn("Device removed from target but failed to update tracking: %d\n", ret); + if (target_instance->state != MK_STATE_READY) { + pr_err("PCI assignment changes require instance %d to be ready\n", + instance_id); + ret = -EBUSY; + } else { + ret = mk_instance_remove_pci_device(target_instance, domain, bus, + devfn); } - - pr_info("Multikernel hotplug: Device %04x:%02x:%02x.%x successfully removed from instance %d\n", - domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), instance_id); - - ret = 0; -out: mk_instance_put(target_instance); return ret; } diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index a3aa40eb1e886b..b730fe3650774f 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -12,12 +12,36 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, int resources_node, size_t dtb_size); struct mk_instance *mk_instance_find_by_name(const char *name); int mk_instance_destroy(struct mk_instance *instance); +int mk_instance_release_resources(struct mk_instance *instance); /* dts.c */ int mk_dt_parse_resources(const void *fdt, int resources_node, const char *instance_name, struct mk_dt_config *config); int mk_dt_generate_instance_dtb(struct mk_instance *instance, void **out_dtb, size_t *out_size); +int mk_dt_parse_pci_host_bridges(const void *fdt, int resources_node, + struct list_head *bridges, int *count, + bool *valid); +int mk_pci_host_bridges_clone(struct list_head *dst, int *dst_count, + bool *dst_valid, const struct list_head *src, + int src_count, bool src_valid); +void mk_pci_host_bridges_free(struct list_head *bridges, int *count, + bool *valid); + +/* pci.c */ +int mk_pci_lease_system_init(void); +void mk_pci_lease_system_cleanup(void); +void mk_pci_lease_instance_init(struct mk_instance *instance); +bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance); +int mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count); +int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn); +int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn); +int mk_pci_release_assignments(struct mk_instance *instance); +int mk_instance_force_halt(struct mk_instance *instance); /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index a0573fc23a1c99..5c0ef9f757ccf5 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -239,6 +239,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, struct kernfs_node *kn; struct mk_dt_config config; void *dtb_copy; + int release_ret; int ret; int allocated_id; @@ -269,6 +270,8 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); + mk_pci_lease_instance_init(instance); + INIT_LIST_HEAD(&instance->pci_host_bridges); INIT_LIST_HEAD(&instance->platform_devices); kref_init(&instance->refcount); @@ -333,7 +336,16 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, kfree(instance->dtb_data); instance->dtb_data = NULL; err_free_resources: - mk_instance_free_memory(instance); + release_ret = mk_instance_release_resources(instance); + if (release_ret) { + pr_crit("Retaining failed instance '%s' because PCI cleanup failed: %d\n", + name, release_ret); + list_add_tail(&instance->list, &mk_instance_list); + kernfs_activate(kn); + mk_instance_set_state(instance, MK_STATE_FAILED); + mk_dt_config_free(&config); + return release_ret; + } err_free_idr: idr_remove(&mk_instance_idr, instance->id); err_remove_dir: @@ -435,6 +447,8 @@ static int mk_kernfs_rmdir(struct kernfs_node *kn) */ int mk_instance_destroy(struct mk_instance *instance) { + int ret; + lockdep_assert_held(&mk_instance_mutex); if (!instance) { @@ -454,6 +468,14 @@ int mk_instance_destroy(struct mk_instance *instance) return -EBUSY; } + ret = mk_instance_release_resources(instance); + if (ret) { + pr_crit("Cannot remove instance '%s' while PCI cleanup is unsafe: %d\n", + instance->name, ret); + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; + } + list_del(&instance->list); idr_remove(&mk_instance_idr, instance->id); if (instance->kn) { diff --git a/kernel/multikernel/kho.c b/kernel/multikernel/kho.c index 176360dba697e6..54537e4c95adb2 100644 --- a/kernel/multikernel/kho.c +++ b/kernel/multikernel/kho.c @@ -14,7 +14,6 @@ #include #include #include -#include #ifdef CONFIG_KEXEC_HANDOVER #include #include @@ -402,8 +401,12 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char INIT_LIST_HEAD(&instance->list); kref_init(&instance->refcount); INIT_LIST_HEAD(&instance->pci_devices); + mk_pci_lease_instance_init(instance); instance->pci_devices_valid = false; instance->pci_device_count = 0; + INIT_LIST_HEAD(&instance->pci_host_bridges); + instance->pci_host_bridges_valid = false; + instance->pci_host_bridge_count = 0; INIT_LIST_HEAD(&instance->platform_devices); instance->platform_devices_valid = false; instance->platform_device_count = 0; @@ -451,19 +454,13 @@ static int __init mk_kho_copy_pci_devices(const struct mk_dt_config *config, instance->pci_devices_valid = true; list_for_each_entry(src_dev, &config->pci_devices, list) { - dst_dev = kzalloc(sizeof(*dst_dev), GFP_KERNEL); + dst_dev = kmemdup(src_dev, sizeof(*dst_dev), GFP_KERNEL); if (!dst_dev) { pr_err("Failed to allocate PCI device entry\n"); return -ENOMEM; } - dst_dev->vendor = src_dev->vendor; - dst_dev->device = src_dev->device; - dst_dev->domain = src_dev->domain; - dst_dev->bus = src_dev->bus; - dst_dev->slot = src_dev->slot; - dst_dev->func = src_dev->func; - + INIT_LIST_HEAD(&dst_dev->list); list_add_tail(&dst_dev->list, &instance->pci_devices); instance->pci_device_count++; } @@ -472,6 +469,17 @@ static int __init mk_kho_copy_pci_devices(const struct mk_dt_config *config, return 0; } +static int __init mk_kho_copy_pci_host_bridges(const struct mk_dt_config *config, + struct mk_instance *instance) +{ + return mk_pci_host_bridges_clone(&instance->pci_host_bridges, + &instance->pci_host_bridge_count, + &instance->pci_host_bridges_valid, + &config->pci_host_bridges, + config->pci_host_bridge_count, + config->pci_host_bridges_valid); +} + static int __init mk_kho_copy_platform_devices(const struct mk_dt_config *config, struct mk_instance *instance) { @@ -770,6 +778,12 @@ int __init mk_kho_restore_dtbs(void) goto cleanup_devices; } + ret = mk_kho_copy_pci_host_bridges(&config, instance); + if (ret) { + pr_err("Failed to copy PCI host bridge metadata: %d\n", ret); + goto cleanup_devices; + } + ret = mk_kho_copy_platform_devices(&config, instance); if (ret) { pr_err("Failed to copy platform devices: %d\n", ret); @@ -796,6 +810,9 @@ int __init mk_kho_restore_dtbs(void) return 0; cleanup_devices: + mk_pci_host_bridges_free(&instance->pci_host_bridges, + &instance->pci_host_bridge_count, + &instance->pci_host_bridges_valid); if (instance->pci_devices_valid) { struct mk_pci_device *pci_dev, *tmp_pci; list_for_each_entry_safe(pci_dev, tmp_pci, &instance->pci_devices, list) { @@ -827,89 +844,6 @@ int __init mk_kho_restore_dtbs(void) /* Run at early_initcall to enforce CPU restrictions before per-CPU allocations */ early_initcall(mk_kho_restore_dtbs); -/** - * mk_pci_should_probe - Check if PCI probing should occur at all - * @bus: PCI bus - * @devfn: device/function number - * - * Called BEFORE any PCI config space reads to determine if probing - * should proceed. This prevents config space accesses to devices - * that are not in the whitelist. - * - * Returns: true if probing should proceed, false to skip entirely - */ -bool mk_pci_should_probe(struct pci_bus *bus, int devfn) -{ - struct mk_pci_device *pci_dev; - u16 domain = pci_domain_nr(bus); - u8 bus_num = bus->number; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - u8 hdr_type; - - if (!root_instance) - return true; - - if (!root_instance->dtb_data) - return true; - - if (!root_instance->pci_devices_valid || root_instance->pci_device_count == 0) - return false; - - list_for_each_entry(pci_dev, &root_instance->pci_devices, list) { - if (pci_dev->domain != domain) - continue; - - /* Exact location match - always allow */ - if (pci_dev->bus == bus_num && - pci_dev->slot == slot && - pci_dev->func == func) - return true; - } - - /* - * Check if any whitelisted device is on a downstream bus. - * If so, this might be a bridge in the path to that device. - */ - list_for_each_entry(pci_dev, &root_instance->pci_devices, list) { - if (pci_dev->domain == domain && pci_dev->bus > bus_num) - goto check_bridge; - } - return false; - -check_bridge: - /* - * There's a whitelisted device on a downstream bus. Check if this - * is a bridge that serves it. - */ - if (pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type) == 0) { - bool is_bridge = ((hdr_type & PCI_HEADER_TYPE_MASK) == PCI_HEADER_TYPE_BRIDGE); - - if (is_bridge) { - u8 secondary_bus = 0, subordinate_bus = 0; - - pci_bus_read_config_byte(bus, devfn, PCI_SECONDARY_BUS, &secondary_bus); - pci_bus_read_config_byte(bus, devfn, PCI_SUBORDINATE_BUS, &subordinate_bus); - - /* - * Allow bridge if there's a whitelisted device on any bus - * between secondary and subordinate (inclusive). - */ - if (secondary_bus > 0 && subordinate_bus >= secondary_bus) { - list_for_each_entry(pci_dev, &root_instance->pci_devices, list) { - if (pci_dev->domain == domain && - pci_dev->bus >= secondary_bus && - pci_dev->bus <= subordinate_bus) - return true; - } - } - } - } - - return false; -} -EXPORT_SYMBOL_GPL(mk_pci_should_probe); - bool mk_platform_device_allowed(const char *name, const char *hid) { struct mk_platform_device *plat_dev; @@ -951,12 +885,6 @@ int __init mk_kho_restore_dtbs(void) return 0; } -bool mk_pci_should_probe(struct pci_bus *bus, int devfn) -{ - return true; -} -EXPORT_SYMBOL_GPL(mk_pci_should_probe); - bool mk_platform_device_allowed(const char *name, const char *hid) { return true; diff --git a/kernel/multikernel/mem.c b/kernel/multikernel/mem.c index 5c2da8dadb12ed..001d75bcc0368d 100644 --- a/kernel/multikernel/mem.c +++ b/kernel/multikernel/mem.c @@ -285,17 +285,30 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) phys_addr_t phys_addr; int ret; + if (!instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + if (mk_pci_iommu_lease_active_locked(instance)) { + pr_err("Cannot add memory to instance %d while an IOMMU lease is active\n", + instance->id); + ret = -EBUSY; + goto out_unlock; + } + phys_addr = multikernel_alloc(size); if (!phys_addr) { pr_err("Failed to allocate %zu bytes from multikernel pool for instance %d\n", size, instance->id); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region = kzalloc(sizeof(*region), GFP_KERNEL); if (!region) { multikernel_free(phys_addr, size); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region->res.name = kasprintf(GFP_KERNEL, "mk-instance-%d-%s-region-%d", @@ -303,7 +316,8 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) if (!region->res.name) { kfree(region); multikernel_free(phys_addr, size); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region->res.start = phys_addr; @@ -318,7 +332,7 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) kfree(region->res.name); kfree(region); multikernel_free(phys_addr, size); - return ret; + goto out_unlock; } INIT_LIST_HEAD(®ion->list); @@ -329,7 +343,10 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), size >> 20, instance->id, instance->name); - return 0; + ret = 0; +out_unlock: + mutex_unlock(&instance->resource_mutex); + return ret; } /** @@ -350,10 +367,19 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, { struct mk_memory_region *region, *tmp; bool found = false; + int ret; if (!instance) return -EINVAL; + mutex_lock(&instance->resource_mutex); + if (mk_pci_iommu_lease_active_locked(instance)) { + pr_err("Cannot remove memory from instance %d while an IOMMU lease is active\n", + instance->id); + ret = -EBUSY; + goto out_unlock; + } + list_for_each_entry_safe(region, tmp, &instance->memory_regions, list) { if (region->res.start == phys_addr && resource_size(®ion->res) == size) { @@ -381,10 +407,14 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), instance->id, instance->name); - return -ENOENT; + ret = -ENOENT; + } else { + ret = 0; } - return 0; +out_unlock: + mutex_unlock(&instance->resource_mutex); + return ret; } /** diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c new file mode 100644 index 00000000000000..39c2988c5be3f4 --- /dev/null +++ b/kernel/multikernel/pci.c @@ -0,0 +1,1181 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Multikernel PCI assignment policy + * + * Keeps assigned-device discovery, identity presentation, bridge traversal, + * and resource restoration independent from the KHO transport that populated + * the current instance. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +struct mk_pci_assignment { + struct list_head instance_node; + struct list_head active_node; + struct list_head transaction_node; + struct mk_instance *instance; + struct mk_pci_device *inventory; + struct pci_dev *vf; + struct pci_dev *pf; + const struct device_driver *host_driver; + struct iommu_group *iommu_group; + struct iommu_domain *iommu_domain; + char *host_driver_override; + struct mutex iommu_mutex; /* Serializes IOMMU activation and teardown. */ + unsigned int iommu_mapped_regions; + bool iommu_dma_owner; + bool iommu_attached; + bool iommu_override_active; + struct work_struct failure_work; + atomic_t failure_pending; + bool assigned; + bool inventory_moved; + bool expected_unbind; +}; + +static DEFINE_MUTEX(mk_pci_lease_mutex); +static DEFINE_SPINLOCK(mk_pci_active_lock); +static LIST_HEAD(mk_pci_active_assignments); +static bool mk_pci_notifier_registered; + +static bool mk_pci_device_live(struct pci_dev *pdev) +{ + return device_is_registered(&pdev->dev) && + !pci_dev_is_disconnected(pdev) && + pci_device_is_present(pdev); +} + +static bool mk_pci_device_matches_bdf(const struct mk_pci_device *device, + u16 domain, u8 bus, u8 devfn) +{ + return device->domain == domain && device->bus == bus && + device->slot == PCI_SLOT(devfn) && + device->func == PCI_FUNC(devfn); +} + +static struct mk_pci_device * +mk_pci_find_device_bdf(struct list_head *devices, u16 domain, u8 bus, u8 devfn) +{ + struct mk_pci_device *device; + + list_for_each_entry(device, devices, list) { + if (mk_pci_device_matches_bdf(device, domain, bus, devfn)) + return device; + } + + return NULL; +} + +static bool mk_pci_inventory_matches(const struct mk_pci_device *left, + const struct mk_pci_device *right) +{ + return left->vendor == right->vendor && + left->device == right->device && + mk_pci_device_matches_bdf(left, right->domain, right->bus, + PCI_DEVFN(right->slot, right->func)); +} + +static struct mk_pci_device * +mk_pci_find_root_inventory(const struct mk_pci_device *requested) +{ + struct mk_pci_device *device; + + list_for_each_entry(device, &root_instance->pci_devices, list) { + if (mk_pci_inventory_matches(device, requested)) + return device; + } + + return NULL; +} + +static struct mk_pci_device * +mk_pci_find_root_bdf(u16 domain, u8 bus, u8 devfn) +{ + return mk_pci_find_device_bdf(&root_instance->pci_devices, domain, bus, + devfn); +} + +static struct mk_pci_assignment * +mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_assignment *assignment; + + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + if (mk_pci_device_matches_bdf(assignment->inventory, domain, bus, + devfn)) + return assignment; + } + + return NULL; +} + +static void mk_pci_assignment_failure_work(struct work_struct *work) +{ + struct mk_pci_assignment *assignment = + container_of(work, struct mk_pci_assignment, failure_work); + struct mk_instance *instance = assignment->instance; + int ret; + + pr_err("PCI assignment lease for %s was lost by instance %d (%s)\n", + pci_name(assignment->vf), instance->id, instance->name); + + if (READ_ONCE(instance->state) == MK_STATE_ACTIVE) { + ret = mk_instance_force_halt(instance); + if (ret) + pr_err("Failed to force halt instance %d after PCI lease loss: %d\n", + instance->id, ret); + } + + mk_instance_set_state(instance, MK_STATE_FAILED); +} + +static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment) +{ + if (atomic_cmpxchg(&assignment->failure_pending, 0, 1)) + return; + + if (!schedule_work(&assignment->failure_work)) + atomic_set(&assignment->failure_pending, 0); +} + +static int mk_pci_bus_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct pci_dev *pdev = to_pci_dev(data); + struct mk_pci_assignment *assignment; + unsigned long flags; + + if (action != BUS_NOTIFY_DEL_DEVICE && + action != BUS_NOTIFY_REMOVED_DEVICE && + action != BUS_NOTIFY_UNBOUND_DRIVER) + return NOTIFY_DONE; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + list_for_each_entry(assignment, &mk_pci_active_assignments, + active_node) { + if (pdev != assignment->vf && pdev != assignment->pf) + continue; + if (pdev == assignment->vf && + action == BUS_NOTIFY_UNBOUND_DRIVER && + assignment->expected_unbind) + continue; + mk_pci_schedule_failure(assignment); + } + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + return NOTIFY_OK; +} + +static struct notifier_block mk_pci_bus_notifier = { + .notifier_call = mk_pci_bus_notify, +}; + +static void +mk_pci_release_bound_driver(struct mk_pci_assignment *assignment) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = true; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + device_release_driver(&assignment->vf->dev); + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); +} + +#if IS_ENABLED(CONFIG_IOMMU_API) +#define MK_PCI_ASSIGNMENT_DRIVER_NAME "multikernel-pci-assignment" + +static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, + const struct pci_device_id *id); +static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev); + +static struct pci_driver mk_pci_assignment_driver = { + .name = MK_PCI_ASSIGNMENT_DRIVER_NAME, + .probe = mk_pci_iommu_assignment_probe, + .remove = mk_pci_iommu_assignment_remove, + .driver_managed_dma = true, +}; + +struct mk_pci_iommu_group_check { + struct device *vf; + unsigned int count; +}; + +static int mk_pci_iommu_check_group_device(struct device *dev, void *data) +{ + struct mk_pci_iommu_group_check *check = data; + + check->count++; + return dev == check->vf ? 0 : -EXDEV; +} + +static void mk_pci_iommu_free_resv_regions(struct list_head *regions) +{ + struct iommu_resv_region *region, *tmp; + + list_for_each_entry_safe(region, tmp, regions, list) { + list_del(®ion->list); + kfree(region); + } +} + +static int mk_pci_iommu_validate_group(struct mk_pci_assignment *assignment) +{ + struct mk_pci_iommu_group_check check = { + .vf = &assignment->vf->dev, + }; + struct iommu_resv_region *region; + LIST_HEAD(resv_regions); + int ret; + + ret = iommu_group_for_each_dev(assignment->iommu_group, &check, + mk_pci_iommu_check_group_device); + if (ret || check.count != 1) { + pr_err("IOMMU group %d for %s is not an isolated singleton group\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf)); + return ret ?: -EXDEV; + } + + if (!iommu_group_has_isolated_msi(assignment->iommu_group)) { + pr_err("IOMMU group %d for %s lacks isolated MSI delivery\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf)); + return -EPERM; + } + + ret = iommu_get_group_resv_regions(assignment->iommu_group, + &resv_regions); + if (ret) { + mk_pci_iommu_free_resv_regions(&resv_regions); + return ret; + } + + list_for_each_entry(region, &resv_regions, list) { + if (region->type != IOMMU_RESV_DIRECT && + region->type != IOMMU_RESV_DIRECT_RELAXABLE && + region->type != IOMMU_RESV_SW_MSI) + continue; + pr_err("IOMMU group %d for %s requires unsupported reserved region %#llx-%#llx type %u\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf), + (unsigned long long)region->start, + (unsigned long long)(region->start + region->length - 1), + region->type); + ret = -EPERM; + break; + } + + mk_pci_iommu_free_resv_regions(&resv_regions); + return ret; +} + +static int +mk_pci_iommu_validate_region(struct mk_pci_assignment *assignment, + const struct mk_memory_region *region) +{ + struct iommu_domain *domain = assignment->iommu_domain; + resource_size_t start = region->res.start; + resource_size_t size = resource_size(®ion->res); + u64 dma_mask = dma_get_mask(&assignment->vf->dev); + u64 end; + unsigned long min_page_size; + + if (!size || check_add_overflow((u64)start, (u64)size - 1, &end)) + return -EOVERFLOW; + if (!domain->pgsize_bitmap) + return -EOPNOTSUPP; + + min_page_size = 1UL << __ffs(domain->pgsize_bitmap); + if (!IS_ALIGNED(start, min_page_size) || + !IS_ALIGNED(size, min_page_size)) { + pr_err("Instance %d memory %#llx-%#llx is not aligned to IOMMU page size %#lx\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, min_page_size); + return -EINVAL; + } + if (start > ULONG_MAX || end > ULONG_MAX || end > dma_mask) { + pr_err("Instance %d memory %#llx-%#llx exceeds DMA addressability of %s\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, pci_name(assignment->vf)); + return -ERANGE; + } + if (domain->geometry.force_aperture && + (start < domain->geometry.aperture_start || + end > domain->geometry.aperture_end)) { + pr_err("Instance %d memory %#llx-%#llx is outside the IOMMU aperture for %s\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, pci_name(assignment->vf)); + return -ERANGE; + } + + return 0; +} + +static void mk_pci_iommu_unmap_regions(struct mk_pci_assignment *assignment) +{ + struct mk_memory_region *region; + unsigned int remaining = assignment->iommu_mapped_regions; + + list_for_each_entry(region, &assignment->instance->memory_regions, list) { + resource_size_t size; + size_t unmapped; + + if (!remaining) + break; + size = resource_size(®ion->res); + unmapped = iommu_unmap(assignment->iommu_domain, + region->res.start, size); + if (unmapped != size) + pr_err("IOMMU unmapped only %#zx of %#llx bytes for instance %d at %#llx\n", + unmapped, (unsigned long long)size, + assignment->instance->id, + (unsigned long long)region->res.start); + remaining--; + } + if (remaining) + pr_err("IOMMU lease for %s lost %u mapped instance regions\n", + pci_name(assignment->vf), remaining); + assignment->iommu_mapped_regions = 0; +} + +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + bool transactions_drained; + int ret; + + if (!mk_pci_device_live(vf)) + return 0; + + /* + * Releasing DMA ownership restores the group's default domain. Stop new + * DMA first, drain requests already issued, and reset the VF while the + * assignment domain still contains any stragglers. + */ + pci_clear_master(vf); + transactions_drained = pci_wait_for_pending_transaction(vf); + ret = pcie_reset_flr(vf, false); + if (!ret) + return 0; + if (ret == -ENOTTY && transactions_drained) + return 0; + + if (!transactions_drained) + pr_err("Timed out draining DMA from assigned VF %s\n", + pci_name(vf)); + if (ret != -ENOTTY) + pr_err("Failed to reset assigned VF %s: %d\n", + pci_name(vf), ret); + + /* + * Keep the assignment domain attached when the device cannot be made + * safe. The lease owner can retry teardown after the instance halts. + */ + return ret == -ENOTTY ? -ETIMEDOUT : ret; +} + +static void +__mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ + if (assignment->iommu_attached) { + iommu_detach_group(assignment->iommu_domain, + assignment->iommu_group); + assignment->iommu_attached = false; + } + if (assignment->iommu_dma_owner) { + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + } +} + +static void +mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ + mutex_lock(&assignment->iommu_mutex); + __mk_pci_iommu_deactivate_assignment(assignment); + mutex_unlock(&assignment->iommu_mutex); +} + +static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment) +{ + mutex_lock(&assignment->iommu_mutex); + __mk_pci_iommu_deactivate_assignment(assignment); + + if (assignment->iommu_domain) { + mk_pci_iommu_unmap_regions(assignment); + iommu_domain_free(assignment->iommu_domain); + assignment->iommu_domain = NULL; + } + if (assignment->iommu_group) { + iommu_group_put(assignment->iommu_group); + assignment->iommu_group = NULL; + } + mutex_unlock(&assignment->iommu_mutex); +} + +static int mk_pci_iommu_fault(struct iommu_domain *domain, + struct device *dev, unsigned long iova, + int flags, void *token) +{ + struct mk_pci_assignment *assignment = token; + + if (WARN_ON_ONCE(domain != assignment->iommu_domain)) + return -EINVAL; + pr_err_ratelimited("IOMMU fault from %s at IOVA %#lx (%s) for instance %d\n", + dev_name(dev), iova, + flags & IOMMU_FAULT_WRITE ? "write" : "read", + assignment->instance->id); + mk_pci_schedule_failure(assignment); + return 0; +} + +static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) +{ + struct mk_memory_region *region; + int ret; + + if (!device_iommu_mapped(&assignment->vf->dev)) { + pr_err("Cannot assign %s without an active hardware IOMMU\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; + } + if (!assignment->instance->region_count || + list_empty(&assignment->instance->memory_regions)) + return -EINVAL; + + assignment->iommu_group = iommu_group_get(&assignment->vf->dev); + if (!assignment->iommu_group) + return -ENODEV; + + ret = mk_pci_iommu_validate_group(assignment); + if (ret) + goto err_release; + + assignment->iommu_domain = + iommu_paging_domain_alloc(&assignment->vf->dev); + if (IS_ERR(assignment->iommu_domain)) { + ret = PTR_ERR(assignment->iommu_domain); + assignment->iommu_domain = NULL; + goto err_release; + } + + list_for_each_entry(region, &assignment->instance->memory_regions, list) { + resource_size_t size = resource_size(®ion->res); + + ret = mk_pci_iommu_validate_region(assignment, region); + if (ret) + goto err_release; + ret = iommu_map(assignment->iommu_domain, region->res.start, + region->res.start, size, IOMMU_READ | IOMMU_WRITE, + GFP_KERNEL); + if (ret) + goto err_release; + assignment->iommu_mapped_regions++; + } + + iommu_set_fault_handler(assignment->iommu_domain, + mk_pci_iommu_fault, assignment); + pr_info("Prepared host IOMMU domain for %s with %u instance regions\n", + pci_name(assignment->vf), assignment->iommu_mapped_regions); + return 0; + +err_release: + mk_pci_iommu_release_assignment(assignment); + return ret; +} + +static int mk_pci_iommu_commit_assignment(struct mk_pci_assignment *assignment) +{ + int ret; + + if (!assignment->iommu_domain) + return 0; + + if (assignment->vf->driver_override) { + assignment->host_driver_override = + kstrdup(assignment->vf->driver_override, GFP_KERNEL); + if (!assignment->host_driver_override) + return -ENOMEM; + } + + ret = driver_set_override(&assignment->vf->dev, + &assignment->vf->driver_override, + MK_PCI_ASSIGNMENT_DRIVER_NAME, + strlen(MK_PCI_ASSIGNMENT_DRIVER_NAME)); + if (ret) + return ret; + assignment->iommu_override_active = true; + pci_set_drvdata(assignment->vf, assignment); + ret = device_driver_attach(&mk_pci_assignment_driver.driver, + &assignment->vf->dev); + if (ret) + return ret; + if (assignment->vf->dev.driver != &mk_pci_assignment_driver.driver) + return -ENODEV; + return 0; +} + +static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + int ret; + + if (!assignment || assignment->vf != pdev || !assignment->iommu_domain) + return -ENODEV; + ret = iommu_device_claim_dma_owner(&assignment->vf->dev, assignment); + if (ret) + return ret; + assignment->iommu_dma_owner = true; + + ret = iommu_attach_group(assignment->iommu_domain, + assignment->iommu_group); + if (ret) + return ret; + assignment->iommu_attached = true; + pr_info("Attached %s to host-owned IOMMU domain for instance %d\n", + pci_name(assignment->vf), assignment->instance->id); + return 0; +} + +static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) +{ + struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + int ret; + + if (!assignment || assignment->vf != pdev) + return; + + if (READ_ONCE(assignment->expected_unbind)) { + pci_set_drvdata(pdev, NULL); + return; + } + + ret = mk_pci_quiesce_assignment(assignment); + if (ret) { + pr_crit("Keeping IOMMU containment for %s after unsafe driver removal: %d\n", + pci_name(pdev), ret); + mk_pci_schedule_failure(assignment); + } else { + mk_pci_iommu_deactivate_assignment(assignment); + } + pci_set_drvdata(pdev, NULL); +} + +static int mk_pci_restore_host_binding(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + const char *override = assignment->host_driver_override ?: ""; + int ret = 0; + + if (vf->dev.driver == &mk_pci_assignment_driver.driver) { + mk_pci_release_bound_driver(assignment); + } else if (vf->dev.driver) { + pr_err("Cannot release assignment driver from %s: device is bound to %s\n", + pci_name(vf), vf->dev.driver->name); + return -EBUSY; + } + + pci_set_drvdata(vf, NULL); + if (assignment->iommu_override_active) { + ret = driver_set_override(&vf->dev, &vf->driver_override, + override, strlen(override)); + if (ret) + return ret; + assignment->iommu_override_active = false; + } + + mk_pci_iommu_deactivate_assignment(assignment); + if (assignment->host_driver && mk_pci_device_live(vf)) { + if (!vf->dev.driver) { + ret = device_driver_attach(assignment->host_driver, &vf->dev); + if (ret) { + pr_err("Failed to restore driver %s to %s: %d\n", + assignment->host_driver->name, + pci_name(vf), ret); + return ret; + } + } else if (vf->dev.driver != assignment->host_driver) { + pr_err("Cannot restore driver %s to %s: device is bound to %s\n", + assignment->host_driver->name, pci_name(vf), + vf->dev.driver->name); + return -EBUSY; + } + } + return 0; +} + +static int mk_pci_iommu_system_init(void) +{ + return pci_register_driver(&mk_pci_assignment_driver); +} + +static void mk_pci_iommu_system_cleanup(void) +{ + pci_unregister_driver(&mk_pci_assignment_driver); +} +#else +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static void +mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ +} + +static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) +{ + pr_err("Cannot assign %s without CONFIG_IOMMU_API\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; +} + +static int mk_pci_iommu_commit_assignment(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment) +{ +} + +static int mk_pci_restore_host_binding(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static int mk_pci_iommu_system_init(void) +{ + return 0; +} + +static void mk_pci_iommu_system_cleanup(void) +{ +} +#endif + +static int +mk_pci_prepare_assignment(struct mk_instance *instance, + const struct mk_pci_device *requested, + struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + struct mk_pci_device *inventory; + struct pci_dev *vf; + struct pci_dev *pf; + struct pci_dev *physfn; + int ret; + + inventory = mk_pci_find_root_inventory(requested); + if (!inventory) { + pr_err("PCI device %04x:%04x@%04x:%02x:%02x.%x is not available in the root pool\n", + requested->vendor, requested->device, requested->domain, + requested->bus, requested->slot, requested->func); + return -ENOENT; + } + + vf = pci_get_domain_bus_and_slot(inventory->domain, + inventory->bus, + PCI_DEVFN(inventory->slot, + inventory->func)); + if (!vf) + return -ENODEV; + + if (vf->vendor != inventory->vendor || + vf->device != inventory->device) { + pr_err("PCI identity changed for %s: expected %04x:%04x, found %04x:%04x\n", + pci_name(vf), inventory->vendor, inventory->device, + vf->vendor, vf->device); + pci_dev_put(vf); + return -ENODEV; + } + + physfn = pci_physfn(vf); + if (!vf->is_virtfn || physfn == vf) { + pr_err("PCI assignment only supports SR-IOV VFs, rejecting %s\n", + pci_name(vf)); + pci_dev_put(vf); + return -EOPNOTSUPP; + } + + if (!mk_pci_device_live(vf) || !mk_pci_device_live(physfn)) { + pci_dev_put(vf); + return -ENODEV; + } + + if (pci_is_dev_assigned(vf) || + mk_pci_find_assignment(instance, inventory->domain, + inventory->bus, + PCI_DEVFN(inventory->slot, + inventory->func))) { + pci_dev_put(vf); + return -EBUSY; + } + + pf = pci_dev_get(physfn); + assignment = kzalloc(sizeof(*assignment), GFP_KERNEL); + if (!assignment) { + pci_dev_put(pf); + pci_dev_put(vf); + return -ENOMEM; + } + + assignment->instance = instance; + assignment->inventory = inventory; + assignment->vf = vf; + assignment->pf = pf; + assignment->host_driver = vf->dev.driver; + if (assignment->host_driver && assignment->host_driver->owner && + !try_module_get(assignment->host_driver->owner)) { + kfree(assignment); + pci_dev_put(pf); + pci_dev_put(vf); + return -ENODEV; + } + + INIT_LIST_HEAD(&assignment->instance_node); + INIT_LIST_HEAD(&assignment->active_node); + INIT_LIST_HEAD(&assignment->transaction_node); + mutex_init(&assignment->iommu_mutex); + INIT_WORK(&assignment->failure_work, mk_pci_assignment_failure_work); + atomic_set(&assignment->failure_pending, 0); + + ret = mk_pci_iommu_prepare_assignment(assignment); + if (ret) + goto err_module; + + list_add_tail(&assignment->instance_node, &instance->pci_assignments); + list_add_tail(&assignment->transaction_node, transaction); + + return 0; + +err_module: + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + kfree(assignment); + pci_dev_put(pf); + pci_dev_put(vf); + return ret; +} + +static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + unsigned long flags; + int ret; + int i; + + if (!mk_pci_device_live(vf) || !mk_pci_device_live(assignment->pf)) + return -ENODEV; + + if (vf->dev.driver != assignment->host_driver || + pci_is_dev_assigned(vf)) + return -EBUSY; + + pci_set_dev_assigned(vf); + assignment->assigned = true; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + list_add_tail(&assignment->active_node, &mk_pci_active_assignments); + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + if (assignment->host_driver) + mk_pci_release_bound_driver(assignment); + + if (vf->dev.driver) + return -EBUSY; + + ret = mk_pci_iommu_commit_assignment(assignment); + if (ret) + return ret; + + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + assignment->inventory->resources[i].start = + vf->resource[i].start; + assignment->inventory->resources[i].end = + vf->resource[i].end; + assignment->inventory->resources[i].flags = + vf->resource[i].flags; + } + assignment->inventory->resources_valid = true; + + list_move_tail(&assignment->inventory->list, + &assignment->instance->pci_devices); + root_instance->pci_device_count--; + assignment->instance->pci_device_count++; + assignment->instance->pci_devices_valid = true; + assignment->inventory_moved = true; + + pr_info("Leased SR-IOV VF %s to instance %d (%s)\n", + pci_name(vf), assignment->instance->id, + assignment->instance->name); + return 0; +} + +static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) +{ + struct mk_instance *instance = assignment->instance; + struct pci_dev *vf = assignment->vf; + unsigned long flags; + int ret; + + if (!assignment->assigned) + goto release_resources; + + ret = mk_pci_quiesce_assignment(assignment); + if (ret) + return ret; + + ret = mk_pci_restore_host_binding(assignment); + if (ret) + return ret; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + if (!list_empty(&assignment->active_node)) + list_del_init(&assignment->active_node); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + if (assignment->assigned) { + pci_clear_dev_assigned(vf); + assignment->assigned = false; + } + +release_resources: + mk_pci_iommu_release_assignment(assignment); + cancel_work_sync(&assignment->failure_work); + + if (assignment->inventory_moved && root_instance) { + list_move_tail(&assignment->inventory->list, + &root_instance->pci_devices); + instance->pci_device_count--; + root_instance->pci_device_count++; + root_instance->pci_devices_valid = true; + assignment->inventory_moved = false; + } + + kfree(assignment->host_driver_override); + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + if (!list_empty(&assignment->transaction_node)) + list_del_init(&assignment->transaction_node); + list_del_init(&assignment->instance_node); + pci_dev_put(assignment->pf); + pci_dev_put(vf); + kfree(assignment); + + return 0; +} + +static int mk_pci_rollback_transaction(struct list_head *transaction) +{ + struct mk_pci_assignment *assignment, *tmp; + int rollback_ret = 0; + int ret; + + list_for_each_entry_safe_reverse(assignment, tmp, transaction, + transaction_node) { + ret = mk_pci_release_assignment(assignment); + if (!ret) + continue; + pr_crit("Failed to roll back PCI assignment for %s: %d\n", + pci_name(assignment->vf), ret); + list_del_init(&assignment->transaction_node); + if (!rollback_ret) + rollback_ret = ret; + } + + return rollback_ret; +} + +static int mk_pci_commit_transaction(struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + int ret; + + list_for_each_entry(assignment, transaction, transaction_node) { + ret = mk_pci_commit_assignment(assignment); + if (ret) + return ret; + } + + while (!list_empty(transaction)) { + assignment = list_first_entry(transaction, + struct mk_pci_assignment, + transaction_node); + list_del_init(&assignment->transaction_node); + } + + return 0; +} + +void mk_pci_lease_instance_init(struct mk_instance *instance) +{ + mutex_init(&instance->resource_mutex); + INIT_LIST_HEAD(&instance->pci_assignments); +} + +bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance) +{ + if (!instance) + return false; + + lockdep_assert_held(&instance->resource_mutex); + return !list_empty(&instance->pci_assignments); +} + +int mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count) +{ + struct mk_pci_device *requested; + LIST_HEAD(transaction); + int prepared = 0; + int ret = 0; + int rollback_ret; + + if (!instance || instance == root_instance || !requested_devices || + requested_count < 0) + return -EINVAL; + if (!root_instance || !root_instance->pci_devices_valid) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + list_for_each_entry(requested, requested_devices, list) { + ret = mk_pci_prepare_assignment(instance, requested, + &transaction); + if (ret) + goto rollback; + prepared++; + } + + if (prepared != requested_count) { + ret = -EINVAL; + goto rollback; + } + + ret = mk_pci_commit_transaction(&transaction); + if (ret) + goto rollback; + goto out; + +rollback: + rollback_ret = mk_pci_rollback_transaction(&transaction); + if (rollback_ret) + ret = rollback_ret; +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_device *inventory; + LIST_HEAD(transaction); + int ret; + int rollback_ret; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + if (instance->state != MK_STATE_READY) { + ret = -EBUSY; + goto out; + } + if (!root_instance || !root_instance->pci_devices_valid) { + ret = -EINVAL; + goto out; + } + + inventory = mk_pci_find_root_bdf(domain, bus, devfn); + if (!inventory) { + ret = -ENOENT; + goto out; + } + + ret = mk_pci_prepare_assignment(instance, inventory, &transaction); + if (ret) + goto rollback; + ret = mk_pci_commit_transaction(&transaction); + if (ret) + goto rollback; + goto out; + +rollback: + rollback_ret = mk_pci_rollback_transaction(&transaction); + if (rollback_ret) + ret = rollback_ret; +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_assignment *assignment; + int ret; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + if (instance->state != MK_STATE_READY) { + ret = -EBUSY; + goto out; + } + + assignment = mk_pci_find_assignment(instance, domain, bus, devfn); + if (!assignment) { + ret = -ENOENT; + goto out; + } + + ret = mk_pci_release_assignment(assignment); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +int mk_pci_release_assignments(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + int ret = 0; + + if (!instance || instance == root_instance) + return 0; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + while (!list_empty(&instance->pci_assignments)) { + assignment = list_last_entry(&instance->pci_assignments, + struct mk_pci_assignment, + instance_node); + ret = mk_pci_release_assignment(assignment); + if (ret) { + pr_crit("Instance %d retains unsafe PCI lease for %s: %d\n", + instance->id, pci_name(assignment->vf), ret); + break; + } + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +int mk_pci_lease_system_init(void) +{ + int ret; + + ret = mk_pci_iommu_system_init(); + if (ret) + return ret; + ret = bus_register_notifier(&pci_bus_type, &mk_pci_bus_notifier); + if (ret) { + mk_pci_iommu_system_cleanup(); + return ret; + } + mk_pci_notifier_registered = true; + return 0; +} + +void mk_pci_lease_system_cleanup(void) +{ + if (mk_pci_notifier_registered) { + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; + } + mk_pci_iommu_system_cleanup(); +} + +static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) +{ + if (!root_instance || !root_instance->dtb_data || + !root_instance->pci_devices_valid) + return NULL; + + return mk_pci_find_device_bdf(&root_instance->pci_devices, + pci_domain_nr(bus), bus->number, devfn); +} + +/** + * mk_pci_get_assigned_identity - Get the identity presented to an instance + * @bus: PCI bus + * @devfn: device/function number + * @vendor: assigned Vendor ID + * @device_id: assigned Device ID + * + * Returns: true when assignment metadata contains an exact location match. + */ +bool mk_pci_get_assigned_identity(struct pci_bus *bus, int devfn, + u16 *vendor, u16 *device_id) +{ + struct mk_pci_device *device = mk_pci_find_assigned(bus, devfn); + + if (!device) + return false; + + if (vendor) + *vendor = device->vendor; + if (device_id) + *device_id = device->device; + return true; +} + +static void mk_pci_restore_resources(struct pci_dev *dev) +{ + struct mk_pci_device *device; + int i; + + device = mk_pci_find_assigned(dev->bus, dev->devfn); + if (!device || !device->resources_valid) + return; + + dev->non_compliant_bars = true; + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + dev->resource[i].start = device->resources[i].start; + dev->resource[i].end = device->resources[i].end; + dev->resource[i].flags = device->resources[i].flags; + } + pr_info("Restored PCI BAR resources for %s\n", pci_name(dev)); +} + +DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, mk_pci_restore_resources);