diff --git a/hw/remote/vfio-user-obj.c b/hw/remote/vfio-user-obj.c index 12ecdab6de..d84c6f7781 100644 --- a/hw/remote/vfio-user-obj.c +++ b/hw/remote/vfio-user-obj.c @@ -373,9 +373,9 @@ static int vfu_object_mr_rw(MemoryRegion *mr, uint8_t *buf, hwaddr offset, ram_ptr = memory_region_get_ram_ptr(mr); if (is_write) { - memcpy((ram_ptr + offset), buf, size); + qemu_ram_move((ram_ptr + offset), buf, size); } else { - memcpy(buf, (ram_ptr + offset), size); + qemu_ram_move(buf, (ram_ptr + offset), size); } return 0; diff --git a/include/system/memory.h b/include/system/memory.h index 05c65fc69b..3812cfdf26 100644 --- a/include/system/memory.h +++ b/include/system/memory.h @@ -2918,6 +2918,39 @@ void address_space_register_map_client(AddressSpace *as, QEMUBH *bh); void address_space_unregister_map_client(AddressSpace *as, QEMUBH *bh); /* Internal functions, part of the implementation of address_space_read. */ + +/** + * qemu_ram_move: move data from or to ramblock + * + * @dst: destination where the data is moved to + * @src: source where the data is moved from + * @n: length of data to be moved + * + * Move @n bytes from @src to @dst, the memory areas may overlap. This + * provides the same semantics as memmove(), plus an additional stronger + * guarantee: if @n is 1, 2 or 4 or 8 bytes, and @src and @dst are both + * naturally aligned for that access size, then both the load and the store + * will be done as a single atomic access (with the semantics of + * qatomic_read() and qatomic_set()). + * + * This is the underlying function that we use to implement accesses by + * a guest vCPU or a device DMA operation to a ram block. The atomic + * guarantee is needed for two major cases: (A) When the ram block is + * backed by a PCI BAR passed through from a host device (and so it might + * be hardware registers that must be accessed exactly once at the right + * width); (B) When an emulated device updates a data structure shared in + * guest memory with guest software (e.g. a network device's set of tx and + * rx descriptor blocks), if a write to memory is accidentally performed + * multiple times then it can break the guest code when it busy polls the + * guest memory. + * + * We don't attempt to perform the exact access when it would be unaligned + * because this can't be done on all host architectures. Although this is + * strictly speaking not doing what would happen on real hardware, we don't + * think there are going to be situations where that matters in practice. + */ +void qemu_ram_move(void *dst, const void *src, size_t n); + MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, void *buf, hwaddr len); MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr, @@ -2935,15 +2968,8 @@ static inline bool memory_region_supports_direct_access(const MemoryRegion *mr) if (memory_region_is_romd(mr)) { return true; } - if (!memory_region_is_ram(mr)) { - return false; - } - /* - * RAM DEVICE regions can be accessed directly using memcpy, but it might - * be MMIO and access using mempy can be wrong (e.g., using instructions not - * intended for MMIO access). So we treat this as IO. - */ - return !memory_region_is_ram_device(mr); + + return memory_region_is_ram(mr); } static inline bool memory_access_is_direct(const MemoryRegion *mr, @@ -2991,7 +3017,7 @@ MemTxResult address_space_read(AddressSpace *as, hwaddr addr, mr = flatview_translate(fv, addr, &addr1, &l, false, attrs); if (len == l && memory_access_is_direct(mr, false, attrs)) { ptr = qemu_map_ram_ptr(mr->ram_block, addr1); - memcpy(buf, ptr, len); + qemu_ram_move(buf, ptr, len); } else { result = flatview_read_continue(fv, addr, attrs, buf, len, addr1, l, mr); diff --git a/system/memory.c b/system/memory.c index f381c99ce5..dbb8df70ba 100644 --- a/system/memory.c +++ b/system/memory.c @@ -1361,43 +1361,6 @@ const MemoryRegionOps unassigned_mem_ops = { .endianness = DEVICE_NATIVE_ENDIAN, }; -static uint64_t memory_region_ram_device_read(void *opaque, - hwaddr addr, unsigned size) -{ - MemoryRegion *mr = opaque; - uint64_t data = ldn_he_p(mr->ram_block->host + addr, size); - - trace_memory_region_ram_device_read(get_cpu_index(), mr, addr, data, size); - - return data; -} - -static void memory_region_ram_device_write(void *opaque, hwaddr addr, - uint64_t data, unsigned size) -{ - MemoryRegion *mr = opaque; - - trace_memory_region_ram_device_write(get_cpu_index(), mr, addr, data, size); - - stn_he_p(mr->ram_block->host + addr, size, data); -} - -static const MemoryRegionOps ram_device_mem_ops = { - .read = memory_region_ram_device_read, - .write = memory_region_ram_device_write, - .endianness = HOST_BIG_ENDIAN ? DEVICE_BIG_ENDIAN : DEVICE_LITTLE_ENDIAN, - .valid = { - .min_access_size = 1, - .max_access_size = 8, - .unaligned = true, - }, - .impl = { - .min_access_size = 1, - .max_access_size = 8, - .unaligned = true, - }, -}; - bool memory_region_access_valid(MemoryRegion *mr, hwaddr addr, unsigned size, @@ -1674,10 +1637,8 @@ void memory_region_init_ram_device_ptr(MemoryRegion *mr, Object *owner, const char *name, uint64_t size, void *ptr) { - memory_region_init_io(mr, owner, &ram_device_mem_ops, mr, name, size); - mr->ram = true; + memory_region_init_ram_ptr(mr, owner, name, size, ptr); mr->ram_device = true; - memory_region_set_ram_ptr(mr, size, ptr); } void memory_region_init_alias(MemoryRegion *mr, Object *owner, diff --git a/system/physmem.c b/system/physmem.c index 4e26f1a1d4..c0f7180ed9 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -3166,6 +3166,50 @@ void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size) invalidate_and_set_dirty(mr, addr, size); } +void qemu_ram_move(void *dst, const void *src, size_t n) +{ + uintptr_t test, len; + + if (n == 0) { + return; + } + + /* + * Calculate "the lowest set bit" over @src, @dst and @n, result put + * into @len (which guarantees a power-of-two). With that and the + * later check (len!=n), it makes sure that we will only do the atomic + * ops when: + * + * (1) @n is a power-of-two + * (2) @src and @dst addresses are both aligned to @n + */ + test = (uintptr_t)src | (uintptr_t)dst | n; + len = test & -test; + + /* Overlapping buffers, unaligned or oversized access */ + if (n > 8 || len != n) { + memmove(dst, src, n); + return; + } + + switch (len) { + case 1: + qatomic_set((uint8_t *)dst, qatomic_read((uint8_t *)src)); + break; + case 2: + qatomic_set((uint16_t *)dst, qatomic_read((uint16_t *)src)); + break; + case 4: + qatomic_set((uint32_t *)dst, qatomic_read((uint32_t *)src)); + break; + case 8: + qatomic_set((uint64_t *)dst, qatomic_read((uint64_t *)src)); + break; + default: + g_assert_not_reached(); + } +} + int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr) { unsigned access_size_max = mr->ops->valid.max_access_size; @@ -3278,7 +3322,7 @@ static MemTxResult flatview_write_continue_step(MemTxAttrs attrs, uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l, false, true); - memmove(ram_ptr, buf, *l); + qemu_ram_move(ram_ptr, buf, *l); invalidate_and_set_dirty(mr, mr_addr, *l); return MEMTX_OK; @@ -3371,7 +3415,7 @@ static MemTxResult flatview_read_continue_step(MemTxAttrs attrs, uint8_t *buf, uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l, false, false); - memcpy(buf, ram_ptr, *l); + qemu_ram_move(buf, ram_ptr, *l); return MEMTX_OK; } diff --git a/system/trace-events b/system/trace-events index 6d29a823f0..0d2b80fc26 100644 --- a/system/trace-events +++ b/system/trace-events @@ -20,8 +20,6 @@ memory_region_ops_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, u memory_region_ops_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size, const char *name) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u name '%s'" memory_region_subpage_read(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset 0x%"PRIx64" value 0x%"PRIx64" size %u" memory_region_subpage_write(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset 0x%"PRIx64" value 0x%"PRIx64" size %u" -memory_region_ram_device_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u" -memory_region_ram_device_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u" memory_region_sync_dirty(const char *mr, const char *listener, int global) "mr '%s' listener '%s' synced (global=%d)" flatview_new(void *view, void *root) "%p (root %p)" flatview_destroy(void *view, void *root) "%p (root %p)"