diff --git a/arch/wasm/Kconfig b/arch/wasm/Kconfig index acb19dbac7318d..b6f1490a70fc38 100644 --- a/arch/wasm/Kconfig +++ b/arch/wasm/Kconfig @@ -5,8 +5,6 @@ config WASM select ARCH_WANTS_BUDDY_ALLOCATOR_BOTTOM_UP select ARCH_NO_PREEMPT select ARCH_NO_SWAP - select ARCH_USE_QUEUED_RWLOCKS - select ARCH_USE_QUEUED_SPINLOCKS select BUG select FLATMEM select GENERIC_CPU_DEVICES diff --git a/arch/wasm/include/asm/spinlock.h b/arch/wasm/include/asm/spinlock.h index 0ca66855247624..a659838684ffc8 100644 --- a/arch/wasm/include/asm/spinlock.h +++ b/arch/wasm/include/asm/spinlock.h @@ -1,3 +1,165 @@ -#include -#include -#define smp_mb__after_spinlock() smp_mb() +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_WASM_SPINLOCK_H +#define _ASM_WASM_SPINLOCK_H + +#include +#include + +#include + +static __always_inline u32 wasm_lock_load(const u32 *word) +{ + return __atomic_load_n(word, __ATOMIC_SEQ_CST); +} + +static __always_inline void wasm_lock_wait(u32 *word, u32 value) +{ + __builtin_wasm_memory_atomic_wait32((int *)word, value, -1); +} + +static __always_inline void wasm_lock_wake_all(u32 *word) +{ + __builtin_wasm_memory_atomic_notify((int *)word, INT_MAX); +} + +static __always_inline int arch_spin_is_locked(arch_spinlock_t *lock) +{ + return wasm_lock_load(&lock->owner) != wasm_lock_load(&lock->next); +} + +static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) +{ + return lock.owner == lock.next; +} + +static __always_inline int arch_spin_is_contended(arch_spinlock_t *lock) +{ + u32 owner = wasm_lock_load(&lock->owner); + u32 next = wasm_lock_load(&lock->next); + + return (u32)(next - owner) > 1; +} +#define arch_spin_is_contended arch_spin_is_contended + +static __always_inline void arch_spin_lock(arch_spinlock_t *lock) +{ + u32 ticket = __atomic_fetch_add(&lock->next, 1, __ATOMIC_SEQ_CST); + u32 owner; + + while ((owner = wasm_lock_load(&lock->owner)) != ticket) + wasm_lock_wait(&lock->owner, owner); +} + +static __always_inline int arch_spin_trylock(arch_spinlock_t *lock) +{ + u32 owner = wasm_lock_load(&lock->owner); + u32 next = wasm_lock_load(&lock->next); + + if (owner != next) + return 0; + + return __atomic_compare_exchange_n(&lock->next, &next, next + 1, + false, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST); +} + +static __always_inline void arch_spin_unlock(arch_spinlock_t *lock) +{ + __atomic_fetch_add(&lock->owner, 1, __ATOMIC_SEQ_CST); + wasm_lock_wake_all(&lock->owner); +} + +static __always_inline void arch_read_lock(arch_rwlock_t *lock) +{ + for (;;) { + u32 owner = wasm_lock_load(&lock->writer_owner); + + if (owner != wasm_lock_load(&lock->writer_next)) { + wasm_lock_wait(&lock->writer_owner, owner); + continue; + } + + __atomic_fetch_add(&lock->readers, 1, __ATOMIC_SEQ_CST); + if (wasm_lock_load(&lock->writer_owner) == + wasm_lock_load(&lock->writer_next)) + return; + + if (__atomic_fetch_sub(&lock->readers, 1, + __ATOMIC_SEQ_CST) == 1) + wasm_lock_wake_all(&lock->readers); + } +} + +static __always_inline int arch_read_trylock(arch_rwlock_t *lock) +{ + if (wasm_lock_load(&lock->writer_owner) != + wasm_lock_load(&lock->writer_next)) + return 0; + + __atomic_fetch_add(&lock->readers, 1, __ATOMIC_SEQ_CST); + if (wasm_lock_load(&lock->writer_owner) == + wasm_lock_load(&lock->writer_next)) + return 1; + + if (__atomic_fetch_sub(&lock->readers, 1, __ATOMIC_SEQ_CST) == 1) + wasm_lock_wake_all(&lock->readers); + return 0; +} + +static __always_inline void arch_read_unlock(arch_rwlock_t *lock) +{ + if (__atomic_fetch_sub(&lock->readers, 1, __ATOMIC_SEQ_CST) == 1) + wasm_lock_wake_all(&lock->readers); +} + +static __always_inline void arch_write_lock(arch_rwlock_t *lock) +{ + u32 ticket = __atomic_fetch_add(&lock->writer_next, 1, + __ATOMIC_SEQ_CST); + u32 value; + + while ((value = wasm_lock_load(&lock->writer_owner)) != ticket) + wasm_lock_wait(&lock->writer_owner, value); + + while ((value = wasm_lock_load(&lock->readers)) != 0) + wasm_lock_wait(&lock->readers, value); +} + +static __always_inline int arch_write_trylock(arch_rwlock_t *lock) +{ + u32 owner = wasm_lock_load(&lock->writer_owner); + u32 next = wasm_lock_load(&lock->writer_next); + + if (owner != next) + return 0; + + if (!__atomic_compare_exchange_n(&lock->writer_next, &next, next + 1, + false, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST)) + return 0; + + if (wasm_lock_load(&lock->readers) == 0) + return 1; + + __atomic_fetch_add(&lock->writer_owner, 1, __ATOMIC_SEQ_CST); + wasm_lock_wake_all(&lock->writer_owner); + return 0; +} + +static __always_inline void arch_write_unlock(arch_rwlock_t *lock) +{ + __atomic_fetch_add(&lock->writer_owner, 1, __ATOMIC_SEQ_CST); + wasm_lock_wake_all(&lock->writer_owner); +} + +static __always_inline int arch_rwlock_is_contended(arch_rwlock_t *lock) +{ + return wasm_lock_load(&lock->writer_owner) != + wasm_lock_load(&lock->writer_next); +} +#define arch_rwlock_is_contended arch_rwlock_is_contended + +/* Upgrade the lock acquisition to the ordering required by the scheduler. */ +#define smp_mb__after_spinlock() smp_mb() + +#endif /* _ASM_WASM_SPINLOCK_H */ diff --git a/arch/wasm/include/asm/spinlock_types.h b/arch/wasm/include/asm/spinlock_types.h index 78fa87290761cd..080044f8d44bec 100644 --- a/arch/wasm/include/asm/spinlock_types.h +++ b/arch/wasm/include/asm/spinlock_types.h @@ -1,2 +1,32 @@ -#include -#include \ No newline at end of file +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_WASM_SPINLOCK_TYPES_H +#define _ASM_WASM_SPINLOCK_TYPES_H + +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H +#error "Please do not include this file directly." +#endif + +#include + +/* + * Keep every concurrently accessed field in its own 32-bit word. Wasm only + * guarantees sequential consistency between atomic accesses to identical + * byte ranges, so packed ticket or queued locks are not suitable here. + */ +typedef struct { + u32 owner; + u32 next; +} arch_spinlock_t; + +#define __ARCH_SPIN_LOCK_UNLOCKED { .owner = 0, .next = 0 } + +typedef struct { + u32 readers; + u32 writer_owner; + u32 writer_next; +} arch_rwlock_t; + +#define __ARCH_RW_LOCK_UNLOCKED \ + { .readers = 0, .writer_owner = 0, .writer_next = 0 } + +#endif /* _ASM_WASM_SPINLOCK_TYPES_H */