[Linux] Implement fast mutex similar to Windows version
This commit is contained in:
@@ -10,6 +10,10 @@
|
||||
#include "xenia/base/mutex.h"
|
||||
#if XE_PLATFORM_WIN32 == 1
|
||||
#include "xenia/base/platform_win.h"
|
||||
#elif XE_PLATFORM_LINUX == 1
|
||||
#include <linux/futex.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace xe {
|
||||
@@ -70,6 +74,179 @@ bool xe_fast_mutex::try_lock() {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
#elif XE_PLATFORM_LINUX == 1 && XE_ENABLE_FAST_LINUX_MUTEX == 1
|
||||
|
||||
namespace {
|
||||
|
||||
inline int futex_wait(std::atomic<uint32_t>* addr, uint32_t expected) {
|
||||
return syscall(SYS_futex, addr, FUTEX_WAIT_PRIVATE, expected, nullptr,
|
||||
nullptr, 0);
|
||||
}
|
||||
|
||||
inline int futex_wake(std::atomic<uint32_t>* addr, int count) {
|
||||
return syscall(SYS_futex, addr, FUTEX_WAKE_PRIVATE, count, nullptr, nullptr,
|
||||
0);
|
||||
}
|
||||
|
||||
inline pid_t gettid() { return static_cast<pid_t>(syscall(SYS_gettid)); }
|
||||
|
||||
} // namespace
|
||||
|
||||
// xe_global_mutex implementation (recursive)
|
||||
void xe_global_mutex::lock() {
|
||||
pid_t self = gettid();
|
||||
|
||||
// Fast path: check if we already own it (recursive lock)
|
||||
if (owner_.load(std::memory_order_relaxed) == self) {
|
||||
++recursion_count_;
|
||||
return;
|
||||
}
|
||||
|
||||
// Try to acquire with a simple CAS first (uncontended case)
|
||||
uint32_t expected = 0;
|
||||
if (XE_LIKELY(state_.compare_exchange_strong(
|
||||
expected, 1, std::memory_order_acquire, std::memory_order_relaxed))) {
|
||||
owner_.store(self, std::memory_order_relaxed);
|
||||
recursion_count_ = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
lock_slow();
|
||||
}
|
||||
|
||||
void xe_global_mutex::lock_slow() {
|
||||
pid_t self = gettid();
|
||||
|
||||
// Spin phase
|
||||
for (int i = 0; i < XE_LINUX_MUTEX_SPINCOUNT; ++i) {
|
||||
#if XE_ARCH_AMD64 == 1
|
||||
_mm_pause();
|
||||
#endif
|
||||
uint32_t expected = 0;
|
||||
if (state_.compare_exchange_strong(expected, 1, std::memory_order_acquire,
|
||||
std::memory_order_relaxed)) {
|
||||
owner_.store(self, std::memory_order_relaxed);
|
||||
recursion_count_ = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Slow path: use futex
|
||||
while (true) {
|
||||
// Mark as contended (state = 2) and wait
|
||||
uint32_t state = state_.exchange(2, std::memory_order_acquire);
|
||||
if (state == 0) {
|
||||
// We got the lock while marking contended
|
||||
owner_.store(self, std::memory_order_relaxed);
|
||||
recursion_count_ = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// Wait on futex
|
||||
futex_wait(&state_, 2);
|
||||
|
||||
// Try to acquire after wakeup
|
||||
uint32_t expected = 0;
|
||||
if (state_.compare_exchange_strong(expected, 2, std::memory_order_acquire,
|
||||
std::memory_order_relaxed)) {
|
||||
owner_.store(self, std::memory_order_relaxed);
|
||||
recursion_count_ = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void xe_global_mutex::unlock() {
|
||||
if (--recursion_count_ > 0) {
|
||||
return; // Still have recursive locks
|
||||
}
|
||||
|
||||
owner_.store(0, std::memory_order_relaxed);
|
||||
|
||||
// If state was 2 (contended), we need to wake a waiter
|
||||
if (state_.exchange(0, std::memory_order_release) == 2) {
|
||||
futex_wake(&state_, 1);
|
||||
}
|
||||
}
|
||||
|
||||
bool xe_global_mutex::try_lock() {
|
||||
pid_t self = gettid();
|
||||
|
||||
// Check for recursive lock
|
||||
if (owner_.load(std::memory_order_relaxed) == self) {
|
||||
++recursion_count_;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t expected = 0;
|
||||
if (state_.compare_exchange_strong(expected, 1, std::memory_order_acquire,
|
||||
std::memory_order_relaxed)) {
|
||||
owner_.store(self, std::memory_order_relaxed);
|
||||
recursion_count_ = 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// xe_fast_mutex implementation (non-recursive)
|
||||
void xe_fast_mutex::lock() {
|
||||
// Fast path: uncontended
|
||||
uint32_t expected = 0;
|
||||
if (XE_LIKELY(state_.compare_exchange_strong(
|
||||
expected, 1, std::memory_order_acquire, std::memory_order_relaxed))) {
|
||||
return;
|
||||
}
|
||||
|
||||
lock_slow();
|
||||
}
|
||||
|
||||
void xe_fast_mutex::lock_slow() {
|
||||
// Spin phase
|
||||
for (int i = 0; i < XE_LINUX_MUTEX_SPINCOUNT; ++i) {
|
||||
#if XE_ARCH_AMD64 == 1
|
||||
_mm_pause();
|
||||
#endif
|
||||
uint32_t expected = 0;
|
||||
if (state_.compare_exchange_strong(expected, 1, std::memory_order_acquire,
|
||||
std::memory_order_relaxed)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Slow path: use futex
|
||||
while (true) {
|
||||
// Mark as contended (state = 2) and wait
|
||||
uint32_t state = state_.exchange(2, std::memory_order_acquire);
|
||||
if (state == 0) {
|
||||
// We got the lock while marking contended
|
||||
return;
|
||||
}
|
||||
|
||||
// Wait on futex
|
||||
futex_wait(&state_, 2);
|
||||
|
||||
// Try to acquire after wakeup
|
||||
uint32_t expected = 0;
|
||||
if (state_.compare_exchange_strong(expected, 2, std::memory_order_acquire,
|
||||
std::memory_order_relaxed)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void xe_fast_mutex::unlock() {
|
||||
// If state was 2 (contended), we need to wake a waiter
|
||||
if (state_.exchange(0, std::memory_order_release) == 2) {
|
||||
futex_wake(&state_, 1);
|
||||
}
|
||||
}
|
||||
|
||||
bool xe_fast_mutex::try_lock() {
|
||||
uint32_t expected = 0;
|
||||
return state_.compare_exchange_strong(expected, 1, std::memory_order_acquire,
|
||||
std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
#endif
|
||||
global_mutex_type& global_critical_region::mutex() {
|
||||
static global_mutex_type global_mutex;
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
|
||||
#ifndef XENIA_BASE_MUTEX_H_
|
||||
#define XENIA_BASE_MUTEX_H_
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include "platform.h"
|
||||
@@ -19,6 +20,7 @@
|
||||
#endif
|
||||
#include "memory.h"
|
||||
#define XE_ENABLE_FAST_WIN32_MUTEX 1
|
||||
#define XE_ENABLE_FAST_LINUX_MUTEX 1
|
||||
namespace xe {
|
||||
|
||||
#if XE_PLATFORM_WIN32 == 1 && XE_ENABLE_FAST_WIN32_MUTEX == 1
|
||||
@@ -80,6 +82,78 @@ class xe_unlikely_mutex {
|
||||
void unlock() { mut.exchange(0); }
|
||||
bool try_lock() { return _tryget(); }
|
||||
};
|
||||
using xe_mutex = xe_fast_mutex;
|
||||
#elif XE_PLATFORM_LINUX == 1 && XE_ENABLE_FAST_LINUX_MUTEX == 1
|
||||
|
||||
#define XE_LINUX_MUTEX_SPINCOUNT 128
|
||||
|
||||
// Fast recursive mutex for Linux using futex
|
||||
// Mimics Windows CRITICAL_SECTION behavior: spin before blocking
|
||||
class alignas(4096) xe_global_mutex {
|
||||
std::atomic<uint32_t> state_{0}; // 0 = unlocked, 1 = locked, 2 = contended
|
||||
std::atomic<pid_t> owner_{0};
|
||||
uint32_t recursion_count_{0};
|
||||
|
||||
void lock_slow();
|
||||
|
||||
public:
|
||||
xe_global_mutex() = default;
|
||||
~xe_global_mutex() = default;
|
||||
|
||||
void lock();
|
||||
void unlock();
|
||||
bool try_lock();
|
||||
};
|
||||
using global_mutex_type = xe_global_mutex;
|
||||
|
||||
// Fast non-recursive mutex for Linux using futex
|
||||
class alignas(64) xe_fast_mutex {
|
||||
std::atomic<uint32_t> state_{0}; // 0 = unlocked, 1 = locked, 2 = contended
|
||||
|
||||
void lock_slow();
|
||||
|
||||
public:
|
||||
xe_fast_mutex() = default;
|
||||
~xe_fast_mutex() = default;
|
||||
|
||||
void lock();
|
||||
void unlock();
|
||||
bool try_lock();
|
||||
};
|
||||
|
||||
// xe_unlikely_mutex remains a simple spinlock for Linux too
|
||||
class xe_unlikely_mutex {
|
||||
std::atomic<uint32_t> mut{0};
|
||||
bool _tryget() {
|
||||
uint32_t lock_expected = 0;
|
||||
return mut.compare_exchange_strong(
|
||||
lock_expected, 1, std::memory_order_acquire, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
public:
|
||||
xe_unlikely_mutex() = default;
|
||||
~xe_unlikely_mutex() = default;
|
||||
|
||||
void lock() {
|
||||
if (XE_LIKELY(_tryget())) {
|
||||
return;
|
||||
}
|
||||
// Spin a bit before yielding
|
||||
for (int i = 0; i < XE_LINUX_MUTEX_SPINCOUNT; ++i) {
|
||||
#if XE_ARCH_AMD64 == 1
|
||||
_mm_pause();
|
||||
#endif
|
||||
if (_tryget()) return;
|
||||
}
|
||||
// Fall back to yielding
|
||||
while (!_tryget()) {
|
||||
std::this_thread::yield();
|
||||
}
|
||||
}
|
||||
void unlock() { mut.store(0, std::memory_order_release); }
|
||||
bool try_lock() { return _tryget(); }
|
||||
};
|
||||
|
||||
using xe_mutex = xe_fast_mutex;
|
||||
#else
|
||||
using global_mutex_type = std::recursive_mutex;
|
||||
@@ -150,11 +224,7 @@ class global_critical_region {
|
||||
return global_unique_lock_type(mutex());
|
||||
}
|
||||
|
||||
static inline void PrepareToAcquire() {
|
||||
#if XE_PLATFORM_WIN32 == 1
|
||||
swcache::PrefetchW(&mutex());
|
||||
#endif
|
||||
}
|
||||
static inline void PrepareToAcquire() { swcache::PrefetchW(&mutex()); }
|
||||
|
||||
// Acquires a deferred lock on the global critical section.
|
||||
static inline global_unique_lock_type AcquireDeferred() {
|
||||
|
||||
Reference in New Issue
Block a user