Commit 3359c639 authored by Martin Karsten's avatar Martin Karsten
Browse files

- generalize mutex lock spinning

parent 808f6154
Pipeline #50721 failed with stage
......@@ -2,15 +2,21 @@
TYPES=(
"p:pthread:FredMutex"
"f:cond:SpinCondMutex<WorkerLock, 0, 0, 0>"
"f:pscond:SpinCondMutex<WorkerLock, 4, 1024, 16, PauseSpin>"
"f:yscond:SpinCondMutex<WorkerLock, 4, 1024, 16, YieldSpin>"
"f:fibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 0, 0, 0>"
"f:psfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 4, 1024, 16, PauseSpin>"
"f:ysfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 4, 1024, 16, YieldSpin>"
"f:fast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 0, 0, 0>"
"f:psfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 4, 1024, 16, PauseSpin>"
"f:ysfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 4, 1024, 16, YieldSpin>"
"f:cond:SpinCondMutex<WorkerLock, 0, 0, 0, 0>"
"f:xcond:SpinCondMutex<WorkerLock, 1, 64, 1, 0>"
"f:ycond:SpinCondMutex<WorkerLock, 1, 64, 1, 1>"
"f:pscond:SpinCondMutex<WorkerLock, 4, 1024, 16, 0, PauseSpin>"
"f:yscond:SpinCondMutex<WorkerLock, 4, 1024, 16, 0, YieldSpin>"
"f:fibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 0, 0, 0, 0>"
"f:xfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 1, 64, 1, 0>"
"f:yfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 1, 64, 1, 1>"
"f:psfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 4, 1024, 16, 0, PauseSpin>"
"f:ysfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 4, 1024, 16, 0, YieldSpin>"
"f:fast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 0, 0, 0, 0>"
"f:xfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 1, 64, 1, 0>"
"f:yfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 1, 64, 1, 1>"
"f:psfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 4, 1024, 16, 0, PauseSpin>"
"f:ysfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 4, 1024, 16, 0, YieldSpin>"
# "f:fifo:LockedMutex<WorkerLock, true>"
# "f:simple:SimpleMutex0<false>"
# "f:direct:SimpleMutex0<true>"
......
......@@ -659,11 +659,31 @@ public:
}
};
template<typename Semaphore, int SpinStart, int SpinEnd, int SpinCount, typename SpinOp = PauseSpin>
template<int SpinStart, int SpinEnd, int SpinCount, int YieldCount, typename SpinOp, typename T, typename Func>
static inline bool Spin(Fred* cf, T* This, Func&& tryLock) {
SpinOp spinOp;
int ycnt = 0;
int scnt = 0;
int spin = SpinStart;
for (;;) {
if (tryLock(This, cf)) return false;
if (scnt < SpinCount) {
for (int i = 0; i < spin; i += 1) spinOp();
if (spin < SpinEnd) spin += spin;
else scnt += 1;
} else if (ycnt < YieldCount) {
ycnt += 1;
Fred::yield();
} else {
return true;
}
}
}
template<typename Semaphore, int SpinStart, int SpinEnd, int SpinCount, int YieldCount, typename SpinOp = PauseSpin>
class SpinSemMutex {
Fred* volatile owner;
Semaphore sem;
SpinOp spinOp;
template<typename... Args>
bool tryOnly(const Args&... args) { return false; }
......@@ -671,9 +691,9 @@ class SpinSemMutex {
template<typename... Args>
bool tryOnly(bool wait) { return !wait; }
bool tryLock(Fred* cf) {
static bool tryLock(SpinSemMutex* This, Fred* cf) {
Fred* exp = nullptr;
return __atomic_compare_exchange_n(&owner, &exp, cf, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
return __atomic_compare_exchange_n(&This->owner, &exp, cf, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
protected:
......@@ -682,21 +702,11 @@ protected:
Fred* cf = Context::CurrFred();
if (OwnerLock && cf == owner) return true;
RASSERT(cf != owner, FmtHex(cf), FmtHex(owner));
if (tryOnly(args...)) return tryLock(cf);
int cnt = 0;
int spin = SpinStart;
for (;;) {
if (tryLock(cf)) return true;
if (cnt < SpinCount) {
for (int i = 0; i < spin; i += 1) spinOp();
if (spin < SpinEnd) spin += spin;
else cnt += 1;
} else {
cnt = 0;
spin = SpinStart;
if (!sem.P(args...)) return false;
}
if (tryOnly(args...)) return tryLock(this, cf);
while (Spin<SpinStart,SpinEnd,SpinCount,YieldCount,SpinOp>(cf, this, tryLock)) {
if (!sem.P(args...)) return false;
}
return true;
}
public:
......@@ -742,12 +752,11 @@ public:
};
// inspired by Linux pthread mutex/futex implementation
template<typename Lock, int SpinStart, int SpinEnd, int SpinCount, typename SpinOp = PauseSpin>
template<typename Lock, int SpinStart, int SpinEnd, int SpinCount, int YieldCount, typename SpinOp = PauseSpin>
class SpinCondMutex {
ConditionalQueue<Lock> queue;
volatile size_t value;
Fred* volatile owner;
SpinOp spinOp;
template<typename... Args>
bool tryOnly(const Args&... args) { return false; }
......@@ -755,6 +764,14 @@ class SpinCondMutex {
template<typename... Args>
bool tryOnly(bool wait) { return !wait; }
static bool tryLock(SpinCondMutex* This, Fred* cf) {
if (__atomic_exchange_n(&This->value, 2, __ATOMIC_ACQUIRE) == 0) {
This->owner = cf;
return true;
}
return false;
}
protected:
template<bool OwnerLock, typename... Args>
bool internalAcquire(const Args&... args) {
......@@ -767,25 +784,11 @@ protected:
return true;
}
if (tryOnly(args...)) return false;
int cnt = 0;
int spin = SpinStart;
if (exp > 1) goto skipFirstTest;
for (;;) {
if (__atomic_exchange_n(&value, 2, __ATOMIC_ACQUIRE) == 0) {
owner = cf;
return true;
}
skipFirstTest:
if (cnt < SpinCount) {
for (int i = 0; i < spin; i += 1) spinOp();
if (spin < SpinEnd) spin += spin;
else cnt += 1;
} else {
cnt = 0;
spin = SpinStart;
queue.block(cf, [this]() { return this->value == 2; }, args...);
}
if (exp < 2 && tryLock(this, cf)) return true;
while (Spin<SpinStart,SpinEnd,SpinCount,YieldCount,SpinOp>(cf, this, tryLock)) {
queue.block(cf, [this]() { return this->value == 2; }, args...);
}
return true;
}
public:
......@@ -836,13 +839,13 @@ public:
#if defined(FAST_MUTEX_TYPE)
typedef FAST_MUTEX_TYPE FastMutex;
#else
typedef SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 0, 0, 0> FastMutex;
typedef SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 1, 64, 1, 1> FastMutex;
#endif
#if defined(FRED_MUTEX_TYPE)
typedef FRED_MUTEX_TYPE FredMutex;
#else
typedef SpinCondMutex<WorkerLock, 0, 0, 0> FredMutex;
typedef SpinCondMutex<WorkerLock, 1, 64, 1, 1> FredMutex;
#endif
typedef Condition<> FredCondition;
......
......@@ -186,7 +186,7 @@ public:
bool wait() {
size_t cnt = __atomic_fetch_add(&counter, 1, __ATOMIC_SEQ_CST);
size_t tgt = cnt + target - (cnt % target);
while (ssize_t(tgt - counter) > 0) Pause(); // works with overflow
while slowpath(ssize_t(tgt - counter) > 0) Pause(); // works with overflow
return (cnt == tgt - 1);
}
void cleanup() {}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment