- generalize mutex lock spinning

lockperf.sh

@@ -2,15 +2,21 @@
 
 TYPES=(
   "p:pthread:FredMutex"
-    "f:cond:SpinCondMutex<WorkerLock, 0, 0, 0>"
-  "f:pscond:SpinCondMutex<WorkerLock, 4, 1024, 16, PauseSpin>"
-  "f:yscond:SpinCondMutex<WorkerLock, 4, 1024, 16, YieldSpin>"
-    "f:fibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 0, 0, 0>"
-  "f:psfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 4, 1024, 16, PauseSpin>"
-  "f:ysfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 4, 1024, 16, YieldSpin>"
-    "f:fast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 0, 0, 0>"
-  "f:psfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 4, 1024, 16, PauseSpin>"
-  "f:ysfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 4, 1024, 16, YieldSpin>"
+    "f:cond:SpinCondMutex<WorkerLock, 0, 0, 0, 0>"
+   "f:xcond:SpinCondMutex<WorkerLock, 1, 64, 1, 0>"
+   "f:ycond:SpinCondMutex<WorkerLock, 1, 64, 1, 1>"
+  "f:pscond:SpinCondMutex<WorkerLock, 4, 1024, 16, 0, PauseSpin>"
+  "f:yscond:SpinCondMutex<WorkerLock, 4, 1024, 16, 0, YieldSpin>"
+    "f:fibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 0, 0, 0, 0>"
+   "f:xfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 1, 64, 1, 0>"
+   "f:yfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 1, 64, 1, 1>"
+  "f:psfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 4, 1024, 16, 0, PauseSpin>"
+  "f:ysfibre:SpinSemMutex<LockedSemaphore<WorkerLock, true>, 4, 1024, 16, 0, YieldSpin>"
+    "f:fast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 0, 0, 0, 0>"
+   "f:xfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 1, 64, 1, 0>"
+   "f:yfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 1, 64, 1, 1>"
+  "f:psfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 4, 1024, 16, 0, PauseSpin>"
+  "f:ysfast:SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 4, 1024, 16, 0, YieldSpin>"
 #  "f:fifo:LockedMutex<WorkerLock, true>"
 #  "f:simple:SimpleMutex0<false>"
 #  "f:direct:SimpleMutex0<true>"

src/runtime/BlockingSync.h

@@ -659,11 +659,31 @@ public:
   }
 };
 
-template<typename Semaphore, int SpinStart, int SpinEnd, int SpinCount, typename SpinOp = PauseSpin>
+template<int SpinStart, int SpinEnd, int SpinCount, int YieldCount, typename SpinOp, typename T, typename Func>
+static inline bool Spin(Fred* cf, T* This, Func&& tryLock) {
+  SpinOp spinOp;
+  int ycnt = 0;
+  int scnt = 0;
+  int spin = SpinStart;
+  for (;;) {
+    if (tryLock(This, cf)) return false;
+    if (scnt < SpinCount) {
+      for (int i = 0; i < spin; i += 1) spinOp();
+      if (spin < SpinEnd) spin += spin;
+      else scnt += 1;
+    } else if (ycnt < YieldCount) {
+      ycnt += 1;
+      Fred::yield();
+    } else {
+      return true;
+    }
+  }
+}
+
+template<typename Semaphore, int SpinStart, int SpinEnd, int SpinCount, int YieldCount, typename SpinOp = PauseSpin>
 class SpinSemMutex {
   Fred* volatile owner;
   Semaphore sem;
-  SpinOp spinOp;
 
   template<typename... Args>
   bool tryOnly(const Args&... args) { return false; }
@@ -671,9 +691,9 @@ class SpinSemMutex {
   template<typename... Args>
   bool tryOnly(bool wait) { return !wait; }
 
-  bool tryLock(Fred* cf) {
+  static bool tryLock(SpinSemMutex* This, Fred* cf) {
     Fred* exp = nullptr;
-    return __atomic_compare_exchange_n(&owner, &exp, cf, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+    return __atomic_compare_exchange_n(&This->owner, &exp, cf, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
   }
 
 protected:
@@ -682,21 +702,11 @@ protected:
     Fred* cf = Context::CurrFred();
     if (OwnerLock && cf == owner) return true;
     RASSERT(cf != owner, FmtHex(cf), FmtHex(owner));
-    if (tryOnly(args...)) return tryLock(cf);
-    int cnt = 0;
-    int spin = SpinStart;
-    for (;;) {
-      if (tryLock(cf)) return true;
-      if (cnt < SpinCount) {
-        for (int i = 0; i < spin; i += 1) spinOp();
-        if (spin < SpinEnd) spin += spin;
-        else cnt += 1;
-      } else {
-        cnt = 0;
-        spin = SpinStart;
-        if (!sem.P(args...)) return false;
-      }
+    if (tryOnly(args...)) return tryLock(this, cf);
+    while (Spin<SpinStart,SpinEnd,SpinCount,YieldCount,SpinOp>(cf, this, tryLock)) {
+      if (!sem.P(args...)) return false;
     }
+    return true;
   }
 
 public:
@@ -742,12 +752,11 @@ public:
 };
 
 // inspired by Linux pthread mutex/futex implementation
-template<typename Lock, int SpinStart, int SpinEnd, int SpinCount, typename SpinOp = PauseSpin>
+template<typename Lock, int SpinStart, int SpinEnd, int SpinCount, int YieldCount, typename SpinOp = PauseSpin>
 class SpinCondMutex {
   ConditionalQueue<Lock> queue;
   volatile size_t value;
   Fred* volatile owner;
-  SpinOp spinOp;
 
   template<typename... Args>
   bool tryOnly(const Args&... args) { return false; }
@@ -755,6 +764,14 @@ class SpinCondMutex {
   template<typename... Args>
   bool tryOnly(bool wait) { return !wait; }
 
+  static bool tryLock(SpinCondMutex* This, Fred* cf) {
+    if (__atomic_exchange_n(&This->value, 2, __ATOMIC_ACQUIRE) == 0) {
+      This->owner = cf;
+      return true;
+    }
+    return false;
+  }
+
 protected:
   template<bool OwnerLock, typename... Args>
   bool internalAcquire(const Args&... args) {
@@ -767,25 +784,11 @@ protected:
       return true;
     }
     if (tryOnly(args...)) return false;
-    int cnt = 0;
-    int spin = SpinStart;
-    if (exp > 1) goto skipFirstTest;
-    for (;;) {
-      if (__atomic_exchange_n(&value, 2, __ATOMIC_ACQUIRE) == 0) {
-        owner = cf;
-        return true;
-      }
-skipFirstTest:
-      if (cnt < SpinCount) {
-        for (int i = 0; i < spin; i += 1) spinOp();
-        if (spin < SpinEnd) spin += spin;
-        else cnt += 1;
-      } else {
-        cnt = 0;
-        spin = SpinStart;
-        queue.block(cf, [this]() { return this->value == 2; }, args...);
-      }
+    if (exp < 2 && tryLock(this, cf)) return true;
+    while (Spin<SpinStart,SpinEnd,SpinCount,YieldCount,SpinOp>(cf, this, tryLock)) {
+      queue.block(cf, [this]() { return this->value == 2; }, args...);
     }
+    return true;
   }
 
 public:
@@ -836,13 +839,13 @@ public:
 #if defined(FAST_MUTEX_TYPE)
 typedef FAST_MUTEX_TYPE FastMutex;
 #else
-typedef SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 0, 0, 0> FastMutex;
+typedef SpinSemMutex<FredBenaphore<LimitedSemaphore0<MCSLock>,true>, 1, 64, 1, 1> FastMutex;
 #endif
 
 #if defined(FRED_MUTEX_TYPE)
 typedef FRED_MUTEX_TYPE FredMutex;
 #else
-typedef SpinCondMutex<WorkerLock, 0, 0, 0> FredMutex;
+typedef SpinCondMutex<WorkerLock, 1, 64, 1, 1> FredMutex;
 #endif
 
 typedef Condition<>                 FredCondition;

src/runtime/SpinLocks.h

@@ -186,7 +186,7 @@ public:
   bool wait() {
     size_t cnt = __atomic_fetch_add(&counter, 1, __ATOMIC_SEQ_CST);
     size_t tgt = cnt + target - (cnt % target);
-    while (ssize_t(tgt - counter) > 0) Pause(); // works with overflow
+    while slowpath(ssize_t(tgt - counter) > 0) Pause(); // works with overflow
     return (cnt == tgt - 1);
   }
   void cleanup() {}