--- /usr/src/sys/conf/options 2009-06-17 00:59:51.000000000 +0200 +++ usr/src/sys/conf/options 2009-06-07 03:49:08.000000000 +0200 @@ -131,6 +131,7 @@ MFI_DEBUG opt_mfi.h MFI_DECODE_LOG opt_mfi.h MPROF_BUFFERS opt_mprof.h MPROF_HASH_SIZE opt_mprof.h +NO_ADAPTIVE_LOCKMGRS NO_ADAPTIVE_MUTEXES opt_adaptive_mutexes.h NO_ADAPTIVE_RWLOCKS NO_ADAPTIVE_SX --- /usr/src/sys/kern/kern_lock.c 2009-06-02 16:43:21.000000000 +0200 +++ usr/src/sys/kern/kern_lock.c 2009-06-17 00:44:14.000000000 +0200 @@ -28,12 +28,14 @@ #include "opt_ddb.h" #include "opt_kdtrace.h" +#include "opt_no_adaptive_lockmgrs.h" #include __FBSDID("$FreeBSD: head/sys/kern/kern_lock.c 193307 2009-06-02 13:03:35Z attilio $"); #include #include +#include #include #include #include @@ -43,6 +45,7 @@ __FBSDID("$FreeBSD: head/sys/kern/kern_l #ifdef DEBUG_LOCKS #include #endif +#include #include #include @@ -51,7 +54,14 @@ __FBSDID("$FreeBSD: head/sys/kern/kern_l #include #endif -CTASSERT((LK_NOSHARE & LO_CLASSFLAGS) == LK_NOSHARE); +CTASSERT(((LK_NOADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) == + (LK_NOADAPTIVE | LK_NOSHARE)); +CTASSERT(LK_UNLOCKED == (LK_UNLOCKED & + ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS))); + +#if defined(SMP) && !defined(NO_ADAPTIVE_LOCKMGRS) +#define ADAPTIVE_LOCKMGRS +#endif #define SQ_EXCLUSIVE_QUEUE 0 #define SQ_SHARED_QUEUE 1 @@ -106,6 +116,7 @@ CTASSERT((LK_NOSHARE & LO_CLASSFLAGS) == #define LK_CAN_SHARE(x) \ (((x) & LK_SHARE) && (((x) & LK_EXCLUSIVE_WAITERS) == 0 || \ + ((x) & LK_EXCLUSIVE_SPINNERS) == 0 || \ curthread->td_lk_slocks || (curthread->td_pflags & TDP_DEADLKTREAT))) #define LK_TRYOP(x) \ ((x) & LK_NOWAIT) @@ -115,6 +126,10 @@ CTASSERT((LK_NOSHARE & LO_CLASSFLAGS) == #define LK_TRYWIT(x) \ (LK_TRYOP(x) ? LOP_TRYLOCK : 0) +#define LK_CAN_ADAPT(lk, f) \ + (((lk)->lock_object.lo_flags & LK_NOADAPTIVE) == 0 && \ + ((f) & LK_SLEEPFAIL) == 0) + #define lockmgr_disowned(lk) \ (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC) @@ -145,6 +160,14 @@ struct lock_class lock_class_lockmgr = { #endif }; +#ifdef ADAPTIVE_LOCKMGRS +static u_int alk_retries = 10; +static u_int alk_loops = 10000; +SYSCTL_NODE(_debug, OID_AUTO, lockmgr, CTLFLAG_RD, NULL, "lockmgr debugging"); +SYSCTL_UINT(_debug_lockmgr, OID_AUTO, retries, CTLFLAG_RW, &alk_retries, 0, ""); +SYSCTL_UINT(_debug_lockmgr, OID_AUTO, loops, CTLFLAG_RW, &alk_loops, 0, ""); +#endif + static __inline struct thread * lockmgr_xholder(struct lock *lk) { @@ -233,9 +256,9 @@ wakeupshlk(struct lock *lk, const char * * lock quickly. */ if ((x & LK_ALL_WAITERS) == 0) { - MPASS(x == LK_SHARERS_LOCK(1)); - if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1), - LK_UNLOCKED)) + MPASS((x & ~LK_EXCLUSIVE_SPINNERS) == + LK_SHARERS_LOCK(1)); + if (atomic_cmpset_ptr(&lk->lk_lock, x, LK_UNLOCKED)) break; continue; } @@ -245,7 +268,7 @@ wakeupshlk(struct lock *lk, const char * * path in order to handle wakeups correctly. */ sleepq_lock(&lk->lock_object); - x = lk->lk_lock & LK_ALL_WAITERS; + x = lk->lk_lock & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); v = LK_UNLOCKED; /* @@ -256,7 +279,8 @@ wakeupshlk(struct lock *lk, const char * queue = SQ_EXCLUSIVE_QUEUE; v |= (x & LK_SHARED_WAITERS); } else { - MPASS(x == LK_SHARED_WAITERS); + MPASS((x & ~LK_EXCLUSIVE_SPINNERS) == + LK_SHARED_WAITERS); queue = SQ_SHARED_QUEUE; } @@ -326,7 +350,7 @@ lockinit(struct lock *lk, int pri, const iflags |= LO_WITNESS; if (flags & LK_QUIET) iflags |= LO_QUIET; - iflags |= flags & LK_NOSHARE; + iflags |= flags & (LK_NOADAPTIVE | LK_NOSHARE); lk->lk_lock = LK_UNLOCKED; lk->lk_recurse = 0; @@ -359,6 +383,10 @@ __lockmgr_args(struct lock *lk, u_int fl uint64_t waittime = 0; int contested = 0; #endif +#ifdef ADAPTIVE_LOCKMGRS + volatile struct thread *owner; + u_int i, spintries = 0; +#endif error = 0; tid = (uintptr_t)curthread; @@ -436,6 +464,59 @@ __lockmgr_args(struct lock *lk, u_int fl break; } +#ifdef ADAPTIVE_LOCKMGRS + /* + * If the owner is running on another CPU, spin until + * the owner stops running or the state of the lock + * changes. + */ + if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 && + LK_HOLDER(x) != LK_KERNPROC) { + owner = (struct thread *)LK_HOLDER(x); + if (LOCK_LOG_TEST(&lk->lock_object, 0)) + CTR3(KTR_LOCK, + "%s: spinning on %p held by %p", + __func__, lk, owner); + + /* + * If we are holding also an interlock drop it + * in order to avoid a deadlock if the lockmgr + * owner is adaptively spinning on the + * interlock itself. + */ + if (flags & LK_INTERLOCK) { + class->lc_unlock(ilk); + flags &= ~LK_INTERLOCK; + } + GIANT_SAVE(); + while (LK_HOLDER(lk->lk_lock) == + (uintptr_t)owner && TD_IS_RUNNING(owner)) + cpu_spinwait(); + } else if (LK_CAN_ADAPT(lk, flags) && + (x & LK_SHARE) !=0 && LK_SHARERS(x) && + spintries < alk_retries) { + if (flags & LK_INTERLOCK) { + class->lc_unlock(ilk); + flags &= ~LK_INTERLOCK; + } + GIANT_SAVE(); + spintries++; + for (i = 0; i < alk_loops; i++) { + if (LOCK_LOG_TEST(&lk->lock_object, 0)) + CTR4(KTR_LOCK, + "%s: shared spinning on %p with %u and %u", + __func__, lk, spintries, i); + x = lk->lk_lock; + if ((x & LK_SHARE) == 0 || + LK_CAN_SHARE(x) != 0) + break; + cpu_spinwait(); + } + if (i != alk_loops) + continue; + } +#endif + /* * Acquire the sleepqueue chain lock because we * probabilly will need to manipulate waiters flags. @@ -452,6 +533,24 @@ __lockmgr_args(struct lock *lk, u_int fl continue; } +#ifdef ADAPTIVE_LOCKMGRS + /* + * The current lock owner might have started executing + * on another CPU (or the lock could have changed + * owner) while we were waiting on the turnstile + * chain lock. If so, drop the turnstile lock and try + * again. + */ + if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 && + LK_HOLDER(x) != LK_KERNPROC) { + owner = (struct thread *)LK_HOLDER(x); + if (TD_IS_RUNNING(owner)) { + sleepq_release(&lk->lock_object); + continue; + } + } +#endif + /* * Try to set the LK_SHARED_WAITERS flag. If we fail, * loop back and retry. @@ -497,13 +596,15 @@ __lockmgr_args(struct lock *lk, u_int fl break; case LK_UPGRADE: _lockmgr_assert(lk, KA_SLOCKED, file, line); - x = lk->lk_lock & LK_ALL_WAITERS; + v = lk->lk_lock; + x = v & LK_ALL_WAITERS; + v &= LK_EXCLUSIVE_SPINNERS; /* * Try to switch from one shared lock to an exclusive one. * We need to preserve waiters flags during the operation. */ - if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x, + if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v, tid | x)) { LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, line); @@ -575,13 +676,69 @@ __lockmgr_args(struct lock *lk, u_int fl break; } +#ifdef ADAPTIVE_LOCKMGRS + /* + * If the owner is running on another CPU, spin until + * the owner stops running or the state of the lock + * changes. + */ + x = lk->lk_lock; + if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 && + LK_HOLDER(x) != LK_KERNPROC) { + owner = (struct thread *)LK_HOLDER(x); + if (LOCK_LOG_TEST(&lk->lock_object, 0)) + CTR3(KTR_LOCK, + "%s: spinning on %p held by %p", + __func__, lk, owner); + + /* + * If we are holding also an interlock drop it + * in order to avoid a deadlock if the lockmgr + * owner is adaptively spinning on the + * interlock itself. + */ + if (flags & LK_INTERLOCK) { + class->lc_unlock(ilk); + flags &= ~LK_INTERLOCK; + } + GIANT_SAVE(); + while (LK_HOLDER(lk->lk_lock) == + (uintptr_t)owner && TD_IS_RUNNING(owner)) + cpu_spinwait(); + } else if (LK_CAN_ADAPT(lk, flags) && + (x & LK_SHARE) != 0 && LK_SHARERS(x) && + spintries < alk_retries) { + if ((x & LK_EXCLUSIVE_SPINNERS) == 0 && + !atomic_cmpset_ptr(&lk->lk_lock, x, + x | LK_EXCLUSIVE_SPINNERS)) + continue; + if (flags & LK_INTERLOCK) { + class->lc_unlock(ilk); + flags &= ~LK_INTERLOCK; + } + GIANT_SAVE(); + spintries++; + for (i = 0; i < alk_loops; i++) { + if (LOCK_LOG_TEST(&lk->lock_object, 0)) + CTR4(KTR_LOCK, + "%s: shared spinning on %p with %u and %u", + __func__, lk, spintries, i); + if ((lk->lk_lock & + LK_EXCLUSIVE_SPINNERS) == 0) + break; + cpu_spinwait(); + } + if (i != alk_loops) + continue; + } +#endif + /* * Acquire the sleepqueue chain lock because we * probabilly will need to manipulate waiters flags. */ sleepq_lock(&lk->lock_object); x = lk->lk_lock; - v = x & LK_ALL_WAITERS; /* * if the lock has been released while we spun on @@ -592,6 +749,24 @@ __lockmgr_args(struct lock *lk, u_int fl continue; } +#ifdef ADAPTIVE_LOCKMGRS + /* + * The current lock owner might have started executing + * on another CPU (or the lock could have changed + * owner) while we were waiting on the turnstile + * chain lock. If so, drop the turnstile lock and try + * again. + */ + if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 && + LK_HOLDER(x) != LK_KERNPROC) { + owner = (struct thread *)LK_HOLDER(x); + if (TD_IS_RUNNING(owner)) { + sleepq_release(&lk->lock_object); + continue; + } + } +#endif + /* * The lock can be in the state where there is a * pending queue of waiters, but still no owner. @@ -601,7 +776,9 @@ __lockmgr_args(struct lock *lk, u_int fl * claim lock ownership and return, preserving waiters * flags. */ - if (x == (LK_UNLOCKED | v)) { + v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); + if ((x & ~v) == LK_UNLOCKED) { + v &= ~LK_EXCLUSIVE_SPINNERS; if (atomic_cmpset_acq_ptr(&lk->lk_lock, x, tid | v)) { sleepq_release(&lk->lock_object); @@ -666,7 +843,9 @@ __lockmgr_args(struct lock *lk, u_int fl * In order to preserve waiters flags, just spin. */ for (;;) { - x = lk->lk_lock & LK_ALL_WAITERS; + x = lk->lk_lock; + MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); + x &= LK_ALL_WAITERS; if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, LK_SHARERS_LOCK(1) | x)) break; @@ -712,7 +891,7 @@ __lockmgr_args(struct lock *lk, u_int fl break; sleepq_lock(&lk->lock_object); - x = lk->lk_lock & LK_ALL_WAITERS; + x = lk->lk_lock; v = LK_UNLOCKED; /* @@ -720,11 +899,13 @@ __lockmgr_args(struct lock *lk, u_int fl * preference in order to avoid deadlock with * shared runners up. */ + MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); if (x & LK_EXCLUSIVE_WAITERS) { queue = SQ_EXCLUSIVE_QUEUE; v |= (x & LK_SHARED_WAITERS); } else { - MPASS(x == LK_SHARED_WAITERS); + MPASS((x & LK_ALL_WAITERS) == + LK_SHARED_WAITERS); queue = SQ_SHARED_QUEUE; } @@ -777,7 +958,6 @@ __lockmgr_args(struct lock *lk, u_int fl */ sleepq_lock(&lk->lock_object); x = lk->lk_lock; - v = x & LK_ALL_WAITERS; /* * if the lock has been released while we spun on @@ -788,8 +968,9 @@ __lockmgr_args(struct lock *lk, u_int fl continue; } - if (x == (LK_UNLOCKED | v)) { - v = x; + v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); + if ((x & ~v) == LK_UNLOCKED) { + v = (x & ~LK_EXCLUSIVE_SPINNERS); if (v & LK_EXCLUSIVE_WAITERS) { queue = SQ_EXCLUSIVE_QUEUE; v &= ~LK_EXCLUSIVE_WAITERS; @@ -902,7 +1083,9 @@ _lockmgr_disown(struct lock *lk, const c * In order to preserve waiters flags, just spin. */ for (;;) { - x = lk->lk_lock & LK_ALL_WAITERS; + x = lk->lk_lock; + MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); + x &= LK_ALL_WAITERS; if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, LK_KERNPROC | x)) return; @@ -933,6 +1116,8 @@ lockmgr_printinfo(struct lock *lk) printf(" with exclusive waiters pending\n"); if (x & LK_SHARED_WAITERS) printf(" with shared waiters pending\n"); + if (x & LK_EXCLUSIVE_SPINNERS) + printf(" with exclusive spinners pending\n"); STACK_PRINT(lk); } @@ -1094,5 +1279,10 @@ db_show_lockmgr(struct lock_object *lock default: db_printf("none\n"); } + db_printf(" spinners: "); + if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS) + db_printf("exclusive\n"); + else + db_printf("none\n"); } #endif --- /usr/src/sys/sys/lockmgr.h 2009-05-07 18:04:34.000000000 +0200 +++ usr/src/sys/sys/lockmgr.h 2009-06-07 03:49:55.000000000 +0200 @@ -39,13 +39,14 @@ #define LK_SHARE 0x01 #define LK_SHARED_WAITERS 0x02 #define LK_EXCLUSIVE_WAITERS 0x04 +#define LK_EXCLUSIVE_SPINNERS 0x08 #define LK_ALL_WAITERS \ (LK_SHARED_WAITERS | LK_EXCLUSIVE_WAITERS) #define LK_FLAGMASK \ - (LK_SHARE | LK_ALL_WAITERS) + (LK_SHARE | LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS) #define LK_HOLDER(x) ((x) & ~LK_FLAGMASK) -#define LK_SHARERS_SHIFT 3 +#define LK_SHARERS_SHIFT 4 #define LK_SHARERS(x) (LK_HOLDER(x) >> LK_SHARERS_SHIFT) #define LK_SHARERS_LOCK(x) ((x) << LK_SHARERS_SHIFT | LK_SHARE) #define LK_ONE_SHARER (1 << LK_SHARERS_SHIFT) @@ -141,6 +142,7 @@ _lockmgr_args_rw(struct lock *lk, u_int #define LK_NOSHARE 0x000008 #define LK_NOWITNESS 0x000010 #define LK_QUIET 0x000020 +#define LK_NOADAPTIVE 0x000040 /* * Additional attributes to be used in lockmgr().