diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c index d5cf65c27de..4b1b409288c 100644 --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -140,7 +140,7 @@ LK_CAN_SHARE(uintptr_t x, int flags, bool fp) #define lockmgr_xlocked_v(v) \ (((v) & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread) -#define lockmgr_xlocked(lk) lockmgr_xlocked_v((lk)->lk_lock) +#define lockmgr_xlocked(lk) lockmgr_xlocked_v(lockmgr_read_value(lk)) static void assert_lockmgr(const struct lock_object *lock, int how); #ifdef DDB @@ -167,6 +167,12 @@ struct lock_class lock_class_lockmgr = { #endif }; +static __read_mostly bool lk_adaptive = true; +static SYSCTL_NODE(_debug, OID_AUTO, lockmgr, CTLFLAG_RD, NULL, "lockmgr debugging"); +SYSCTL_BOOL(_debug_lockmgr, OID_AUTO, adaptive_spinning, CTLFLAG_RW, &lk_adaptive, + 0, ""); +#define lockmgr_delay locks_delay + struct lockmgr_wait { const char *iwmesg; int ipri; @@ -233,7 +239,7 @@ static void lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line) { - if (LK_HOLDER(lk->lk_lock) != LK_KERNPROC) { + if (LK_HOLDER(lockmgr_read_value(lk)) != LK_KERNPROC) { WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line); TD_LOCKS_DEC(curthread); } @@ -246,7 +252,7 @@ lockmgr_xholder(const struct lock *lk) { uintptr_t x; - x = lk->lk_lock; + x = lockmgr_read_value(lk); return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x)); } @@ -309,7 +315,7 @@ wakeupshlk(struct lock *lk, const char *file, int line) wakeup_swapper = 0; for (;;) { - x = lk->lk_lock; + x = lockmgr_read_value(lk); if (lockmgr_sunlock_try(lk, &x)) break; @@ -318,7 +324,7 @@ wakeupshlk(struct lock *lk, const char *file, int line) * path in order to handle wakeups correctly. */ sleepq_lock(&lk->lock_object); - orig_x = lk->lk_lock; + orig_x = lockmgr_read_value(lk); retry_sleepq: x = orig_x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); v = LK_UNLOCKED; @@ -515,7 +521,6 @@ lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags, bool fp) * waiters, if we fail to acquire the shared lock * loop back and retry. */ - *xp = lk->lk_lock; while (LK_CAN_SHARE(*xp, flags, fp)) { if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp, *xp + LK_ONE_SHARER)) { @@ -541,6 +546,35 @@ lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp) return (false); } +static bool +lockmgr_slock_adaptive(struct lock_delay_arg *lda, struct lock *lk, uintptr_t *xp, + int flags) +{ + struct thread *owner; + uintptr_t x; + + x = *xp; + MPASS(x != LK_UNLOCKED); + owner = (struct thread *)LK_HOLDER(x); + for (;;) { + if (owner == (struct thread *)LK_KERNPROC) + return (false); + if ((x & LK_SHARE) && LK_SHARERS(x) > 0) + return (false); + if (owner == NULL) + return (false); + if (!TD_IS_RUNNING(owner)) + return (false); + lock_delay(lda); + x = lockmgr_read_value(lk); + if (LK_CAN_SHARE(x, flags, false)) { + *xp = x; + return (true); + } + owner = (struct thread *)LK_HOLDER(x); + } +} + static __noinline int lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, const char *file, int line, struct lockmgr_wait *lwa) @@ -557,6 +591,7 @@ lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, uint64_t waittime = 0; int contested = 0; #endif + struct lock_delay_arg lda; if (KERNEL_PANICKED()) goto out; @@ -566,9 +601,18 @@ lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, if (LK_CAN_WITNESS(flags)) WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER, file, line, flags & LK_INTERLOCK ? ilk : NULL); + lock_delay_arg_init(&lda, &lockmgr_delay); + if (!lk_adaptive) + flags &= ~LK_ADAPTIVE; + x = lockmgr_read_value(lk); for (;;) { if (lockmgr_slock_try(lk, &x, flags, false)) break; + + if ((flags & (LK_ADAPTIVE | LK_INTERLOCK)) == LK_ADAPTIVE) { + if (lockmgr_slock_adaptive(&lda, lk, &x, flags)) + continue; + } #ifdef HWPMC_HOOKS PMC_SOFT_CALL( , , lock, failed); #endif @@ -603,7 +647,7 @@ lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, * probabilly will need to manipulate waiters flags. */ sleepq_lock(&lk->lock_object); - x = lk->lk_lock; + x = lockmgr_read_value(lk); retry_sleepq: /* @@ -660,6 +704,7 @@ lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, } LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", __func__, lk); + x = lockmgr_read_value(lk); } if (error == 0) { #ifdef KDTRACE_HOOKS @@ -682,6 +727,34 @@ lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, return (error); } +static bool +lockmgr_xlock_adaptive(struct lock_delay_arg *lda, struct lock *lk, uintptr_t *xp) +{ + struct thread *owner; + uintptr_t x; + + x = *xp; + MPASS(x != LK_UNLOCKED); + owner = (struct thread *)LK_HOLDER(x); + for (;;) { + if (owner == NULL) + return (false); + if ((x & LK_SHARE) && LK_SHARERS(x) > 0) + return (false); + if (owner == (struct thread *)LK_KERNPROC) + return (false); + if (!TD_IS_RUNNING(owner)) + return (false); + lock_delay(lda); + x = lockmgr_read_value(lk); + if (x == LK_UNLOCKED) { + *xp = x; + return (true); + } + owner = (struct thread *)LK_HOLDER(x); + } +} + static __noinline int lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, const char *file, int line, struct lockmgr_wait *lwa) @@ -699,6 +772,7 @@ lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, uint64_t waittime = 0; int contested = 0; #endif + struct lock_delay_arg lda; if (KERNEL_PANICKED()) goto out; @@ -736,6 +810,7 @@ lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, panic("%s: recursing on non recursive lockmgr %p " "@ %s:%d\n", __func__, lk, file, line); } + atomic_set_ptr(&lk->lk_lock, LK_WRITER_RECURSED); lk->lk_recurse++; LOCK_LOG2(lk, "%s: %p recursing", __func__, lk); LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, @@ -746,10 +821,19 @@ lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, goto out; } + x = LK_UNLOCKED; + lock_delay_arg_init(&lda, &lockmgr_delay); + if (!lk_adaptive) + flags &= ~LK_ADAPTIVE; for (;;) { - if (lk->lk_lock == LK_UNLOCKED && - atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) - break; + if (x == LK_UNLOCKED) { + if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x, tid)) + break; + } + if ((flags & (LK_ADAPTIVE | LK_INTERLOCK)) == LK_ADAPTIVE) { + if (lockmgr_xlock_adaptive(&lda, lk, &x)) + continue; + } #ifdef HWPMC_HOOKS PMC_SOFT_CALL( , , lock, failed); #endif @@ -772,7 +856,7 @@ lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, * probabilly will need to manipulate waiters flags. */ sleepq_lock(&lk->lock_object); - x = lk->lk_lock; + x = lockmgr_read_value(lk); retry_sleepq: /* @@ -852,6 +936,7 @@ lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, } LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", __func__, lk); + x = lockmgr_read_value(lk); } if (error == 0) { #ifdef KDTRACE_HOOKS @@ -878,9 +963,8 @@ static __noinline int lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk, const char *file, int line, struct lockmgr_wait *lwa) { - uintptr_t tid, x, v; + uintptr_t tid, v, setv; int error = 0; - int wakeup_swapper = 0; int op; if (KERNEL_PANICKED()) @@ -889,48 +973,47 @@ lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk, tid = (uintptr_t)curthread; _lockmgr_assert(lk, KA_SLOCKED, file, line); - v = lk->lk_lock; - x = v & LK_ALL_WAITERS; - v &= LK_EXCLUSIVE_SPINNERS; - - /* - * Try to switch from one shared lock to an exclusive one. - * We need to preserve waiters flags during the operation. - */ - if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v, - tid | x)) { - LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, - line); - WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE | - LK_TRYWIT(flags), file, line); - LOCKSTAT_RECORD0(lockmgr__upgrade, lk); - TD_SLOCKS_DEC(curthread); - goto out; - } op = flags & LK_TYPE_MASK; + v = lockmgr_read_value(lk); + for (;;) { + if (LK_SHARERS_LOCK(v) > 1) { + if (op == LK_TRYUPGRADE) { + LOCK_LOG2(lk, "%s: %p failed the nowait upgrade", + __func__, lk); + error = EBUSY; + goto out; + } + if (lockmgr_sunlock_try(lk, &v)) { + lockmgr_note_shared_release(lk, file, line); + goto out_xlock; + } + } + MPASS((v & ~LK_ALL_WAITERS) == LK_SHARERS_LOCK(1)); - /* - * In LK_TRYUPGRADE mode, do not drop the lock, - * returning EBUSY instead. - */ - if (op == LK_TRYUPGRADE) { - LOCK_LOG2(lk, "%s: %p failed the nowait upgrade", - __func__, lk); - error = EBUSY; - goto out; + setv = tid; + setv |= (v & LK_ALL_WAITERS); + + /* + * Try to switch from one shared lock to an exclusive one. + * We need to preserve waiters flags during the operation. + */ + if (atomic_fcmpset_ptr(&lk->lk_lock, &v, setv)) { + LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, + line); + WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE | + LK_TRYWIT(flags), file, line); + LOCKSTAT_RECORD0(lockmgr__upgrade, lk); + TD_SLOCKS_DEC(curthread); + goto out; + } } - /* - * We have been unable to succeed in upgrading, so just - * give up the shared lock. - */ - lockmgr_note_shared_release(lk, file, line); - wakeup_swapper |= wakeupshlk(lk, file, line); +out_xlock: error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa); flags &= ~LK_INTERLOCK; out: - lockmgr_exit(flags, ilk, wakeup_swapper); + lockmgr_exit(flags, ilk, 0); return (error); } @@ -955,6 +1038,7 @@ lockmgr_lock_flags(struct lock *lk, u_int flags, struct lock_object *ilk, file, line, flags & LK_INTERLOCK ? ilk : NULL); if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE)) break; + x = lockmgr_read_value(lk); if (lockmgr_slock_try(lk, &x, flags, true)) { lockmgr_note_shared_acquire(lk, 0, 0, file, line, flags); @@ -970,7 +1054,7 @@ lockmgr_lock_flags(struct lock *lk, u_int flags, struct lock_object *ilk, LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ? ilk : NULL); tid = (uintptr_t)curthread; - if (lk->lk_lock == LK_UNLOCKED && + if (lockmgr_read_value(lk) == LK_UNLOCKED && atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) { lockmgr_note_exclusive_acquire(lk, 0, 0, file, line, flags); @@ -1041,9 +1125,11 @@ lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_obje * The lock is held in exclusive mode. * If the lock is recursed also, then unrecurse it. */ - if (lockmgr_xlocked_v(x) && lockmgr_recursed(lk)) { + if (lockmgr_recursed_v(x)) { LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk); lk->lk_recurse--; + if (lk->lk_recurse == 0) + atomic_clear_ptr(&lk->lk_lock, LK_WRITER_RECURSED); goto out; } if (tid != LK_KERNPROC) @@ -1054,7 +1140,7 @@ lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_obje goto out; sleepq_lock(&lk->lock_object); - x = lk->lk_lock; + x = lockmgr_read_value(lk); v = LK_UNLOCKED; /* @@ -1138,12 +1224,13 @@ lockmgr_slock(struct lock *lk, u_int flags, const char *file, int line) if (LK_CAN_WITNESS(flags)) WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER, file, line, NULL); + x = lockmgr_read_value(lk); if (__predict_true(lockmgr_slock_try(lk, &x, flags, true))) { lockmgr_note_shared_acquire(lk, 0, 0, file, line, flags); return (0); } - return (lockmgr_slock_hard(lk, flags, NULL, file, line, NULL)); + return (lockmgr_slock_hard(lk, flags | LK_ADAPTIVE, NULL, file, line, NULL)); } int @@ -1164,7 +1251,7 @@ lockmgr_xlock(struct lock *lk, u_int flags, const char *file, int line) return (0); } - return (lockmgr_xlock_hard(lk, flags, NULL, file, line, NULL)); + return (lockmgr_xlock_hard(lk, flags | LK_ADAPTIVE, NULL, file, line, NULL)); } int @@ -1178,7 +1265,7 @@ lockmgr_unlock(struct lock *lk) line = __LINE__; _lockmgr_assert(lk, KA_LOCKED, file, line); - x = lk->lk_lock; + x = lockmgr_read_value(lk); if (__predict_true(x & LK_SHARE) != 0) { lockmgr_note_shared_release(lk, file, line); if (lockmgr_sunlock_try(lk, &x)) { @@ -1189,9 +1276,8 @@ lockmgr_unlock(struct lock *lk) } else { tid = (uintptr_t)curthread; lockmgr_note_exclusive_release(lk, file, line); - if (!lockmgr_recursed(lk) && - atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) { - LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_WRITER); + if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) { + LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk,LOCKSTAT_WRITER); } else { return (lockmgr_xunlock_hard(lk, x, LK_RELEASE, NULL, file, line)); } @@ -1292,7 +1378,7 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, * In order to preserve waiters flags, just spin. */ for (;;) { - x = lk->lk_lock; + x = lockmgr_read_value(lk); MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); x &= LK_ALL_WAITERS; if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, @@ -1305,7 +1391,7 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, break; case LK_RELEASE: _lockmgr_assert(lk, KA_LOCKED, file, line); - x = lk->lk_lock; + x = lockmgr_read_value(lk); if (__predict_true(x & LK_SHARE) != 0) { lockmgr_note_shared_release(lk, file, line); @@ -1359,7 +1445,7 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, * probabilly will need to manipulate waiters flags. */ sleepq_lock(&lk->lock_object); - x = lk->lk_lock; + x = lockmgr_read_value(lk); /* * if the lock has been released while we spun on @@ -1545,7 +1631,7 @@ _lockmgr_disown(struct lock *lk, const char *file, int line) * In order to preserve waiters flags, just spin. */ for (;;) { - x = lk->lk_lock; + x = lockmgr_read_value(lk); MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); x &= LK_ALL_WAITERS; if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, @@ -1597,7 +1683,7 @@ lockstatus(const struct lock *lk) int ret; ret = LK_SHARED; - x = lk->lk_lock; + x = lockmgr_read_value(lk); v = LK_HOLDER(x); if ((x & LK_SHARE) == 0) { diff --git a/sys/sys/lockmgr.h b/sys/sys/lockmgr.h index 8e5e4d43255..fc3046f8c85 100644 --- a/sys/sys/lockmgr.h +++ b/sys/sys/lockmgr.h @@ -42,13 +42,14 @@ #define LK_SHARED_WAITERS 0x02 #define LK_EXCLUSIVE_WAITERS 0x04 #define LK_EXCLUSIVE_SPINNERS 0x08 +#define LK_WRITER_RECURSED 0x10 #define LK_ALL_WAITERS \ (LK_SHARED_WAITERS | LK_EXCLUSIVE_WAITERS) #define LK_FLAGMASK \ - (LK_SHARE | LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS) + (LK_SHARE | LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS | LK_WRITER_RECURSED) #define LK_HOLDER(x) ((x) & ~LK_FLAGMASK) -#define LK_SHARERS_SHIFT 4 +#define LK_SHARERS_SHIFT 5 #define LK_SHARERS(x) (LK_HOLDER(x) >> LK_SHARERS_SHIFT) #define LK_SHARERS_LOCK(x) ((x) << LK_SHARERS_SHIFT | LK_SHARE) #define LK_ONE_SHARER (1 << LK_SHARERS_SHIFT) @@ -119,6 +120,7 @@ _lockmgr_args_rw(struct lock *lk, u_int flags, struct rwlock *ilk, /* * Define aliases in order to complete lockmgr KPI. */ +#define lockmgr_read_value(lk) ((lk)->lk_lock) #define lockmgr(lk, flags, ilk) \ _lockmgr_args((lk), (flags), (ilk), LK_WMESG_DEFAULT, \ LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, LOCK_FILE, LOCK_LINE) @@ -130,8 +132,10 @@ _lockmgr_args_rw(struct lock *lk, u_int flags, struct rwlock *ilk, LOCK_FILE, LOCK_LINE) #define lockmgr_disown(lk) \ _lockmgr_disown((lk), LOCK_FILE, LOCK_LINE) +#define lockmgr_recursed_v(v) \ + (v & LK_WRITER_RECURSED) #define lockmgr_recursed(lk) \ - ((lk)->lk_recurse != 0) + lockmgr_recursed_v((lk)->lk_lock) #define lockmgr_rw(lk, flags, ilk) \ _lockmgr_args_rw((lk), (flags), (ilk), LK_WMESG_DEFAULT, \ LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, LOCK_FILE, LOCK_LINE) @@ -166,6 +170,7 @@ _lockmgr_args_rw(struct lock *lk, u_int flags, struct rwlock *ilk, #define LK_SLEEPFAIL 0x000800 #define LK_TIMELOCK 0x001000 #define LK_NODDLKTREAT 0x002000 +#define LK_ADAPTIVE 0x004000 /* * Operations for lockmgr(). diff --git a/sys/ufs/ffs/ffs_vnops.c b/sys/ufs/ffs/ffs_vnops.c index f97de5432b6..000ded6cbba 100644 --- a/sys/ufs/ffs/ffs_vnops.c +++ b/sys/ufs/ffs/ffs_vnops.c @@ -445,6 +445,7 @@ ffs_lock(ap) struct lock *lkp; int result; + ap->a_flags |= LK_ADAPTIVE; switch (ap->a_flags & LK_TYPE_MASK) { case LK_SHARED: case LK_UPGRADE: @@ -482,6 +483,7 @@ ffs_lock(ap) } return (result); #else + ap->a_flags |= LK_ADAPTIVE; return (VOP_LOCK1_APV(&ufs_vnodeops, ap)); #endif }