diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 82b7330..4c0e496 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -99,7 +99,7 @@ void mi_startup(void);				/* Should be elsewhere */
 static struct session session0;
 static struct pgrp pgrp0;
 struct	proc proc0;
-struct thread0_storage thread0_st __aligned(16);
+struct thread0_storage thread0_st __aligned(32);
 struct	vmspace vmspace0;
 struct	proc *initproc;
 
diff --git a/sys/kern/kern_condvar.c b/sys/kern/kern_condvar.c
index 5b10914..98f7ae0 100644
--- a/sys/kern/kern_condvar.c
+++ b/sys/kern/kern_condvar.c
@@ -122,7 +122,7 @@ _cv_wait(struct cv *cvp, struct lock_object *lock)
 	    "Waiting on \"%s\"", cvp->cv_description);
 	class = LOCK_CLASS(lock);
 
-	if (SCHEDULER_STOPPED())
+	if (SCHEDULER_STOPPED_TD(td))
 		return;
 
 	sleepq_lock(cvp);
@@ -176,7 +176,7 @@ _cv_wait_unlock(struct cv *cvp, struct lock_object *lock)
 	    ("cv_wait_unlock cannot be used with Giant"));
 	class = LOCK_CLASS(lock);
 
-	if (SCHEDULER_STOPPED()) {
+	if (SCHEDULER_STOPPED_TD(td)) {
 		class->lc_unlock(lock);
 		return;
 	}
@@ -227,7 +227,7 @@ _cv_wait_sig(struct cv *cvp, struct lock_object *lock)
 	    "Waiting on \"%s\"", cvp->cv_description);
 	class = LOCK_CLASS(lock);
 
-	if (SCHEDULER_STOPPED())
+	if (SCHEDULER_STOPPED_TD(td))
 		return (0);
 
 	sleepq_lock(cvp);
@@ -287,7 +287,7 @@ _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt,
 	    "Waiting on \"%s\"", cvp->cv_description);
 	class = LOCK_CLASS(lock);
 
-	if (SCHEDULER_STOPPED())
+	if (SCHEDULER_STOPPED_TD(td))
 		return (0);
 
 	sleepq_lock(cvp);
@@ -349,7 +349,7 @@ _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock,
 	    "Waiting on \"%s\"", cvp->cv_description);
 	class = LOCK_CLASS(lock);
 
-	if (SCHEDULER_STOPPED())
+	if (SCHEDULER_STOPPED_TD(td))
 		return (0);
 
 	sleepq_lock(cvp);
diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
index 1b10623..13a8ca9 100644
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -168,6 +168,72 @@ SYSCTL_UINT(_debug_lockmgr, OID_AUTO, retries, CTLFLAG_RW, &alk_retries, 0, "");
 SYSCTL_UINT(_debug_lockmgr, OID_AUTO, loops, CTLFLAG_RW, &alk_loops, 0, "");
 #endif
 
+static bool __always_inline lockmgr_slock_try(struct lock *lk, uintptr_t *xp,
+    int flags);
+static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t x);
+
+static void
+lockmgr_note_shared_acquire(struct lock *lk, int contested,
+    uint64_t waittime, const char *file, int line, int flags)
+{
+
+	lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,
+	    file, line);
+	LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);
+	WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);
+	TD_LOCKS_INC(curthread);
+	TD_SLOCKS_INC(curthread);
+	STACK_SAVE(lk);
+}
+
+static void
+lockmgr_note_shared_release(struct lock *lk, const char *file, int line)
+{
+
+	lock_profile_release_lock(&lk->lock_object);
+	WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
+	LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
+	TD_LOCKS_DEC(curthread);
+	TD_SLOCKS_DEC(curthread);
+}
+
+static void
+lockmgr_note_exclusive_acquire(struct lock *lk, int contested,
+    uint64_t waittime, const char *file, int line, int flags)
+{
+
+	lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,
+	    file, line);
+	LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
+	WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file,
+	    line);
+	TD_LOCKS_INC(curthread);
+	STACK_SAVE(lk);
+}
+
+static void
+lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line)
+{
+
+	lock_profile_release_lock(&lk->lock_object);
+	LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
+	    line);
+	WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
+	TD_LOCKS_DEC(curthread);
+}
+
+static void
+lockmgr_note_exclusive_upgrade(struct lock *lk, const char *file, int line,
+    int flags)
+{
+
+	LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
+	    line);
+	WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
+	    LK_TRYWIT(flags), file, line);
+	TD_SLOCKS_DEC(curthread);
+}
+
 static __inline struct thread *
 lockmgr_xholder(const struct lock *lk)
 {
@@ -234,35 +300,11 @@ wakeupshlk(struct lock *lk, const char *file, int line)
 	u_int realexslp;
 	int queue, wakeup_swapper;
 
-	WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
-	LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
-
 	wakeup_swapper = 0;
 	for (;;) {
 		x = lk->lk_lock;
-
-		/*
-		 * If there is more than one shared lock held, just drop one
-		 * and return.
-		 */
-		if (LK_SHARERS(x) > 1) {
-			if (atomic_cmpset_rel_ptr(&lk->lk_lock, x,
-			    x - LK_ONE_SHARER))
-				break;
-			continue;
-		}
-
-		/*
-		 * If there are not waiters on the exclusive queue, drop the
-		 * lock quickly.
-		 */
-		if ((x & LK_ALL_WAITERS) == 0) {
-			MPASS((x & ~LK_EXCLUSIVE_SPINNERS) ==
-			    LK_SHARERS_LOCK(1));
-			if (atomic_cmpset_rel_ptr(&lk->lk_lock, x, LK_UNLOCKED))
-				break;
-			continue;
-		}
+		if (lockmgr_sunlock_try(lk, x))
+			break;
 
 		/*
 		 * We should have a sharer with waiters, so enter the hard
@@ -332,9 +374,7 @@ wakeupshlk(struct lock *lk, const char *file, int line)
 		break;
 	}
 
-	lock_profile_release_lock(&lk->lock_object);
-	TD_LOCKS_DEC(curthread);
-	TD_SLOCKS_DEC(curthread);
+	lockmgr_note_shared_release(lk, file, line);
 	return (wakeup_swapper);
 }
 
@@ -448,6 +488,165 @@ lockdestroy(struct lock *lk)
 	lock_destroy(&lk->lock_object);
 }
 
+static bool __always_inline
+lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags)
+{
+
+	/*
+	 * If no other thread has an exclusive lock, or
+	 * no exclusive waiter is present, bump the count of
+	 * sharers.  Since we have to preserve the state of
+	 * waiters, if we fail to acquire the shared lock
+	 * loop back and retry.
+	 */
+	*xp = lk->lk_lock;
+	while (LK_CAN_SHARE(*xp, flags)) {
+		if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp,
+		    *xp + LK_ONE_SHARER)) {
+			return (true);
+		}
+	}
+	return (false);
+}
+
+static bool __always_inline
+lockmgr_sunlock_try(struct lock *lk, uintptr_t x)
+{
+
+	for (;;) {
+		/*
+		 * If there is more than one shared lock held, just drop one
+		 * and return.
+		 */
+		if (LK_SHARERS(x) > 1) {
+			if (atomic_fcmpset_rel_ptr(&lk->lk_lock, &x,
+			    x - LK_ONE_SHARER))
+				return (true);
+			continue;
+		}
+
+		/*
+		 * If there are not waiters on the exclusive queue, drop the
+		 * lock quickly.
+		 */
+		if ((x & LK_ALL_WAITERS) == 0) {
+			MPASS((x & ~LK_EXCLUSIVE_SPINNERS) ==
+			    LK_SHARERS_LOCK(1));
+			if (atomic_fcmpset_rel_ptr(&lk->lk_lock, &x,
+			    LK_UNLOCKED))
+				return (true);
+			continue;
+		}
+		break;
+	}
+	return (false);
+}
+
+int
+lockmgr_lock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk,
+    const char *file, int line)
+{
+	struct lock_class *class;
+	uintptr_t x, v, tid;
+	u_int op;
+	bool locked;
+
+	op = flags & LK_TYPE_MASK;
+	locked = false;
+	switch (op) {
+	case LK_SHARED:
+		if (LK_CAN_WITNESS(flags))
+			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
+			    file, line, flags & LK_INTERLOCK ? ilk : NULL);
+		if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE))
+			break;
+		if (lockmgr_slock_try(lk, &x, flags)) {
+			lockmgr_note_shared_acquire(lk, 0, 0,
+			    file, line, flags);
+			locked = true;
+		}
+		break;
+	case LK_EXCLUSIVE:
+		if (LK_CAN_WITNESS(flags))
+			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
+			    LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
+			    ilk : NULL);
+		tid = (uintptr_t)curthread;
+		if (lk->lk_lock == LK_UNLOCKED &&
+		    atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
+			lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
+			    flags);
+			locked = true;
+		}
+		break;
+	case LK_UPGRADE:
+	case LK_TRYUPGRADE:
+		_lockmgr_assert(lk, KA_SLOCKED, file, line);
+		tid = (uintptr_t)curthread;
+		v = lk->lk_lock;
+		x = v & LK_ALL_WAITERS;
+		v &= LK_EXCLUSIVE_SPINNERS;
+		if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v,
+		    tid | x)) {
+			lockmgr_note_exclusive_upgrade(lk, file, line, flags);
+			locked = true;
+		}
+		break;
+	default:
+		break;
+	}
+	if (__predict_true(locked)) {
+		if (__predict_false(flags & LK_INTERLOCK)) {
+			class = LOCK_CLASS(ilk);
+			class->lc_unlock(ilk);
+		}
+		return (0);
+	} else {
+		return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT,
+		    LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line));
+	}
+}
+
+int
+lockmgr_unlock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk)
+{
+	struct lock_class *class;
+	uintptr_t x, tid;
+	bool unlocked;
+	const char *file;
+	int line;
+
+	file = __FILE__;
+	line = __LINE__;
+
+	_lockmgr_assert(lk, KA_LOCKED, file, line);
+	unlocked = false;
+	x = lk->lk_lock;
+	if (__predict_true(x & LK_SHARE) != 0) {
+		if (lockmgr_sunlock_try(lk, x)) {
+			lockmgr_note_shared_release(lk, file, line);
+			unlocked = true;
+		}
+	} else {
+		tid = (uintptr_t)curthread;
+		if (!lockmgr_recursed(lk) &&
+		    atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
+			lockmgr_note_exclusive_release(lk, file, line);
+			unlocked = true;
+		}
+	}
+	if (__predict_true(unlocked)) {
+		if (__predict_false(flags & LK_INTERLOCK)) {
+			class = LOCK_CLASS(ilk);
+			class->lc_unlock(ilk);
+		}
+		return (0);
+	} else {
+		return (__lockmgr_args(lk, flags | LK_RELEASE, ilk, LK_WMESG_DEFAULT,
+		    LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, LOCK_FILE, LOCK_LINE));
+	}
+}
+
 int
 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
     const char *wmesg, int pri, int timo, const char *file, int line)
@@ -518,21 +717,8 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
 			    file, line, flags & LK_INTERLOCK ? ilk : NULL);
 		for (;;) {
-			x = lk->lk_lock;
-
-			/*
-			 * If no other thread has an exclusive lock, or
-			 * no exclusive waiter is present, bump the count of
-			 * sharers.  Since we have to preserve the state of
-			 * waiters, if we fail to acquire the shared lock
-			 * loop back and retry.
-			 */
-			if (LK_CAN_SHARE(x, flags)) {
-				if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
-				    x + LK_ONE_SHARER))
-					break;
-				continue;
-			}
+			if (lockmgr_slock_try(lk, &x, flags))
+				break;
 #ifdef HWPMC_HOOKS
 			PMC_SOFT_CALL( , , lock, failed);
 #endif
@@ -697,15 +883,13 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 			    __func__, lk);
 		}
 		if (error == 0) {
-			lock_profile_obtain_lock_success(&lk->lock_object,
-			    contested, waittime, file, line);
-			LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file,
-			    line);
-			WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file,
-			    line);
-			TD_LOCKS_INC(curthread);
-			TD_SLOCKS_INC(curthread);
-			STACK_SAVE(lk);
+#ifdef LOCK_PROFILING
+			lockmgr_note_shared_acquire(lk, contested, waittime,
+			    file, line, flags);
+#else
+			lockmgr_note_shared_acquire(lk, 0, 0, file, line,
+			    flags);
+#endif
 		}
 		break;
 	case LK_UPGRADE:
@@ -968,14 +1152,13 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 			    __func__, lk);
 		}
 		if (error == 0) {
-			lock_profile_obtain_lock_success(&lk->lock_object,
-			    contested, waittime, file, line);
-			LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
-			    lk->lk_recurse, file, line);
-			WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
-			    LK_TRYWIT(flags), file, line);
-			TD_LOCKS_INC(curthread);
-			STACK_SAVE(lk);
+#ifdef LOCK_PROFILING
+			lockmgr_note_exclusive_acquire(lk, contested, waittime,
+			    file, line, flags);
+#else
+			lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
+			    flags);
+#endif
 		}
 		break;
 	case LK_DOWNGRADE:
diff --git a/sys/kern/kern_lockstat.c b/sys/kern/kern_lockstat.c
index 10da98b..679a01c 100644
--- a/sys/kern/kern_lockstat.c
+++ b/sys/kern/kern_lockstat.c
@@ -62,7 +62,7 @@ SDT_PROBE_DEFINE1(lockstat, , , sx__downgrade, "struct sx *");
 
 SDT_PROBE_DEFINE2(lockstat, , , thread__spin, "struct mtx *", "uint64_t");
 
-int __read_mostly lockstat_enabled;
+volatile int __read_mostly lockstat_enabled;
 
 uint64_t 
 lockstat_nsecs(struct lock_object *lo)
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index 35bbd5f..f2aed49 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -140,63 +140,27 @@ struct lock_class lock_class_mtx_spin = {
 #ifdef ADAPTIVE_MUTEXES
 static SYSCTL_NODE(_debug, OID_AUTO, mtx, CTLFLAG_RD, NULL, "mtx debugging");
 
-static struct lock_delay_config __read_mostly mtx_delay = {
-	.initial	= 1000,
-	.step		= 500,
-	.min		= 100,
-	.max		= 5000,
-};
+static struct lock_delay_config __read_mostly mtx_delay;
 
-SYSCTL_INT(_debug_mtx, OID_AUTO, delay_initial, CTLFLAG_RW, &mtx_delay.initial,
-    0, "");
-SYSCTL_INT(_debug_mtx, OID_AUTO, delay_step, CTLFLAG_RW, &mtx_delay.step,
-    0, "");
-SYSCTL_INT(_debug_mtx, OID_AUTO, delay_min, CTLFLAG_RW, &mtx_delay.min,
+SYSCTL_INT(_debug_mtx, OID_AUTO, delay_base, CTLFLAG_RW, &mtx_delay.base,
     0, "");
 SYSCTL_INT(_debug_mtx, OID_AUTO, delay_max, CTLFLAG_RW, &mtx_delay.max,
     0, "");
 
-static void
-mtx_delay_sysinit(void *dummy)
-{
-
-	mtx_delay.initial = mp_ncpus * 25;
-	mtx_delay.step = (mp_ncpus * 25) / 2;
-	mtx_delay.min = mp_ncpus * 5;
-	mtx_delay.max = mp_ncpus * 25 * 10;
-}
-LOCK_DELAY_SYSINIT(mtx_delay_sysinit);
+LOCK_DELAY_SYSINIT_DEFAULT(mtx_delay);
 #endif
 
 static SYSCTL_NODE(_debug, OID_AUTO, mtx_spin, CTLFLAG_RD, NULL,
     "mtx spin debugging");
 
-static struct lock_delay_config __read_mostly mtx_spin_delay = {
-	.initial        = 1000,
-	.step           = 500,
-	.min            = 100,
-	.max            = 5000,
-};
+static struct lock_delay_config __read_mostly mtx_spin_delay;
 
-SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_initial, CTLFLAG_RW,
-    &mtx_spin_delay.initial, 0, "");
-SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_step, CTLFLAG_RW, &mtx_spin_delay.step,
-    0, "");
-SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_min, CTLFLAG_RW, &mtx_spin_delay.min,
-    0, "");
-SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_max, CTLFLAG_RW, &mtx_spin_delay.max,
-    0, "");
+SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_base, CTLFLAG_RW,
+    &mtx_spin_delay.base, 0, "");
+SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_max, CTLFLAG_RW,
+    &mtx_spin_delay.max, 0, "");
 
-static void
-mtx_spin_delay_sysinit(void *dummy)
-{
-
-	mtx_spin_delay.initial = mp_ncpus * 25;
-	mtx_spin_delay.step = (mp_ncpus * 25) / 2;
-	mtx_spin_delay.min = mp_ncpus * 5;
-	mtx_spin_delay.max = mp_ncpus * 25 * 10;
-}
-LOCK_DELAY_SYSINIT(mtx_spin_delay_sysinit);
+LOCK_DELAY_SYSINIT_DEFAULT(mtx_spin_delay);
 
 /*
  * System-wide mutexes
@@ -265,9 +229,7 @@ void
 __mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
 {
 	struct mtx *m;
-
-	if (SCHEDULER_STOPPED())
-		return;
+	uintptr_t tid, v;
 
 	m = mtxlock2mtx(c);
 
@@ -282,7 +244,13 @@ __mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
 	WITNESS_CHECKORDER(&m->lock_object, (opts & ~MTX_RECURSE) |
 	    LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
 
-	__mtx_lock(m, curthread, opts, file, line);
+	tid = (uintptr_t)curthread;
+	v = MTX_UNOWNED;
+	if (!_mtx_obtain_lock_fetch(m, &v, tid))
+		_mtx_lock_sleep(m, v, tid, opts, file, line);
+	else
+		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire,
+		    m, 0, 0, file, line);
 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
 	    line);
 	WITNESS_LOCK(&m->lock_object, (opts & ~MTX_RECURSE) | LOP_EXCLUSIVE,
@@ -295,9 +263,6 @@ __mtx_unlock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
 {
 	struct mtx *m;
 
-	if (SCHEDULER_STOPPED())
-		return;
-
 	m = mtxlock2mtx(c);
 
 	KASSERT(m->mtx_lock != MTX_DESTROYED,
@@ -310,7 +275,11 @@ __mtx_unlock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
 	    line);
 	mtx_assert(m, MA_OWNED);
 
+#ifdef LOCK_PROFILING
+	__mtx_unlock_sleep(c, opts, file, line);
+#else
 	__mtx_unlock(m, curthread, opts, file, line);
+#endif
 	TD_LOCKS_DEC(curthread);
 }
 
@@ -405,13 +374,18 @@ int
 _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
 {
 	struct mtx *m;
+	struct thread *td;
+	uintptr_t tid, v;
 #ifdef LOCK_PROFILING
 	uint64_t waittime = 0;
 	int contested = 0;
 #endif
 	int rval;
+	bool recursed;
 
-	if (SCHEDULER_STOPPED())
+	td = curthread;
+	tid = (uintptr_t)td;
+	if (SCHEDULER_STOPPED_TD(td))
 		return (1);
 
 	m = mtxlock2mtx(c);
@@ -425,13 +399,26 @@ _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
 	    ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
 	    file, line));
 
-	if (mtx_owned(m) && ((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
-	    (opts & MTX_RECURSE) != 0)) {
-		m->mtx_recurse++;
-		atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
-		rval = 1;
-	} else
-		rval = _mtx_obtain_lock(m, (uintptr_t)curthread);
+	rval = 1;
+	recursed = false;
+	v = MTX_UNOWNED;
+	for (;;) {
+		if (_mtx_obtain_lock_fetch(m, &v, tid))
+			break;
+		if (v == MTX_UNOWNED)
+			continue;
+		if (v == tid &&
+		    ((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
+		    (opts & MTX_RECURSE) != 0)) {
+				 m->mtx_recurse++;
+				 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
+				 recursed = true;
+				 break;
+		}
+		rval = 0;
+		break;
+	}
+
 	opts &= ~MTX_RECURSE;
 
 	LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line);
@@ -439,10 +426,9 @@ _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
 		WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK,
 		    file, line);
 		TD_LOCKS_INC(curthread);
-		if (m->mtx_recurse == 0)
+		if (!recursed)
 			LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire,
 			    m, contested, waittime, file, line);
-
 	}
 
 	return (rval);
@@ -454,13 +440,17 @@ _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
  * We call this if the lock is either contested (i.e. we need to go to
  * sleep waiting for it), or if we need to recurse on it.
  */
+#if LOCK_DEBUG > 0
 void
-__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
+__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, int opts,
     const char *file, int line)
+#else
+void
+__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, uintptr_t tid)
+#endif
 {
 	struct mtx *m;
 	struct turnstile *ts;
-	uintptr_t v;
 #ifdef ADAPTIVE_MUTEXES
 	volatile struct thread *owner;
 #endif
@@ -478,6 +468,7 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
 	u_int sleep_cnt = 0;
 	int64_t sleep_time = 0;
 	int64_t all_time = 0;
+	int doing_lockstat;
 #endif
 
 	if (SCHEDULER_STOPPED())
@@ -489,21 +480,26 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
 	lock_delay_arg_init(&lda, NULL);
 #endif
 	m = mtxlock2mtx(c);
-	v = MTX_READ_VALUE(m);
+	if (__predict_false(v == MTX_UNOWNED))
+		v = MTX_READ_VALUE(m);
 
 	if (__predict_false(lv_mtx_owner(v) == (struct thread *)tid)) {
 		KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
 		    (opts & MTX_RECURSE) != 0,
 	    ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
 		    m->lock_object.lo_name, file, line));
+#if LOCK_DEBUG > 0
 		opts &= ~MTX_RECURSE;
+#endif
 		m->mtx_recurse++;
 		atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
 		if (LOCK_LOG_TEST(&m->lock_object, opts))
 			CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
 		return;
 	}
+#if LOCK_DEBUG > 0
 	opts &= ~MTX_RECURSE;
+#endif
 
 #ifdef HWPMC_HOOKS
 	PMC_SOFT_CALL( , , lock, failed);
@@ -515,14 +511,19 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
 		    "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
 		    m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
 #ifdef KDTRACE_HOOKS
-	all_time -= lockstat_nsecs(&m->lock_object);
+#ifdef LOCK_PROFILING
+	doing_lockstat = 1;
+#else
+	doing_lockstat = lockstat_enabled;
+#endif
+	if (__predict_false(doing_lockstat))
+		all_time -= lockstat_nsecs(&m->lock_object);
 #endif
 
 	for (;;) {
 		if (v == MTX_UNOWNED) {
-			if (_mtx_obtain_lock(m, tid))
+			if (_mtx_obtain_lock_fetch(m, &v, tid))
 				break;
-			v = MTX_READ_VALUE(m);
 			continue;
 		}
 #ifdef KDTRACE_HOOKS
@@ -623,9 +624,6 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
 #endif
 		v = MTX_READ_VALUE(m);
 	}
-#ifdef KDTRACE_HOOKS
-	all_time += lockstat_nsecs(&m->lock_object);
-#endif
 #ifdef KTR
 	if (cont_logged) {
 		CTR4(KTR_CONTENTION,
@@ -633,6 +631,11 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
 		    m->lock_object.lo_name, (void *)tid, file, line);
 	}
 #endif
+#ifdef KDTRACE_HOOKS
+	if (__predict_true(!doing_lockstat))
+		return;
+	all_time += lockstat_nsecs(&m->lock_object);
+#endif
 	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, m, contested,
 	    waittime, file, line);
 #ifdef KDTRACE_HOOKS
@@ -640,7 +643,7 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
 		LOCKSTAT_RECORD1(adaptive__block, m, sleep_time);
 
 	/*
-	 * Only record the loops spinning and not sleeping. 
+	 * Only record the loops spinning and not sleeping.
 	 */
 	if (lda.spin_cnt > sleep_cnt)
 		LOCKSTAT_RECORD1(adaptive__spin, m, all_time - sleep_time);
@@ -674,12 +677,11 @@ _mtx_lock_spin_failed(struct mtx *m)
  * is handled inline.
  */
 void
-_mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts,
-    const char *file, int line)
+_mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
+    int opts, const char *file, int line)
 {
 	struct mtx *m;
 	struct lock_delay_arg lda;
-	uintptr_t v;
 #ifdef LOCK_PROFILING
 	int contested = 0;
 	uint64_t waittime = 0;
@@ -694,6 +696,14 @@ _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts,
 	lock_delay_arg_init(&lda, &mtx_spin_delay);
 	m = mtxlock2mtx(c);
 
+	if (__predict_false(v == MTX_UNOWNED))
+		v = MTX_READ_VALUE(m);
+
+	if (__predict_false(v == tid)) {
+		m->mtx_recurse++;
+		return;
+	}
+
 	if (LOCK_LOG_TEST(&m->lock_object, opts))
 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
 	KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid),
@@ -706,12 +716,10 @@ _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts,
 #ifdef KDTRACE_HOOKS
 	spin_time -= lockstat_nsecs(&m->lock_object);
 #endif
-	v = MTX_READ_VALUE(m);
 	for (;;) {
 		if (v == MTX_UNOWNED) {
-			if (_mtx_obtain_lock(m, tid))
+			if (_mtx_obtain_lock_fetch(m, &v, tid))
 				break;
-			v = MTX_READ_VALUE(m);
 			continue;
 		}
 		/* Give interrupts a chance while we spin. */
@@ -783,6 +791,7 @@ thread_lock_flags_(struct thread *td, int opts, const char *file, int line)
 #endif
 	for (;;) {
 retry:
+		v = MTX_UNOWNED;
 		spinlock_enter();
 		m = td->td_lock;
 		KASSERT(m->mtx_lock != MTX_DESTROYED,
@@ -796,14 +805,11 @@ retry:
 			    m->lock_object.lo_name, file, line));
 		WITNESS_CHECKORDER(&m->lock_object,
 		    opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
-		v = MTX_READ_VALUE(m);
 		for (;;) {
-			if (v == MTX_UNOWNED) {
-				if (_mtx_obtain_lock(m, tid))
-					break;
-				v = MTX_READ_VALUE(m);
+			if (_mtx_obtain_lock_fetch(m, &v, tid))
+				break;
+			if (v == MTX_UNOWNED)
 				continue;
-			}
 			if (v == tid) {
 				m->mtx_recurse++;
 				break;
@@ -888,21 +894,29 @@ thread_lock_set(struct thread *td, struct mtx *new)
 /*
  * __mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
  *
- * We are only called here if the lock is recursed or contested (i.e. we
- * need to wake up a blocked thread).
+ * We are only called here if the lock is recursed, contested (i.e. we
+ * need to wake up a blocked thread) or lockstat probe is active.
  */
+#if LOCK_DEBUG > 0
 void
 __mtx_unlock_sleep(volatile uintptr_t *c, int opts, const char *file, int line)
+#else
+void
+__mtx_unlock_sleep(volatile uintptr_t *c)
+#endif
 {
 	struct mtx *m;
 	struct turnstile *ts;
+	uintptr_t tid, v;
 
 	if (SCHEDULER_STOPPED())
 		return;
 
+	tid = (uintptr_t)curthread;
 	m = mtxlock2mtx(c);
+	v = MTX_READ_VALUE(m);
 
-	if (mtx_recursed(m)) {
+	if (v & MTX_RECURSED) {
 		if (--(m->mtx_recurse) == 0)
 			atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
 		if (LOCK_LOG_TEST(&m->lock_object, opts))
@@ -910,6 +924,10 @@ __mtx_unlock_sleep(volatile uintptr_t *c, int opts, const char *file, int line)
 		return;
 	}
 
+	LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, m);
+	if (v == tid && _mtx_release_lock(m, tid))
+		return;
+
 	/*
 	 * We have to lock the chain before the turnstile so this turnstile
 	 * can be removed from the hash list if it is empty.
diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c
index 99df700..2e70321 100644
--- a/sys/kern/kern_rwlock.c
+++ b/sys/kern/kern_rwlock.c
@@ -100,32 +100,14 @@ static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL,
 SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, "");
 SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, "");
 
-static struct lock_delay_config __read_mostly rw_delay = {
-	.initial	= 1000,
-	.step		= 500,
-	.min		= 100,
-	.max		= 5000,
-};
+static struct lock_delay_config __read_mostly rw_delay;
 
-SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_initial, CTLFLAG_RW, &rw_delay.initial,
-    0, "");
-SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_step, CTLFLAG_RW, &rw_delay.step,
-    0, "");
-SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_min, CTLFLAG_RW, &rw_delay.min,
+SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_base, CTLFLAG_RW, &rw_delay.base,
     0, "");
 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_max, CTLFLAG_RW, &rw_delay.max,
     0, "");
 
-static void
-rw_delay_sysinit(void *dummy)
-{
-
-	rw_delay.initial = mp_ncpus * 25;
-	rw_delay.step = (mp_ncpus * 25) / 2;
-	rw_delay.min = mp_ncpus * 5;
-	rw_delay.max = mp_ncpus * 25 * 10;
-}
-LOCK_DELAY_SYSINIT(rw_delay_sysinit);
+LOCK_DELAY_SYSINIT_DEFAULT(rw_delay);
 #endif
 
 /*
@@ -283,9 +265,7 @@ void
 _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line)
 {
 	struct rwlock *rw;
-
-	if (SCHEDULER_STOPPED())
-		return;
+	uintptr_t tid, v;
 
 	rw = rwlock2rw(c);
 
@@ -296,7 +276,14 @@ _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line)
 	    ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
 	WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
 	    line, NULL);
-	__rw_wlock(rw, curthread, file, line);
+	tid = (uintptr_t)curthread;
+	v = RW_UNLOCKED;
+	if (!_rw_write_lock_fetch(rw, &v, tid))
+		_rw_wlock_hard(rw, v, tid, file, line);
+	else
+		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,
+		    0, 0, file, line, LOCKSTAT_WRITER);
+
 	LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
 	WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
 	TD_LOCKS_INC(curthread);
@@ -306,9 +293,14 @@ int
 __rw_try_wlock(volatile uintptr_t *c, const char *file, int line)
 {
 	struct rwlock *rw;
+	struct thread *td;
+	uintptr_t tid, v;
 	int rval;
+	bool recursed;
 
-	if (SCHEDULER_STOPPED())
+	td = curthread;
+	tid = (uintptr_t)td;
+	if (SCHEDULER_STOPPED_TD(td))
 		return (1);
 
 	rw = rwlock2rw(c);
@@ -319,19 +311,28 @@ __rw_try_wlock(volatile uintptr_t *c, const char *file, int line)
 	KASSERT(rw->rw_lock != RW_DESTROYED,
 	    ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line));
 
-	if (rw_wlocked(rw) &&
-	    (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) {
-		rw->rw_recurse++;
-		rval = 1;
-	} else
-		rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED,
-		    (uintptr_t)curthread);
+	rval = 1;
+	recursed = false;
+	v = RW_UNLOCKED;
+	for (;;) {
+		if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid))
+			break;
+		if (v == RW_UNLOCKED)
+			continue;
+		if (v == tid && (rw->lock_object.lo_flags & LO_RECURSABLE)) {
+			rw->rw_recurse++;
+			atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
+			break;
+		}
+		rval = 0;
+		break;
+	}
 
 	LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line);
 	if (rval) {
 		WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 		    file, line);
-		if (!rw_recursed(rw))
+		if (!recursed)
 			LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire,
 			    rw, 0, 0, file, line, LOCKSTAT_WRITER);
 		TD_LOCKS_INC(curthread);
@@ -344,9 +345,6 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line)
 {
 	struct rwlock *rw;
 
-	if (SCHEDULER_STOPPED())
-		return;
-
 	rw = rwlock2rw(c);
 
 	KASSERT(rw->rw_lock != RW_DESTROYED,
@@ -355,7 +353,13 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line)
 	WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
 	LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
 	    line);
+
+#ifdef LOCK_PROFILING
+	_rw_wunlock_hard(rw, (uintptr_t)curthread, file, line);
+#else
 	__rw_wunlock(rw, curthread, file, line);
+#endif
+
 	TD_LOCKS_DEC(curthread);
 }
 
@@ -366,13 +370,44 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line)
  * is unlocked and has no writer waiters or spinners.  Failing otherwise
  * prioritizes writers before readers.
  */
-#define	RW_CAN_READ(_rw)						\
-    ((curthread->td_rw_rlocks && (_rw) & RW_LOCK_READ) || ((_rw) &	\
+#define	RW_CAN_READ(td, _rw)						\
+    (((td)->td_rw_rlocks && (_rw) & RW_LOCK_READ) || ((_rw) &	\
     (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) ==	\
     RW_LOCK_READ)
 
-void
-__rw_rlock(volatile uintptr_t *c, const char *file, int line)
+static bool __always_inline
+__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp,
+    const char *file, int line)
+{
+
+	/*
+	 * Handle the easy case.  If no other thread has a write
+	 * lock, then try to bump up the count of read locks.  Note
+	 * that we have to preserve the current state of the
+	 * RW_LOCK_WRITE_WAITERS flag.  If we fail to acquire a
+	 * read lock, then rw_lock must have changed, so restart
+	 * the loop.  Note that this handles the case of a
+	 * completely unlocked rwlock since such a lock is encoded
+	 * as a read lock with no waiters.
+	 */
+	while (RW_CAN_READ(td, *vp)) {
+		if (atomic_fcmpset_acq_ptr(&rw->rw_lock, vp,
+			*vp + RW_ONE_READER)) {
+			if (LOCK_LOG_TEST(&rw->lock_object, 0))
+				CTR4(KTR_LOCK,
+				    "%s: %p succeed %p -> %p", __func__,
+				    rw, (void *)*vp,
+				    (void *)(*vp + RW_ONE_READER));
+			td->td_rw_rlocks++;
+			return (true);
+		}
+	}
+	return (false);
+}
+
+static void __noinline
+__rw_rlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v,
+    const char *file, int line)
 {
 	struct rwlock *rw;
 	struct turnstile *ts;
@@ -385,7 +420,6 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
 	uint64_t waittime = 0;
 	int contested = 0;
 #endif
-	uintptr_t v;
 #if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS)
 	struct lock_delay_arg lda;
 #endif
@@ -406,52 +440,15 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
 #endif
 	rw = rwlock2rw(c);
 
-	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
-	    ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d",
-	    curthread, rw->lock_object.lo_name, file, line));
-	KASSERT(rw->rw_lock != RW_DESTROYED,
-	    ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
-	KASSERT(rw_wowner(rw) != curthread,
-	    ("rw_rlock: wlock already held for %s @ %s:%d",
-	    rw->lock_object.lo_name, file, line));
-	WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
-
 #ifdef KDTRACE_HOOKS
 	all_time -= lockstat_nsecs(&rw->lock_object);
 #endif
-	v = RW_READ_VALUE(rw);
 #ifdef KDTRACE_HOOKS
 	state = v;
 #endif
 	for (;;) {
-		/*
-		 * Handle the easy case.  If no other thread has a write
-		 * lock, then try to bump up the count of read locks.  Note
-		 * that we have to preserve the current state of the
-		 * RW_LOCK_WRITE_WAITERS flag.  If we fail to acquire a
-		 * read lock, then rw_lock must have changed, so restart
-		 * the loop.  Note that this handles the case of a
-		 * completely unlocked rwlock since such a lock is encoded
-		 * as a read lock with no waiters.
-		 */
-		if (RW_CAN_READ(v)) {
-			/*
-			 * The RW_LOCK_READ_WAITERS flag should only be set
-			 * if the lock has been unlocked and write waiters
-			 * were present.
-			 */
-			if (atomic_cmpset_acq_ptr(&rw->rw_lock, v,
-			    v + RW_ONE_READER)) {
-				if (LOCK_LOG_TEST(&rw->lock_object, 0))
-					CTR4(KTR_LOCK,
-					    "%s: %p succeed %p -> %p", __func__,
-					    rw, (void *)v,
-					    (void *)(v + RW_ONE_READER));
-				break;
-			}
-			v = RW_READ_VALUE(rw);
-			continue;
-		}
+		if (__rw_rlock_try(rw, td, &v, file, line))
+			break;
 #ifdef KDTRACE_HOOKS
 		lda.spin_cnt++;
 #endif
@@ -493,7 +490,7 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
 			    rw->lock_object.lo_name);
 			for (i = 0; i < rowner_loops; i++) {
 				v = RW_READ_VALUE(rw);
-				if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(v))
+				if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(td, v))
 					break;
 				cpu_spinwait();
 			}
@@ -521,7 +518,7 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
 		 * recheck its state and restart the loop if needed.
 		 */
 		v = RW_READ_VALUE(rw);
-		if (RW_CAN_READ(v)) {
+		if (RW_CAN_READ(td, v)) {
 			turnstile_cancel(ts);
 			continue;
 		}
@@ -546,7 +543,7 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
 		/*
 		 * The lock is held in write mode or it already has waiters.
 		 */
-		MPASS(!RW_CAN_READ(v));
+		MPASS(!RW_CAN_READ(td, v));
 
 		/*
 		 * If the RW_LOCK_READ_WAITERS flag is already set, then
@@ -606,10 +603,36 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
 	 */
 	LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested,
 	    waittime, file, line, LOCKSTAT_READER);
+}
+
+void
+__rw_rlock(volatile uintptr_t *c, const char *file, int line)
+{
+	struct rwlock *rw;
+	struct thread *td;
+	uintptr_t v;
+
+	td = curthread;
+	rw = rwlock2rw(c);
+
+	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(td),
+	    ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d",
+	    td, rw->lock_object.lo_name, file, line));
+	KASSERT(rw->rw_lock != RW_DESTROYED,
+	    ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
+	KASSERT(rw_wowner(rw) != td,
+	    ("rw_rlock: wlock already held for %s @ %s:%d",
+	    rw->lock_object.lo_name, file, line));
+	WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
+
+	v = RW_READ_VALUE(rw);
+	if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__acquire) ||
+	    !__rw_rlock_try(rw, td, &v, file, line)))
+		__rw_rlock_hard(c, td, v, file, line);
+
 	LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
 	WITNESS_LOCK(&rw->lock_object, 0, file, line);
 	TD_LOCKS_INC(curthread);
-	curthread->td_rw_rlocks++;
 }
 
 int
@@ -627,13 +650,13 @@ __rw_try_rlock(volatile uintptr_t *c, const char *file, int line)
 	    ("rw_try_rlock() by idle thread %p on rwlock %s @ %s:%d",
 	    curthread, rw->lock_object.lo_name, file, line));
 
+	x = rw->rw_lock;
 	for (;;) {
-		x = rw->rw_lock;
 		KASSERT(rw->rw_lock != RW_DESTROYED,
 		    ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line));
 		if (!(x & RW_LOCK_READ))
 			break;
-		if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) {
+		if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &x, x + RW_ONE_READER)) {
 			LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,
 			    line);
 			WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line);
@@ -649,61 +672,67 @@ __rw_try_rlock(volatile uintptr_t *c, const char *file, int line)
 	return (0);
 }
 
-void
-_rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
+static bool __always_inline
+__rw_runlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp)
 {
-	struct rwlock *rw;
-	struct turnstile *ts;
-	uintptr_t x, v, queue;
-
-	if (SCHEDULER_STOPPED())
-		return;
 
-	rw = rwlock2rw(c);
-
-	KASSERT(rw->rw_lock != RW_DESTROYED,
-	    ("rw_runlock() of destroyed rwlock @ %s:%d", file, line));
-	__rw_assert(c, RA_RLOCKED, file, line);
-	WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
-	LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
-
-	/* TODO: drop "owner of record" here. */
-	x = RW_READ_VALUE(rw);
 	for (;;) {
 		/*
 		 * See if there is more than one read lock held.  If so,
 		 * just drop one and return.
 		 */
-		if (RW_READERS(x) > 1) {
-			if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
-			    x - RW_ONE_READER)) {
+		if (RW_READERS(*vp) > 1) {
+			if (atomic_fcmpset_rel_ptr(&rw->rw_lock, vp,
+			    *vp - RW_ONE_READER)) {
 				if (LOCK_LOG_TEST(&rw->lock_object, 0))
 					CTR4(KTR_LOCK,
 					    "%s: %p succeeded %p -> %p",
-					    __func__, rw, (void *)x,
-					    (void *)(x - RW_ONE_READER));
-				break;
+					    __func__, rw, (void *)*vp,
+					    (void *)(*vp - RW_ONE_READER));
+				td->td_rw_rlocks--;
+				return (true);
 			}
-			x = RW_READ_VALUE(rw);
 			continue;
 		}
 		/*
 		 * If there aren't any waiters for a write lock, then try
 		 * to drop it quickly.
 		 */
-		if (!(x & RW_LOCK_WAITERS)) {
-			MPASS((x & ~RW_LOCK_WRITE_SPINNER) ==
+		if (!(*vp & RW_LOCK_WAITERS)) {
+			MPASS((*vp & ~RW_LOCK_WRITE_SPINNER) ==
 			    RW_READERS_LOCK(1));
-			if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
+			if (atomic_fcmpset_rel_ptr(&rw->rw_lock, vp,
 			    RW_UNLOCKED)) {
 				if (LOCK_LOG_TEST(&rw->lock_object, 0))
 					CTR2(KTR_LOCK, "%s: %p last succeeded",
 					    __func__, rw);
-				break;
+				td->td_rw_rlocks--;
+				return (true);
 			}
-			x = RW_READ_VALUE(rw);
 			continue;
 		}
+		break;
+	}
+	return (false);
+}
+
+static void __noinline
+__rw_runlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v,
+    const char *file, int line)
+{
+	struct rwlock *rw;
+	struct turnstile *ts;
+	uintptr_t x, queue;
+
+	if (SCHEDULER_STOPPED())
+		return;
+
+	rw = rwlock2rw(c);
+
+	for (;;) {
+		if (__rw_runlock_try(rw, td, &v))
+			break;
+
 		/*
 		 * Ok, we know we have waiters and we think we are the
 		 * last reader, so grab the turnstile lock.
@@ -737,7 +766,7 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
 		if (!atomic_cmpset_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v,
 		    x)) {
 			turnstile_chain_unlock(&rw->lock_object);
-			x = RW_READ_VALUE(rw);
+			v = RW_READ_VALUE(rw);
 			continue;
 		}
 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
@@ -756,21 +785,46 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
 		turnstile_broadcast(ts, queue);
 		turnstile_unpend(ts, TS_SHARED_LOCK);
 		turnstile_chain_unlock(&rw->lock_object);
+		td->td_rw_rlocks--;
 		break;
 	}
 	LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_READER);
+}
+
+void
+_rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
+{
+	struct rwlock *rw;
+	struct thread *td;
+	uintptr_t v;
+
+	rw = rwlock2rw(c);
+
+	KASSERT(rw->rw_lock != RW_DESTROYED,
+	    ("rw_runlock() of destroyed rwlock @ %s:%d", file, line));
+	__rw_assert(c, RA_RLOCKED, file, line);
+	WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
+	LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
+
+	td = curthread;
+	v = RW_READ_VALUE(rw);
+
+	if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__release) ||
+	    !__rw_runlock_try(rw, td, &v)))
+		__rw_runlock_hard(c, td, v, file, line);
+
 	TD_LOCKS_DEC(curthread);
-	curthread->td_rw_rlocks--;
 }
 
+
 /*
  * This function is called when we are unable to obtain a write lock on the
  * first try.  This means that at least one other thread holds either a
  * read or write lock.
  */
 void
-__rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
-    int line)
+__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
+    const char *file, int line)
 {
 	struct rwlock *rw;
 	struct turnstile *ts;
@@ -779,7 +833,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
 	int spintries = 0;
 	int i;
 #endif
-	uintptr_t v, x;
+	uintptr_t x;
 #ifdef LOCK_PROFILING
 	uint64_t waittime = 0;
 	int contested = 0;
@@ -803,13 +857,15 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
 	lock_delay_arg_init(&lda, NULL);
 #endif
 	rw = rwlock2rw(c);
-	v = RW_READ_VALUE(rw);
+	if (__predict_false(v == RW_UNLOCKED))
+		v = RW_READ_VALUE(rw);
 
 	if (__predict_false(lv_rw_wowner(v) == (struct thread *)tid)) {
 		KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
 		    ("%s: recursing but non-recursive rw %s @ %s:%d\n",
 		    __func__, rw->lock_object.lo_name, file, line));
 		rw->rw_recurse++;
+		atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
 			CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
 		return;
@@ -825,9 +881,8 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
 #endif
 	for (;;) {
 		if (v == RW_UNLOCKED) {
-			if (_rw_write_lock(rw, tid))
+			if (_rw_write_lock_fetch(rw, &v, tid))
 				break;
-			v = RW_READ_VALUE(rw);
 			continue;
 		}
 #ifdef KDTRACE_HOOKS
@@ -986,9 +1041,10 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
 }
 
 /*
- * This function is called if the first try at releasing a write lock failed.
- * This means that one of the 2 waiter bits must be set indicating that at
- * least one thread is waiting on this lock.
+ * This function is called if lockstat is active or the first try at releasing
+ * a write lock failed.  The latter means that the lock is recursed or one of
+ * the 2 waiter bits must be set indicating that at least one thread is waiting
+ * on this lock.
  */
 void
 __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
@@ -1003,14 +1059,19 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
 		return;
 
 	rw = rwlock2rw(c);
-
-	if (rw_wlocked(rw) && rw_recursed(rw)) {
-		rw->rw_recurse--;
+	v = RW_READ_VALUE(rw);
+	if (v & RW_LOCK_WRITER_RECURSED) {
+		if (--(rw->rw_recurse) == 0)
+			atomic_clear_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
 			CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
 		return;
 	}
 
+	LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_WRITER);
+	if (v == tid && _rw_write_unlock(rw, tid))
+		return;
+
 	KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
 	    ("%s: neither of the waiter flags are set", __func__));
 
diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c
index 64b7583..3236e03 100644
--- a/sys/kern/kern_sx.c
+++ b/sys/kern/kern_sx.c
@@ -148,32 +148,14 @@ static SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging");
 SYSCTL_UINT(_debug_sx, OID_AUTO, retries, CTLFLAG_RW, &asx_retries, 0, "");
 SYSCTL_UINT(_debug_sx, OID_AUTO, loops, CTLFLAG_RW, &asx_loops, 0, "");
 
-static struct lock_delay_config __read_mostly sx_delay = {
-	.initial	= 1000,
-	.step           = 500,
-	.min		= 100,
-	.max		= 5000,
-};
+static struct lock_delay_config __read_mostly sx_delay;
 
-SYSCTL_INT(_debug_sx, OID_AUTO, delay_initial, CTLFLAG_RW, &sx_delay.initial,
-    0, "");
-SYSCTL_INT(_debug_sx, OID_AUTO, delay_step, CTLFLAG_RW, &sx_delay.step,
-    0, "");
-SYSCTL_INT(_debug_sx, OID_AUTO, delay_min, CTLFLAG_RW, &sx_delay.min,
+SYSCTL_INT(_debug_sx, OID_AUTO, delay_base, CTLFLAG_RW, &sx_delay.base,
     0, "");
 SYSCTL_INT(_debug_sx, OID_AUTO, delay_max, CTLFLAG_RW, &sx_delay.max,
     0, "");
 
-static void
-sx_delay_sysinit(void *dummy)
-{
-
-	sx_delay.initial = mp_ncpus * 25;
-	sx_delay.step = (mp_ncpus * 25) / 2;
-	sx_delay.min = mp_ncpus * 5;
-	sx_delay.max = mp_ncpus * 25 * 10;
-}
-LOCK_DELAY_SYSINIT(sx_delay_sysinit);
+LOCK_DELAY_SYSINIT_DEFAULT(sx_delay);
 #endif
 
 void
@@ -276,29 +258,6 @@ sx_destroy(struct sx *sx)
 }
 
 int
-_sx_slock(struct sx *sx, int opts, const char *file, int line)
-{
-	int error = 0;
-
-	if (SCHEDULER_STOPPED())
-		return (0);
-	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
-	    ("sx_slock() by idle thread %p on sx %s @ %s:%d",
-	    curthread, sx->lock_object.lo_name, file, line));
-	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
-	    ("sx_slock() of destroyed sx @ %s:%d", file, line));
-	WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL);
-	error = __sx_slock(sx, opts, file, line);
-	if (!error) {
-		LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
-		WITNESS_LOCK(&sx->lock_object, 0, file, line);
-		TD_LOCKS_INC(curthread);
-	}
-
-	return (error);
-}
-
-int
 sx_try_slock_(struct sx *sx, const char *file, int line)
 {
 	uintptr_t x;
@@ -310,13 +269,13 @@ sx_try_slock_(struct sx *sx, const char *file, int line)
 	    ("sx_try_slock() by idle thread %p on sx %s @ %s:%d",
 	    curthread, sx->lock_object.lo_name, file, line));
 
+	x = sx->sx_lock;
 	for (;;) {
-		x = sx->sx_lock;
 		KASSERT(x != SX_LOCK_DESTROYED,
 		    ("sx_try_slock() of destroyed sx @ %s:%d", file, line));
 		if (!(x & SX_LOCK_SHARED))
 			break;
-		if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
+		if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, x + SX_ONE_SHARER)) {
 			LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
 			WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
 			LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire,
@@ -333,10 +292,9 @@ sx_try_slock_(struct sx *sx, const char *file, int line)
 int
 _sx_xlock(struct sx *sx, int opts, const char *file, int line)
 {
+	uintptr_t tid, x;
 	int error = 0;
 
-	if (SCHEDULER_STOPPED())
-		return (0);
 	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 	    ("sx_xlock() by idle thread %p on sx %s @ %s:%d",
 	    curthread, sx->lock_object.lo_name, file, line));
@@ -344,7 +302,13 @@ _sx_xlock(struct sx *sx, int opts, const char *file, int line)
 	    ("sx_xlock() of destroyed sx @ %s:%d", file, line));
 	WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
 	    line, NULL);
-	error = __sx_xlock(sx, curthread, opts, file, line);
+	tid = (uintptr_t)curthread;
+	x = SX_LOCK_UNLOCKED;
+	if (!atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
+		error = _sx_xlock_hard(sx, x, tid, opts, file, line);
+	else
+		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
+		    0, 0, file, line, LOCKSTAT_WRITER);
 	if (!error) {
 		LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse,
 		    file, line);
@@ -358,9 +322,14 @@ _sx_xlock(struct sx *sx, int opts, const char *file, int line)
 int
 sx_try_xlock_(struct sx *sx, const char *file, int line)
 {
+	struct thread *td;
+	uintptr_t tid, x;
 	int rval;
+	bool recursed;
 
-	if (SCHEDULER_STOPPED())
+	td = curthread;
+	tid = (uintptr_t)td;
+	if (SCHEDULER_STOPPED_TD(td))
 		return (1);
 
 	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
@@ -369,19 +338,28 @@ sx_try_xlock_(struct sx *sx, const char *file, int line)
 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 	    ("sx_try_xlock() of destroyed sx @ %s:%d", file, line));
 
-	if (sx_xlocked(sx) &&
-	    (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) {
-		sx->sx_recurse++;
-		atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
-		rval = 1;
-	} else
-		rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
-		    (uintptr_t)curthread);
+	rval = 1;
+	recursed = false;
+	x = SX_LOCK_UNLOCKED;
+	for (;;) {
+		if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
+			break;
+		if (x == SX_LOCK_UNLOCKED)
+			continue;
+		if (x == tid && (sx->lock_object.lo_flags & LO_RECURSABLE)) {
+			sx->sx_recurse++;
+			atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
+			break;
+		}
+		rval = 0;
+		break;
+	}
+
 	LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line);
 	if (rval) {
 		WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 		    file, line);
-		if (!sx_recursed(sx))
+		if (!recursed)
 			LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire,
 			    sx, 0, 0, file, line, LOCKSTAT_WRITER);
 		TD_LOCKS_INC(curthread);
@@ -391,33 +369,20 @@ sx_try_xlock_(struct sx *sx, const char *file, int line)
 }
 
 void
-_sx_sunlock(struct sx *sx, const char *file, int line)
-{
-
-	if (SCHEDULER_STOPPED())
-		return;
-	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
-	    ("sx_sunlock() of destroyed sx @ %s:%d", file, line));
-	_sx_assert(sx, SA_SLOCKED, file, line);
-	WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
-	LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
-	__sx_sunlock(sx, file, line);
-	TD_LOCKS_DEC(curthread);
-}
-
-void
 _sx_xunlock(struct sx *sx, const char *file, int line)
 {
 
-	if (SCHEDULER_STOPPED())
-		return;
 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 	    ("sx_xunlock() of destroyed sx @ %s:%d", file, line));
 	_sx_assert(sx, SA_XLOCKED, file, line);
 	WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
 	LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
 	    line);
+#if LOCK_DEBUG > 0
+	_sx_xunlock_hard(sx, (uintptr_t)curthread, file, line);
+#else
 	__sx_xunlock(sx, curthread, file, line);
+#endif
 	TD_LOCKS_DEC(curthread);
 }
 
@@ -530,15 +495,14 @@ sx_downgrade_(struct sx *sx, const char *file, int line)
  * accessible from at least sx.h.
  */
 int
-_sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file,
-    int line)
+_sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t tid, int opts,
+    const char *file, int line)
 {
 	GIANT_DECLARE;
 #ifdef ADAPTIVE_SX
 	volatile struct thread *owner;
 	u_int i, spintries = 0;
 #endif
-	uintptr_t x;
 #ifdef LOCK_PROFILING
 	uint64_t waittime = 0;
 	int contested = 0;
@@ -563,7 +527,8 @@ _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file,
 	lock_delay_arg_init(&lda, NULL);
 #endif
 
-	x = SX_READ_VALUE(sx);
+	if (__predict_false(x == SX_LOCK_UNLOCKED))
+		x = SX_READ_VALUE(sx);
 
 	/* If we already hold an exclusive lock, then recurse. */
 	if (__predict_false(lv_sx_owner(x) == (struct thread *)tid)) {
@@ -587,9 +552,8 @@ _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file,
 #endif
 	for (;;) {
 		if (x == SX_LOCK_UNLOCKED) {
-			if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, tid))
+			if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
 				break;
-			x = SX_READ_VALUE(sx);
 			continue;
 		}
 #ifdef KDTRACE_HOOKS
@@ -799,14 +763,21 @@ _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
 
 	MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
 
-	/* If the lock is recursed, then unrecurse one level. */
-	if (sx_xlocked(sx) && sx_recursed(sx)) {
+	x = SX_READ_VALUE(sx);
+	if (x & SX_LOCK_RECURSED) {
+		/* The lock is recursed, unrecurse one level. */
 		if ((--sx->sx_recurse) == 0)
 			atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
 		if (LOCK_LOG_TEST(&sx->lock_object, 0))
 			CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx);
 		return;
 	}
+
+	LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_WRITER);
+	if (x == tid &&
+	    atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED))
+		return;
+
 	MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS |
 	    SX_LOCK_EXCLUSIVE_WAITERS));
 	if (LOCK_LOG_TEST(&sx->lock_object, 0))
@@ -844,14 +815,32 @@ _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
 		kick_proc0();
 }
 
-/*
- * This function represents the so-called 'hard case' for sx_slock
- * operation.  All 'easy case' failures are redirected to this.  Note
- * that ideally this would be a static function, but it needs to be
- * accessible from at least sx.h.
- */
-int
-_sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
+static bool __always_inline
+__sx_slock_try(struct sx *sx, uintptr_t *xp, const char *file, int line)
+{
+
+	/*
+	 * If no other thread has an exclusive lock then try to bump up
+	 * the count of sharers.  Since we have to preserve the state
+	 * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the
+	 * shared lock loop back and retry.
+	 */
+	while (*xp & SX_LOCK_SHARED) {
+		MPASS(!(*xp & SX_LOCK_SHARED_WAITERS));
+		if (atomic_fcmpset_acq_ptr(&sx->sx_lock, xp,
+		    *xp + SX_ONE_SHARER)) {
+			if (LOCK_LOG_TEST(&sx->lock_object, 0))
+				CTR4(KTR_LOCK, "%s: %p succeed %p -> %p",
+				    __func__, sx, (void *)*xp,
+				    (void *)(*xp + SX_ONE_SHARER));
+			return (true);
+		}
+	}
+	return (false);
+}
+
+static int __noinline
+_sx_slock_hard(struct sx *sx, int opts, const char *file, int line, uintptr_t x)
 {
 	GIANT_DECLARE;
 #ifdef ADAPTIVE_SX
@@ -861,7 +850,6 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
 	uint64_t waittime = 0;
 	int contested = 0;
 #endif
-	uintptr_t x;
 	int error = 0;
 #if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS)
 	struct lock_delay_arg lda;
@@ -883,9 +871,6 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
 #endif
 #ifdef KDTRACE_HOOKS
 	all_time -= lockstat_nsecs(&sx->lock_object);
-#endif
-	x = SX_READ_VALUE(sx);
-#ifdef KDTRACE_HOOKS
 	state = x;
 #endif
 
@@ -894,26 +879,8 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
 	 * shared locks once there is an exclusive waiter.
 	 */
 	for (;;) {
-		/*
-		 * If no other thread has an exclusive lock then try to bump up
-		 * the count of sharers.  Since we have to preserve the state
-		 * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the
-		 * shared lock loop back and retry.
-		 */
-		if (x & SX_LOCK_SHARED) {
-			MPASS(!(x & SX_LOCK_SHARED_WAITERS));
-			if (atomic_cmpset_acq_ptr(&sx->sx_lock, x,
-			    x + SX_ONE_SHARER)) {
-				if (LOCK_LOG_TEST(&sx->lock_object, 0))
-					CTR4(KTR_LOCK,
-					    "%s: %p succeed %p -> %p", __func__,
-					    sx, (void *)x,
-					    (void *)(x + SX_ONE_SHARER));
-				break;
-			}
-			x = SX_READ_VALUE(sx);
-			continue;
-		}
+		if (__sx_slock_try(sx, &x, file, line))
+			break;
 #ifdef KDTRACE_HOOKS
 		lda.spin_cnt++;
 #endif
@@ -1049,53 +1016,66 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
 		    LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
 		    (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
 #endif
-	if (error == 0)
+	if (error == 0) {
 		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
 		    contested, waittime, file, line, LOCKSTAT_READER);
+	}
 	GIANT_RESTORE();
 	return (error);
 }
 
-/*
- * This function represents the so-called 'hard case' for sx_sunlock
- * operation.  All 'easy case' failures are redirected to this.  Note
- * that ideally this would be a static function, but it needs to be
- * accessible from at least sx.h.
- */
-void
-_sx_sunlock_hard(struct sx *sx, const char *file, int line)
+int
+_sx_slock(struct sx *sx, int opts, const char *file, int line)
 {
 	uintptr_t x;
-	int wakeup_swapper;
+	int error;
 
-	if (SCHEDULER_STOPPED())
-		return;
+	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
+	    ("sx_slock() by idle thread %p on sx %s @ %s:%d",
+	    curthread, sx->lock_object.lo_name, file, line));
+	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
+	    ("sx_slock() of destroyed sx @ %s:%d", file, line));
+	WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL);
 
+	error = 0;
 	x = SX_READ_VALUE(sx);
+	if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(sx__acquire) ||
+	    !__sx_slock_try(sx, &x, file, line)))
+		error = _sx_slock_hard(sx, opts, file, line, x);
+	if (error == 0) {
+		LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
+		WITNESS_LOCK(&sx->lock_object, 0, file, line);
+		TD_LOCKS_INC(curthread);
+	}
+	return (error);
+}
+
+static bool __always_inline
+_sx_sunlock_try(struct sx *sx, uintptr_t *xp)
+{
+
 	for (;;) {
 		/*
 		 * We should never have sharers while at least one thread
 		 * holds a shared lock.
 		 */
-		KASSERT(!(x & SX_LOCK_SHARED_WAITERS),
+		KASSERT(!(*xp & SX_LOCK_SHARED_WAITERS),
 		    ("%s: waiting sharers", __func__));
 
 		/*
 		 * See if there is more than one shared lock held.  If
 		 * so, just drop one and return.
 		 */
-		if (SX_SHARERS(x) > 1) {
-			if (atomic_cmpset_rel_ptr(&sx->sx_lock, x,
-			    x - SX_ONE_SHARER)) {
+		if (SX_SHARERS(*xp) > 1) {
+			if (atomic_fcmpset_rel_ptr(&sx->sx_lock, xp,
+			    *xp - SX_ONE_SHARER)) {
 				if (LOCK_LOG_TEST(&sx->lock_object, 0))
 					CTR4(KTR_LOCK,
 					    "%s: %p succeeded %p -> %p",
-					    __func__, sx, (void *)x,
-					    (void *)(x - SX_ONE_SHARER));
-				break;
+					    __func__, sx, (void *)*xp,
+					    (void *)(*xp - SX_ONE_SHARER));
+				return (true);
 			}
-
-			x = SX_READ_VALUE(sx);
 			continue;
 		}
 
@@ -1103,18 +1083,36 @@ _sx_sunlock_hard(struct sx *sx, const char *file, int line)
 		 * If there aren't any waiters for an exclusive lock,
 		 * then try to drop it quickly.
 		 */
-		if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) {
-			MPASS(x == SX_SHARERS_LOCK(1));
-			if (atomic_cmpset_rel_ptr(&sx->sx_lock,
-			    SX_SHARERS_LOCK(1), SX_LOCK_UNLOCKED)) {
+		if (!(*xp & SX_LOCK_EXCLUSIVE_WAITERS)) {
+			MPASS(*xp == SX_SHARERS_LOCK(1));
+			*xp = SX_SHARERS_LOCK(1);
+			if (atomic_fcmpset_rel_ptr(&sx->sx_lock,
+			    xp, SX_LOCK_UNLOCKED)) {
 				if (LOCK_LOG_TEST(&sx->lock_object, 0))
 					CTR2(KTR_LOCK, "%s: %p last succeeded",
 					    __func__, sx);
-				break;
+				return (true);
 			}
-			x = SX_READ_VALUE(sx);
 			continue;
 		}
+		break;
+	}
+	return (false);
+}
+
+static void __noinline
+_sx_sunlock_hard(struct sx *sx, uintptr_t x, const char *file, int line)
+{
+	int wakeup_swapper;
+
+	if (SCHEDULER_STOPPED())
+		return;
+
+	LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_READER);
+
+	for (;;) {
+		if (_sx_sunlock_try(sx, &x))
+			break;
 
 		/*
 		 * At this point, there should just be one sharer with
@@ -1149,6 +1147,25 @@ _sx_sunlock_hard(struct sx *sx, const char *file, int line)
 	}
 }
 
+void
+_sx_sunlock(struct sx *sx, const char *file, int line)
+{
+	uintptr_t x;
+
+	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
+	    ("sx_sunlock() of destroyed sx @ %s:%d", file, line));
+	_sx_assert(sx, SA_SLOCKED, file, line);
+	WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
+	LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
+
+	x = SX_READ_VALUE(sx);
+	if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(sx__release) ||
+	    !_sx_sunlock_try(sx, &x)))
+		_sx_sunlock_hard(sx, x, file, line);
+
+	TD_LOCKS_DEC(curthread);
+}
+
 #ifdef INVARIANT_SUPPORT
 #ifndef INVARIANTS
 #undef	_sx_assert
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 847c6d7..87f1a36 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -162,7 +162,7 @@ _sleep(void *ident, struct lock_object *lock, int priority,
 	else
 		class = NULL;
 
-	if (SCHEDULER_STOPPED()) {
+	if (SCHEDULER_STOPPED_TD(td)) {
 		if (lock != NULL && priority & PDROP)
 			class->lc_unlock(lock);
 		return (0);
@@ -250,7 +250,7 @@ msleep_spin_sbt(void *ident, struct mtx *mtx, const char *wmesg,
 	KASSERT(ident != NULL, ("msleep_spin_sbt: NULL ident"));
 	KASSERT(TD_IS_RUNNING(td), ("msleep_spin_sbt: curthread not running"));
 
-	if (SCHEDULER_STOPPED())
+	if (SCHEDULER_STOPPED_TD(td))
 		return (0);
 
 	sleepq_lock(ident);
@@ -411,7 +411,7 @@ mi_switch(int flags, struct thread *newtd)
 	 */
 	if (kdb_active)
 		kdb_switch();
-	if (SCHEDULER_STOPPED())
+	if (SCHEDULER_STOPPED_TD(td))
 		return;
 	if (flags & SW_VOL) {
 		td->td_ru.ru_nvcsw++;
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
index 3a73098..0c8365a 100644
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -281,7 +281,7 @@ threadinit(void)
 
 	thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
 	    thread_ctor, thread_dtor, thread_init, thread_fini,
-	    16 - 1, UMA_ZONE_NOFREE);
+	    32 - 1, UMA_ZONE_NOFREE);
 	tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash);
 	rw_init(&tidhash_lock, "tidhash");
 }
diff --git a/sys/kern/subr_lock.c b/sys/kern/subr_lock.c
index f83ba58..8584525 100644
--- a/sys/kern/subr_lock.c
+++ b/sys/kern/subr_lock.c
@@ -56,6 +56,9 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/cpufunc.h>
 
+SDT_PROVIDER_DEFINE(lock);
+SDT_PROBE_DEFINE1(lock, , , starvation, "u_int");
+
 CTASSERT(LOCK_CLASS_MAX == 15);
 
 struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = {
@@ -103,32 +106,56 @@ lock_destroy(struct lock_object *lock)
 	lock->lo_flags &= ~LO_INITIALIZED;
 }
 
+static SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging");
+static SYSCTL_NODE(_debug_lock, OID_AUTO, delay, CTLFLAG_RD, NULL,
+    "lock delay");
+
+static u_int __read_mostly starvation_limit = 131072;
+SYSCTL_INT(_debug_lock_delay, OID_AUTO, starvation_limit, CTLFLAG_RW,
+    &starvation_limit, 0, "");
+
+static u_int __read_mostly restrict_starvation = 0;
+SYSCTL_INT(_debug_lock_delay, OID_AUTO, restrict_starvation, CTLFLAG_RW,
+    &restrict_starvation, 0, "");
+
 void
 lock_delay(struct lock_delay_arg *la)
 {
-	u_int i, delay, backoff, min, max;
 	struct lock_delay_config *lc = la->config;
+	u_int i;
 
-	delay = la->delay;
+	la->delay <<= 1;
+	if (__predict_false(la->delay > lc->max))
+		la->delay = lc->max;
 
-	if (delay == 0)
-		delay = lc->initial;
-	else {
-		delay += lc->step;
-		max = lc->max;
-		if (delay > max)
-			delay = max;
+	for (i = la->delay; i > 0; i--)
+		cpu_spinwait();
+
+	la->spin_cnt += la->delay;
+	if (__predict_false(la->spin_cnt > starvation_limit)) {
+		SDT_PROBE1(lock, , , starvation, la->delay);
+		if (restrict_starvation)
+			la->delay = lc->base;
 	}
+}
 
-	backoff = cpu_ticks() % delay;
-	min = lc->min;
-	if (backoff < min)
-		backoff = min;
-	for (i = 0; i < backoff; i++)
-		cpu_spinwait();
+static u_int
+lock_roundup_2(u_int val)
+{
+	u_int res;
+
+	for (res = 1; res <= val; res <<= 1)
+		continue;
+
+	return (res);
+}
 
-	la->delay = delay;
-	la->spin_cnt += backoff;
+void
+lock_delay_default_init(struct lock_delay_config *lc)
+{
+
+	lc->base = lock_roundup_2(mp_ncpus) / 4;
+	lc->max = lc->base * 1024;
 }
 
 #ifdef DDB
@@ -655,7 +682,6 @@ out:
 	critical_exit();
 }
 
-static SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging");
 static SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL,
     "lock profiling");
 SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipspin, CTLFLAG_RW,
diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c
index fe34e56..5c0d1b8 100644
--- a/sys/kern/vfs_default.c
+++ b/sys/kern/vfs_default.c
@@ -520,10 +520,11 @@ vop_stdlock(ap)
 	} */ *ap;
 {
 	struct vnode *vp = ap->a_vp;
+	struct mtx *ilk;
 
-	return (_lockmgr_args(vp->v_vnlock, ap->a_flags, VI_MTX(vp),
-	    LK_WMESG_DEFAULT, LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, ap->a_file,
-	    ap->a_line));
+	ilk = VI_MTX(vp);
+	return (lockmgr_lock_fast_path(vp->v_vnlock, ap->a_flags,
+	    (ilk != NULL) ? &ilk->lock_object : NULL, ap->a_file, ap->a_line));
 }
 
 /* See above. */
@@ -535,8 +536,11 @@ vop_stdunlock(ap)
 	} */ *ap;
 {
 	struct vnode *vp = ap->a_vp;
+	struct mtx *ilk;
 
-	return (lockmgr(vp->v_vnlock, ap->a_flags | LK_RELEASE, VI_MTX(vp)));
+	ilk = VI_MTX(vp);
+	return (lockmgr_unlock_fast_path(vp->v_vnlock, ap->a_flags,
+	    (ilk != NULL) ? &ilk->lock_object : NULL));
 }
 
 /* See above. */
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index 792fbb6..039d068 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -2461,11 +2461,11 @@ vfs_refcount_acquire_if_not_zero(volatile u_int *count)
 {
 	u_int old;
 
+	old = *count;
 	for (;;) {
-		old = *count;
 		if (old == 0)
 			return (0);
-		if (atomic_cmpset_int(count, old, old + 1))
+		if (atomic_fcmpset_int(count, &old, old + 1))
 			return (1);
 	}
 }
@@ -2475,11 +2475,11 @@ vfs_refcount_release_if_not_last(volatile u_int *count)
 {
 	u_int old;
 
+	old = *count;
 	for (;;) {
-		old = *count;
 		if (old == 1)
 			return (0);
-		if (atomic_cmpset_int(count, old, old - 1))
+		if (atomic_fcmpset_int(count, &old, old - 1))
 			return (1);
 	}
 }
diff --git a/sys/sys/lock.h b/sys/sys/lock.h
index dbe715a..08ccaa5 100644
--- a/sys/sys/lock.h
+++ b/sys/sys/lock.h
@@ -154,8 +154,13 @@ struct lock_class {
  * file    - file name
  * line    - line number
  */
+#if LOCK_DEBUG > 0
 #define	LOCK_LOG_TEST(lo, flags)					\
 	(((flags) & LOP_QUIET) == 0 && ((lo)->lo_flags & LO_QUIET) == 0)
+#else
+#define	LOCK_LOG_TEST(lo, flags)	0
+#endif
+
 
 #define	LOCK_LOG_LOCK(opname, lo, flags, recurse, file, line) do {	\
 	if (LOCK_LOG_TEST((lo), (flags)))				\
@@ -202,9 +207,7 @@ extern struct lock_class lock_class_lockmgr;
 extern struct lock_class *lock_classes[];
 
 struct lock_delay_config {
-	u_int initial;
-	u_int step;
-	u_int min;
+	u_int base;
 	u_int max;
 };
 
@@ -215,19 +218,25 @@ struct lock_delay_arg {
 };
 
 static inline void
-lock_delay_arg_init(struct lock_delay_arg *la, struct lock_delay_config *lc) {
+lock_delay_arg_init(struct lock_delay_arg *la, struct lock_delay_config *lc)
+{
 	la->config = lc;
-	la->delay = 0;
+	la->delay = lc->base;
 	la->spin_cnt = 0;
 }
 
 #define	LOCK_DELAY_SYSINIT(func) \
 	SYSINIT(func##_ld, SI_SUB_LOCK, SI_ORDER_ANY, func, NULL)
 
+#define	LOCK_DELAY_SYSINIT_DEFAULT(lc) \
+	SYSINIT(lock_delay_##lc##_ld, SI_SUB_LOCK, SI_ORDER_ANY, \
+	    lock_delay_default_init, &lc)
+
 void	lock_init(struct lock_object *, struct lock_class *,
 	    const char *, const char *, int);
 void	lock_destroy(struct lock_object *);
 void	lock_delay(struct lock_delay_arg *);
+void	lock_delay_default_init(struct lock_delay_config *);
 void	spinlock_enter(void);
 void	spinlock_exit(void);
 void	witness_init(struct lock_object *, const char *);
diff --git a/sys/sys/lockmgr.h b/sys/sys/lockmgr.h
index 3019e4c..6074922 100644
--- a/sys/sys/lockmgr.h
+++ b/sys/sys/lockmgr.h
@@ -68,6 +68,10 @@ struct thread;
  */
 int	 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 	    const char *wmesg, int prio, int timo, const char *file, int line);
+int	 lockmgr_lock_fast_path(struct lock *lk, u_int flags,
+	    struct lock_object *ilk, const char *file, int line);
+int	 lockmgr_unlock_fast_path(struct lock *lk, u_int flags,
+	    struct lock_object *ilk);
 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
 void	 _lockmgr_assert(const struct lock *lk, int what, const char *file, int line);
 #endif
diff --git a/sys/sys/lockstat.h b/sys/sys/lockstat.h
index e550353..6474505 100644
--- a/sys/sys/lockstat.h
+++ b/sys/sys/lockstat.h
@@ -68,7 +68,7 @@ SDT_PROBE_DECLARE(lockstat, , , thread__spin);
 #define	LOCKSTAT_WRITER		0
 #define	LOCKSTAT_READER		1
 
-extern int lockstat_enabled;
+extern volatile int lockstat_enabled;
 
 #ifdef KDTRACE_HOOKS
 
@@ -107,6 +107,13 @@ extern int lockstat_enabled;
 	LOCKSTAT_RECORD1(probe, lp, a);					\
 } while (0)
 
+#ifndef LOCK_PROFILING
+#define	LOCKSTAT_PROFILE_ENABLED(probe)		__predict_false(lockstat_enabled)
+#define	LOCKSTAT_OOL_PROFILE_ENABLED(probe)	LOCKSTAT_PROFILE_ENABLED(probe)
+#else
+#define	LOCKSTAT_OOL_PROFILE_ENABLED(probe)	1
+#endif
+
 struct lock_object;
 uint64_t lockstat_nsecs(struct lock_object *);
 
@@ -130,6 +137,12 @@ uint64_t lockstat_nsecs(struct lock_object *);
 #define	LOCKSTAT_PROFILE_RELEASE_RWLOCK(probe, lp, a)  			\
 	LOCKSTAT_PROFILE_RELEASE_LOCK(probe, lp)
 
+#ifndef LOCK_PROFILING
+#define	LOCKSTAT_PROFILE_ENABLED(probe)		0
+#endif
+#define	LOCKSTAT_OOL_PROFILE_ENABLED(probe)	1
+
 #endif /* !KDTRACE_HOOKS */
+
 #endif /* _KERNEL */
 #endif /* _SYS_LOCKSTAT_H */
diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h
index 11ed9d7..cc56c90 100644
--- a/sys/sys/mutex.h
+++ b/sys/sys/mutex.h
@@ -98,13 +98,19 @@ void	mtx_sysinit(void *arg);
 int	_mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file,
 	    int line);
 void	mutex_init(void);
-void	__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
-	    const char *file, int line);
+#if LOCK_DEBUG > 0
+void	__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
+	    int opts, const char *file, int line);
 void	__mtx_unlock_sleep(volatile uintptr_t *c, int opts, const char *file,
 	    int line);
+#else
+void	__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, uintptr_t tid);
+void	__mtx_unlock_sleep(volatile uintptr_t *c);
+#endif
+
 #ifdef SMP
-void	_mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts,
-	    const char *file, int line);
+void	_mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
+	    int opts, const char *file, int line);
 #endif
 void	__mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file,
 	    int line);
@@ -140,13 +146,20 @@ void	thread_lock_flags_(struct thread *, int, const char *, int);
 	_mtx_destroy(&(m)->mtx_lock)
 #define	mtx_trylock_flags_(m, o, f, l)					\
 	_mtx_trylock_flags_(&(m)->mtx_lock, o, f, l)
-#define	_mtx_lock_sleep(m, t, o, f, l)					\
-	__mtx_lock_sleep(&(m)->mtx_lock, t, o, f, l)
+#if LOCK_DEBUG > 0
+#define	_mtx_lock_sleep(m, v, t, o, f, l)				\
+	__mtx_lock_sleep(&(m)->mtx_lock, v, t, o, f, l)
 #define	_mtx_unlock_sleep(m, o, f, l)					\
 	__mtx_unlock_sleep(&(m)->mtx_lock, o, f, l)
+#else
+#define	_mtx_lock_sleep(m, v, t, o, f, l)				\
+	__mtx_lock_sleep(&(m)->mtx_lock, v, t)
+#define	_mtx_unlock_sleep(m, o, f, l)					\
+	__mtx_unlock_sleep(&(m)->mtx_lock)
+#endif
 #ifdef SMP
-#define	_mtx_lock_spin(m, t, o, f, l)					\
-	_mtx_lock_spin_cookie(&(m)->mtx_lock, t, o, f, l)
+#define	_mtx_lock_spin(m, v, t, o, f, l)				\
+	_mtx_lock_spin_cookie(&(m)->mtx_lock, v, t, o, f, l)
 #endif
 #define	_mtx_lock_flags(m, o, f, l)					\
 	__mtx_lock_flags(&(m)->mtx_lock, o, f, l)
@@ -171,6 +184,9 @@ void	thread_lock_flags_(struct thread *, int, const char *, int);
 #define _mtx_obtain_lock(mp, tid)					\
 	atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid))
 
+#define _mtx_obtain_lock_fetch(mp, vp, tid)				\
+	atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, vp, (tid))
+
 /* Try to release mtx_lock if it is unrecursed and uncontested. */
 #define _mtx_release_lock(mp, tid)					\
 	atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), MTX_UNOWNED)
@@ -188,12 +204,11 @@ void	thread_lock_flags_(struct thread *, int, const char *, int);
 /* Lock a normal mutex. */
 #define __mtx_lock(mp, tid, opts, file, line) do {			\
 	uintptr_t _tid = (uintptr_t)(tid);				\
+	uintptr_t _v = MTX_UNOWNED;					\
 									\
-	if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid)))\
-		_mtx_lock_sleep((mp), _tid, (opts), (file), (line));	\
-	else								\
-		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire,	\
-		    mp, 0, 0, file, line);				\
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__acquire) ||\
+	    !_mtx_obtain_lock_fetch((mp), &_v, _tid)))			\
+		_mtx_lock_sleep((mp), _v, _tid, (opts), (file), (line));\
 } while (0)
 
 /*
@@ -205,14 +220,12 @@ void	thread_lock_flags_(struct thread *, int, const char *, int);
 #ifdef SMP
 #define __mtx_lock_spin(mp, tid, opts, file, line) do {			\
 	uintptr_t _tid = (uintptr_t)(tid);				\
+	uintptr_t _v = MTX_UNOWNED;					\
 									\
 	spinlock_enter();						\
-	if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid))) {\
-		if ((mp)->mtx_lock == _tid)				\
-			(mp)->mtx_recurse++;				\
-		else							\
-			_mtx_lock_spin((mp), _tid, (opts), (file), (line)); \
-	} else 								\
+	if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) 			\
+		_mtx_lock_spin((mp), _v, _tid, (opts), (file), (line)); \
+	else 								\
 		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire,	\
 		    mp, 0, 0, file, line);				\
 } while (0)
@@ -263,9 +276,8 @@ void	thread_lock_flags_(struct thread *, int, const char *, int);
 #define __mtx_unlock(mp, tid, opts, file, line) do {			\
 	uintptr_t _tid = (uintptr_t)(tid);				\
 									\
-	if ((mp)->mtx_recurse == 0)					\
-		LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, mp);	\
-	if ((mp)->mtx_lock != _tid || !_mtx_release_lock((mp), _tid))	\
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__release) ||\
+	    !_mtx_release_lock((mp), _tid)))				\
 		_mtx_unlock_sleep((mp), (opts), (file), (line));	\
 } while (0)
 
diff --git a/sys/sys/rwlock.h b/sys/sys/rwlock.h
index e5b1166..aa4b0cc 100644
--- a/sys/sys/rwlock.h
+++ b/sys/sys/rwlock.h
@@ -58,13 +58,14 @@
 #define	RW_LOCK_READ_WAITERS	0x02
 #define	RW_LOCK_WRITE_WAITERS	0x04
 #define	RW_LOCK_WRITE_SPINNER	0x08
+#define	RW_LOCK_WRITER_RECURSED	0x10
 #define	RW_LOCK_FLAGMASK						\
 	(RW_LOCK_READ | RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS |	\
-	RW_LOCK_WRITE_SPINNER)
+	RW_LOCK_WRITE_SPINNER | RW_LOCK_WRITER_RECURSED)
 #define	RW_LOCK_WAITERS		(RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)
 
 #define	RW_OWNER(x)		((x) & ~RW_LOCK_FLAGMASK)
-#define	RW_READERS_SHIFT	4
+#define	RW_READERS_SHIFT	5
 #define	RW_READERS(x)		(RW_OWNER((x)) >> RW_READERS_SHIFT)
 #define	RW_READERS_LOCK(x)	((x) << RW_READERS_SHIFT | RW_LOCK_READ)
 #define	RW_ONE_READER		(1 << RW_READERS_SHIFT)
@@ -84,6 +85,9 @@
 #define	_rw_write_lock(rw, tid)						\
 	atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid))
 
+#define	_rw_write_lock_fetch(rw, vp, tid)				\
+	atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid))
+
 /* Release a write lock quickly if there are no waiters. */
 #define	_rw_write_unlock(rw, tid)					\
 	atomic_cmpset_rel_ptr(&(rw)->rw_lock, (tid), RW_UNLOCKED)
@@ -97,26 +101,20 @@
 /* Acquire a write lock. */
 #define	__rw_wlock(rw, tid, file, line) do {				\
 	uintptr_t _tid = (uintptr_t)(tid);				\
+	uintptr_t _v = RW_UNLOCKED;					\
 									\
-	if ((rw)->rw_lock != RW_UNLOCKED || !_rw_write_lock((rw), _tid))\
-		_rw_wlock_hard((rw), _tid, (file), (line));		\
-	else 								\
-		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,	\
-		    0, 0, file, line, LOCKSTAT_WRITER);			\
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) ||	\
+	    !_rw_write_lock_fetch((rw), &_v, _tid)))			\
+		_rw_wlock_hard((rw), _v, _tid, (file), (line));		\
 } while (0)
 
 /* Release a write lock. */
 #define	__rw_wunlock(rw, tid, file, line) do {				\
 	uintptr_t _tid = (uintptr_t)(tid);				\
 									\
-	if ((rw)->rw_recurse)						\
-		(rw)->rw_recurse--;					\
-	else {								\
-		LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw,	\
-		    LOCKSTAT_WRITER);					\
-		if ((rw)->rw_lock != _tid || !_rw_write_unlock((rw), _tid))\
-			_rw_wunlock_hard((rw), _tid, (file), (line));	\
-	}								\
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) ||	\
+	    !_rw_write_unlock((rw), _tid)))				\
+		_rw_wunlock_hard((rw), _tid, (file), (line));		\
 } while (0)
 
 /*
@@ -135,8 +133,8 @@ void	_rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line);
 void	__rw_rlock(volatile uintptr_t *c, const char *file, int line);
 int	__rw_try_rlock(volatile uintptr_t *c, const char *file, int line);
 void	_rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line);
-void	__rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
-	    int line);
+void	__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
+	    const char *file, int line);
 void	__rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid,
 	    const char *file, int line);
 int	__rw_try_upgrade(volatile uintptr_t *c, const char *file, int line);
@@ -171,8 +169,8 @@ void	__rw_assert(const volatile uintptr_t *c, int what, const char *file,
 	__rw_try_rlock(&(rw)->rw_lock, f, l)
 #define	_rw_runlock(rw, f, l)						\
 	_rw_runlock_cookie(&(rw)->rw_lock, f, l)
-#define	_rw_wlock_hard(rw, t, f, l)					\
-	__rw_wlock_hard(&(rw)->rw_lock, t, f, l)
+#define	_rw_wlock_hard(rw, v, t, f, l)					\
+	__rw_wlock_hard(&(rw)->rw_lock, v, t, f, l)
 #define	_rw_wunlock_hard(rw, t, f, l)					\
 	__rw_wunlock_hard(&(rw)->rw_lock, t, f, l)
 #define	_rw_try_upgrade(rw, f, l)					\
diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h
index 25423d7..c680ea8 100644
--- a/sys/sys/sdt.h
+++ b/sys/sys/sdt.h
@@ -86,6 +86,7 @@
 #define SDT_PROVIDER_DECLARE(prov)
 #define SDT_PROBE_DEFINE(prov, mod, func, name)
 #define SDT_PROBE_DECLARE(prov, mod, func, name)
+#define SDT_PROBE_ENABLED(prov, mod, func, name)	0
 #define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4)
 #define SDT_PROBE_ARGTYPE(prov, mod, func, name, num, type, xtype)
 
@@ -160,6 +161,9 @@ SET_DECLARE(sdt_argtypes_set, struct sdt_argtype);
 #define SDT_PROBE_DECLARE(prov, mod, func, name)				\
 	extern struct sdt_probe sdt_##prov##_##mod##_##func##_##name[1]
 
+#define SDT_PROBE_ENABLED(prov, mod, func, name)				\
+	__predict_false((sdt_##prov##_##mod##_##func##_##name->id))
+
 #define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4)	do {	\
 	if (__predict_false(sdt_##prov##_##mod##_##func##_##name->id))		\
 		(*sdt_probe_func)(sdt_##prov##_##mod##_##func##_##name->id,	\
diff --git a/sys/sys/sx.h b/sys/sys/sx.h
index e8cffaa..a31c328 100644
--- a/sys/sys/sx.h
+++ b/sys/sys/sx.h
@@ -109,12 +109,10 @@ int	_sx_slock(struct sx *sx, int opts, const char *file, int line);
 int	_sx_xlock(struct sx *sx, int opts, const char *file, int line);
 void	_sx_sunlock(struct sx *sx, const char *file, int line);
 void	_sx_xunlock(struct sx *sx, const char *file, int line);
-int	_sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts,
+int	_sx_xlock_hard(struct sx *sx, uintptr_t v, uintptr_t tid, int opts,
 	    const char *file, int line);
-int	_sx_slock_hard(struct sx *sx, int opts, const char *file, int line);
 void	_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int
 	    line);
-void	_sx_sunlock_hard(struct sx *sx, const char *file, int line);
 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
 void	_sx_assert(const struct sx *sx, int what, const char *file, int line);
 #endif
@@ -147,20 +145,19 @@ struct sx_args {
  * deferred to 'tougher' functions.
  */
 
+#if	(LOCK_DEBUG == 0)
 /* Acquire an exclusive lock. */
 static __inline int
 __sx_xlock(struct sx *sx, struct thread *td, int opts, const char *file,
     int line)
 {
 	uintptr_t tid = (uintptr_t)td;
+	uintptr_t v = SX_LOCK_UNLOCKED;
 	int error = 0;
 
-	if (sx->sx_lock != SX_LOCK_UNLOCKED ||
-	    !atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid))
-		error = _sx_xlock_hard(sx, tid, opts, file, line);
-	else 
-		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
-		    0, 0, file, line, LOCKSTAT_WRITER);
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__acquire) ||
+	    !atomic_fcmpset_acq_ptr(&sx->sx_lock, &v, tid)))
+		error = _sx_xlock_hard(sx, v, tid, opts, file, line);
 
 	return (error);
 }
@@ -171,48 +168,11 @@ __sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line)
 {
 	uintptr_t tid = (uintptr_t)td;
 
-	if (sx->sx_recurse == 0)
-		LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx,
-		    LOCKSTAT_WRITER);
-	if (sx->sx_lock != tid ||
-	    !atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED))
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__release) ||
+	    !atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)))
 		_sx_xunlock_hard(sx, tid, file, line);
 }
-
-/* Acquire a shared lock. */
-static __inline int
-__sx_slock(struct sx *sx, int opts, const char *file, int line)
-{
-	uintptr_t x = sx->sx_lock;
-	int error = 0;
-
-	if (!(x & SX_LOCK_SHARED) ||
-	    !atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER))
-		error = _sx_slock_hard(sx, opts, file, line);
-	else
-		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
-		    0, 0, file, line, LOCKSTAT_READER);
-
-	return (error);
-}
-
-/*
- * Release a shared lock.  We can just drop a single shared lock so
- * long as we aren't trying to drop the last shared lock when other
- * threads are waiting for an exclusive lock.  This takes advantage of
- * the fact that an unlocked lock is encoded as a shared lock with a
- * count of 0.
- */
-static __inline void
-__sx_sunlock(struct sx *sx, const char *file, int line)
-{
-	uintptr_t x = sx->sx_lock;
-
-	LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_READER);
-	if (x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS) ||
-	    !atomic_cmpset_rel_ptr(&sx->sx_lock, x, x - SX_ONE_SHARER))
-		_sx_sunlock_hard(sx, file, line);
-}
+#endif
 
 /*
  * Public interface for lock operations.
@@ -227,12 +187,6 @@ __sx_sunlock(struct sx *sx, const char *file, int line)
 	_sx_xlock((sx), SX_INTERRUPTIBLE, (file), (line))
 #define	sx_xunlock_(sx, file, line)					\
 	_sx_xunlock((sx), (file), (line))
-#define	sx_slock_(sx, file, line)					\
-	(void)_sx_slock((sx), 0, (file), (line))
-#define	sx_slock_sig_(sx, file, line)					\
-	_sx_slock((sx), SX_INTERRUPTIBLE, (file) , (line))
-#define	sx_sunlock_(sx, file, line)					\
-	_sx_sunlock((sx), (file), (line))
 #else
 #define	sx_xlock_(sx, file, line)					\
 	(void)__sx_xlock((sx), curthread, 0, (file), (line))
@@ -240,13 +194,13 @@ __sx_sunlock(struct sx *sx, const char *file, int line)
 	__sx_xlock((sx), curthread, SX_INTERRUPTIBLE, (file), (line))
 #define	sx_xunlock_(sx, file, line)					\
 	__sx_xunlock((sx), curthread, (file), (line))
+#endif	/* LOCK_DEBUG > 0 || SX_NOINLINE */
 #define	sx_slock_(sx, file, line)					\
-	(void)__sx_slock((sx), 0, (file), (line))
+	(void)_sx_slock((sx), 0, (file), (line))
 #define	sx_slock_sig_(sx, file, line)					\
-	__sx_slock((sx), SX_INTERRUPTIBLE, (file), (line))
+	_sx_slock((sx), SX_INTERRUPTIBLE, (file) , (line))
 #define	sx_sunlock_(sx, file, line)					\
-	__sx_sunlock((sx), (file), (line))
-#endif	/* LOCK_DEBUG > 0 || SX_NOINLINE */
+	_sx_sunlock((sx), (file), (line))
 #define	sx_try_slock(sx)	sx_try_slock_((sx), LOCK_FILE, LOCK_LINE)
 #define	sx_try_xlock(sx)	sx_try_xlock_((sx), LOCK_FILE, LOCK_LINE)
 #define	sx_try_upgrade(sx)	sx_try_upgrade_((sx), LOCK_FILE, LOCK_LINE)
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index 2329621..c564547 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -128,7 +128,11 @@ void	kassert_panic(const char *fmt, ...)  __printflike(1, 2);
  * Otherwise, the kernel will deadlock since the scheduler isn't
  * going to run the thread that holds any lock we need.
  */
-#define	SCHEDULER_STOPPED() __predict_false(curthread->td_stopsched)
+#define	SCHEDULER_STOPPED_TD(td)  ({					\
+	MPASS((td) == curthread);					\
+	__predict_false((td)->td_stopsched);				\
+})
+#define	SCHEDULER_STOPPED() SCHEDULER_STOPPED_TD(curthread)
 
 /*
  * Align variables.