--- /usr/src/share/man/man9/sleepqueue.9 2009-12-11 12:08:28.000000000 +0100 +++ usr/src/share/man/man9/sleepqueue.9 2009-12-11 12:13:42.000000000 +0100 @@ -23,7 +23,7 @@ .\" .\" $FreeBSD: src/share/man/man9/sleepqueue.9,v 1.17.2.1.2.1 2009/10/25 01:10:29 kensmith Exp $ .\" -.Dd August 13, 2007 +.Dd December 11, 2009 .Dt SLEEPQUEUE 9 .Os .Sh NAME @@ -41,6 +41,7 @@ .Nm sleepq_remove , .Nm sleepq_signal , .Nm sleepq_set_timeout , +.Nm sleepq_sleepcnt , .Nm sleepq_timedwait , .Nm sleepq_timedwait_sig , .Nm sleepq_wait , @@ -77,6 +78,8 @@ .Fn sleepq_signal "void *wchan" "int flags" "int pri" "int queue" .Ft void .Fn sleepq_set_timeout "void *wchan" "int timo" +.Ft u_int +.Fn sleepq_sleepcnt "void *wchan" "int queue" .Ft int .Fn sleepq_timedwait "void *wchan" .Ft int @@ -348,6 +351,14 @@ One possible use is waking up a specific channel. .Pp The +.Fn sleepq_sleepcnt +function offer a simple way to retrieve the number of threads sleeping for +the specified +.Fa queue , +given a +.Fa wchan . +.Pp +The .Fn sleepq_abort , .Fn sleepq_broadcast , and --- /usr/src/sys/kern/kern_lock.c 2009-12-11 02:57:07.000000000 +0100 +++ usr/src/sys/kern/kern_lock.c 2009-12-11 12:03:56.000000000 +0100 @@ -194,6 +194,8 @@ sleeplk(struct lock *lk, u_int flags, st if (flags & LK_INTERLOCK) class->lc_unlock(ilk); + if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0) + lk->lk_exslpfail++; GIANT_SAVE(); sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ? SLEEPQ_INTERRUPTIBLE : 0), queue); @@ -222,6 +224,7 @@ static __inline int wakeupshlk(struct lock *lk, const char *file, int line) { uintptr_t v, x; + u_int realexslp; int queue, wakeup_swapper; TD_LOCKS_DEC(curthread); @@ -267,13 +270,34 @@ wakeupshlk(struct lock *lk, const char * /* * If the lock has exclusive waiters, give them preference in * order to avoid deadlock with shared runners up. + * If interruptible sleeps left the exclusive queue empty + * avoid a starvation for the threads sleeping on the shared + * queue by giving them precedence and cleaning up the + * exclusive waiters bit anyway. */ - if (x & LK_EXCLUSIVE_WAITERS) { - queue = SQ_EXCLUSIVE_QUEUE; - v |= (x & LK_SHARED_WAITERS); + realexslp = sleepq_sleepcnt(&lk->lock_object, + SQ_EXCLUSIVE_QUEUE); + if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) { + if (lk->lk_exslpfail < realexslp) { + lk->lk_exslpfail = 0; + queue = SQ_EXCLUSIVE_QUEUE; + v |= (x & LK_SHARED_WAITERS); + } else { + lk->lk_exslpfail = 0; + LOCK_LOG2(lk, + "%s: %p has only LK_SLEEPFAIL sleepers", + __func__, lk); + LOCK_LOG2(lk, + "%s: %p waking up threads on the exclusive queue", + __func__, lk); + wakeup_swapper = + sleepq_broadcast(&lk->lock_object, + SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE); + queue = SQ_SHARED_QUEUE; + } + } else { - MPASS((x & ~LK_EXCLUSIVE_SPINNERS) == - LK_SHARED_WAITERS); + MPASS(lk->lk_exslpfail == 0); queue = SQ_SHARED_QUEUE; } @@ -285,7 +309,7 @@ wakeupshlk(struct lock *lk, const char * LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue", __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : "exclusive"); - wakeup_swapper = sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, + wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue); sleepq_release(&lk->lock_object); break; @@ -350,6 +374,7 @@ lockinit(struct lock *lk, int pri, const lk->lk_lock = LK_UNLOCKED; lk->lk_recurse = 0; + lk->lk_exslpfail = 0; lk->lk_timo = timo; lk->lk_pri = pri; lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags); @@ -362,6 +387,7 @@ lockdestroy(struct lock *lk) KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held")); KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed")); + KASSERT(lk->lk_exslpfail == 0, ("lockmgr still exclusive waiters")); lock_destroy(&lk->lock_object); } @@ -373,7 +399,7 @@ __lockmgr_args(struct lock *lk, u_int fl struct lock_class *class; const char *iwmesg; uintptr_t tid, v, x; - u_int op; + u_int op, realexslp; int error, ipri, itimo, queue, wakeup_swapper; #ifdef LOCK_PROFILING uint64_t waittime = 0; @@ -903,14 +929,34 @@ __lockmgr_args(struct lock *lk, u_int fl * If the lock has exclusive waiters, give them * preference in order to avoid deadlock with * shared runners up. + * If interruptible sleeps left the exclusive queue + * empty avoid a starvation for the threads sleeping + * on the shared queue by giving them precedence + * and cleaning up the exclusive waiters bit anyway. */ MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); - if (x & LK_EXCLUSIVE_WAITERS) { - queue = SQ_EXCLUSIVE_QUEUE; - v |= (x & LK_SHARED_WAITERS); + realexslp = sleepq_sleepcnt(&lk->lock_object, + SQ_EXCLUSIVE_QUEUE); + if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) { + if (lk->lk_exslpfail < realexslp) { + lk->lk_exslpfail = 0; + queue = SQ_EXCLUSIVE_QUEUE; + v |= (x & LK_SHARED_WAITERS); + } else { + lk->lk_exslpfail = 0; + LOCK_LOG2(lk, + "%s: %p has only LK_SLEEPFAIL sleepers", + __func__, lk); + LOCK_LOG2(lk, + "%s: %p waking up threads on the exclusive queue", + __func__, lk); + wakeup_swapper = + sleepq_broadcast(&lk->lock_object, + SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE); + queue = SQ_SHARED_QUEUE; + } } else { - MPASS((x & LK_ALL_WAITERS) == - LK_SHARED_WAITERS); + MPASS(lk->lk_exslpfail == 0); queue = SQ_SHARED_QUEUE; } @@ -919,7 +965,7 @@ __lockmgr_args(struct lock *lk, u_int fl __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : "exclusive"); atomic_store_rel_ptr(&lk->lk_lock, v); - wakeup_swapper = sleepq_broadcast(&lk->lock_object, + wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue); sleepq_release(&lk->lock_object); break; @@ -976,14 +1022,47 @@ __lockmgr_args(struct lock *lk, u_int fl v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); if ((x & ~v) == LK_UNLOCKED) { v = (x & ~LK_EXCLUSIVE_SPINNERS); + + /* + * If interruptible sleeps left the exclusive + * queue empty avoid a starvation for the + * threads sleeping on the shared queue by + * giving them precedence and cleaning up the + * exclusive waiters bit anyway. + */ if (v & LK_EXCLUSIVE_WAITERS) { queue = SQ_EXCLUSIVE_QUEUE; v &= ~LK_EXCLUSIVE_WAITERS; } else { MPASS(v & LK_SHARED_WAITERS); + MPASS(lk->lk_exslpfail == 0); queue = SQ_SHARED_QUEUE; v &= ~LK_SHARED_WAITERS; } + if (queue == SQ_EXCLUSIVE_QUEUE) { + realexslp = + sleepq_sleepcnt(&lk->lock_object, + SQ_EXCLUSIVE_QUEUE); + if (lk->lk_exslpfail >= realexslp) { + lk->lk_exslpfail = 0; + queue = SQ_SHARED_QUEUE; + v &= ~LK_SHARED_WAITERS; + if (realexslp != 0) { + LOCK_LOG2(lk, + "%s: %p has only LK_SLEEPFAIL sleepers", + __func__, lk); + LOCK_LOG2(lk, + "%s: %p waking up threads on the exclusive queue", + __func__, lk); + wakeup_swapper = + sleepq_broadcast( + &lk->lock_object, + SLEEPQ_LK, 0, + SQ_EXCLUSIVE_QUEUE); + } + } else + lk->lk_exslpfail = 0; + } if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) { sleepq_release(&lk->lock_object); continue; --- /usr/src/sys/kern/kern_sx.c 2009-12-11 02:57:07.000000000 +0100 +++ usr/src/sys/kern/kern_sx.c 2009-12-11 03:44:00.000000000 +0100 @@ -702,8 +702,12 @@ _sx_xunlock_hard(struct sx *sx, uintptr_ * ideal. It gives precedence to shared waiters if they are * present. For this condition, we have to preserve the * state of the exclusive waiters flag. + * If interruptible sleeps left the shared queue empty avoid a + * starvation for the threads sleeping on the exclusive queue by giving + * them precedence and cleaning up the shared waiters bit anyway. */ - if (sx->sx_lock & SX_LOCK_SHARED_WAITERS) { + if ((sx->sx_lock & SX_LOCK_SHARED_WAITERS) != 0 && + sleepq_sleepcnt(&sx->lock_object, SQ_SHARED_QUEUE) != 0) { queue = SQ_SHARED_QUEUE; x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS); } else --- /usr/src/sys/kern/subr_sleepqueue.c 2009-12-11 02:27:55.000000000 +0100 +++ usr/src/sys/kern/subr_sleepqueue.c 2009-12-11 11:31:37.000000000 +0100 @@ -118,6 +118,7 @@ __FBSDID("$FreeBSD: src/sys/kern/subr_sl */ struct sleepqueue { TAILQ_HEAD(, thread) sq_blocked[NR_SLEEPQS]; /* (c) Blocked threads. */ + u_int sq_blockedcnt[NR_SLEEPQS]; /* (c) N. of blocked threads. */ LIST_ENTRY(sleepqueue) sq_hash; /* (c) Chain and free list. */ LIST_HEAD(, sleepqueue) sq_free; /* (c) Free queues. */ void *sq_wchan; /* (c) Wait channel. */ @@ -306,9 +307,12 @@ sleepq_add(void *wchan, struct lock_obje int i; sq = td->td_sleepqueue; - for (i = 0; i < NR_SLEEPQS; i++) + for (i = 0; i < NR_SLEEPQS; i++) { KASSERT(TAILQ_EMPTY(&sq->sq_blocked[i]), - ("thread's sleep queue %d is not empty", i)); + ("thread's sleep queue %d is not empty", i)); + KASSERT(sq->sq_blockedcnt[i] == 0, + ("thread's sleep queue %d count mismatches", i)); + } KASSERT(LIST_EMPTY(&sq->sq_free), ("thread's sleep queue has a non-empty free list")); KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer")); @@ -334,6 +338,7 @@ sleepq_add(void *wchan, struct lock_obje } thread_lock(td); TAILQ_INSERT_TAIL(&sq->sq_blocked[queue], td, td_slpq); + sq->sq_blockedcnt[queue]++; td->td_sleepqueue = NULL; td->td_sqqueue = queue; td->td_wchan = wchan; @@ -367,6 +372,22 @@ sleepq_set_timeout(void *wchan, int timo } /* + * Return the number of actual sleepers for the specified queue. + */ +u_int +sleepq_sleepcnt(void *wchan, int queue) +{ + struct sleepqueue *sq; + + KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__)); + MPASS((queue >= 0) && (queue < NR_SLEEPQS)); + sq = sleepq_lookup(wchan); + if (sq == NULL) + return (0); + return (sq->sq_blockedcnt[queue]); +} + +/* * Marks the pending sleep of the current thread as interruptible and * makes an initial check for pending signals before putting a thread * to sleep. Enters and exits with the thread lock held. Thread lock @@ -665,6 +686,7 @@ sleepq_resume_thread(struct sleepqueue * mtx_assert(&sc->sc_lock, MA_OWNED); /* Remove the thread from the queue. */ + sq->sq_blockedcnt[td->td_sqqueue]--; TAILQ_REMOVE(&sq->sq_blocked[td->td_sqqueue], td, td_slpq); /* @@ -720,8 +742,10 @@ sleepq_dtor(void *mem, int size, void *a int i; sq = mem; - for (i = 0; i < NR_SLEEPQS; i++) + for (i = 0; i < NR_SLEEPQS; i++) { MPASS(TAILQ_EMPTY(&sq->sq_blocked[i])); + MPASS(sq->sq_blockedcnt[i] == 0); + } } #endif @@ -736,8 +760,10 @@ sleepq_init(void *mem, int size, int fla bzero(mem, size); sq = mem; - for (i = 0; i < NR_SLEEPQS; i++) + for (i = 0; i < NR_SLEEPQS; i++) { TAILQ_INIT(&sq->sq_blocked[i]); + sq->sq_blockedcnt[i] = 0; + } LIST_INIT(&sq->sq_free); return (0); } @@ -1170,6 +1196,7 @@ found: td->td_tid, td->td_proc->p_pid, td->td_name); } + db_printf("(expected: %u)\n", sq->sq_blockedcnt[i]); } } --- /usr/src/sys/sys/_lockmgr.h 2009-12-11 03:58:10.000000000 +0100 +++ usr/src/sys/sys/_lockmgr.h 2009-12-11 03:58:48.000000000 +0100 @@ -38,6 +38,7 @@ struct lock { struct lock_object lock_object; volatile uintptr_t lk_lock; + u_int lk_exslpfail; int lk_timo; int lk_pri; #ifdef DEBUG_LOCKS --- /usr/src/sys/sys/sleepqueue.h 2009-12-11 02:28:05.000000000 +0100 +++ usr/src/sys/sys/sleepqueue.h 2009-12-11 02:47:22.000000000 +0100 @@ -109,6 +109,7 @@ void sleepq_release(void *wchan); void sleepq_remove(struct thread *td, void *wchan); int sleepq_signal(void *wchan, int flags, int pri, int queue); void sleepq_set_timeout(void *wchan, int timo); +u_int sleepq_sleepcnt(void *wchan, int queue); int sleepq_timedwait(void *wchan, int pri); int sleepq_timedwait_sig(void *wchan, int pri); void sleepq_wait(void *wchan, int pri);