--- //depot/projects/smpng/sys/kern/kern_condvar.c 2008/03/13 21:35:00 +++ //depot/user/jhb/lock/kern/kern_condvar.c 2008/05/21 00:13:31 @@ -122,7 +122,7 @@ sleepq_lock(cvp); - cvp->cv_waiters++; + cvp->cv_waiters = 1; DROP_GIANT(); sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); @@ -175,7 +175,7 @@ sleepq_lock(cvp); - cvp->cv_waiters++; + cvp->cv_waiters = 1; DROP_GIANT(); sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); @@ -232,7 +232,7 @@ sleepq_lock(cvp); - cvp->cv_waiters++; + cvp->cv_waiters = 1; DROP_GIANT(); sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | @@ -292,7 +292,7 @@ sleepq_lock(cvp); - cvp->cv_waiters++; + cvp->cv_waiters = 1; DROP_GIANT(); sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); @@ -355,7 +355,7 @@ sleepq_lock(cvp); - cvp->cv_waiters++; + cvp->cv_waiters = 1; DROP_GIANT(); sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | @@ -389,13 +389,15 @@ void cv_signal(struct cv *cvp) { + int wakeup_swapper; + wakeup_swapper = 0; sleepq_lock(cvp); - if (cvp->cv_waiters > 0) { - cvp->cv_waiters--; - sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 0); - } + if (cvp->cv_waiters > 0) + wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 0); sleepq_release(cvp); + if (wakeup_swapper) + kick_proc0(); } /* @@ -405,16 +407,21 @@ void cv_broadcastpri(struct cv *cvp, int pri) { + int wakeup_swapper; + /* * XXX sleepq_broadcast pri argument changed from -1 meaning * no pri to 0 meaning no pri. */ + wakeup_swapper = 0; if (pri == -1) pri = 0; sleepq_lock(cvp); if (cvp->cv_waiters > 0) { cvp->cv_waiters = 0; - sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); + wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); } sleepq_release(cvp); + if (wakeup_swapper) + kick_proc0(); } --- //depot/projects/smpng/sys/kern/kern_sig.c 2008/03/24 19:59:34 +++ //depot/user/jhb/lock/kern/kern_sig.c 2008/05/21 00:13:31 @@ -1942,6 +1942,7 @@ struct sigacts *ps; int intrval; int ret = 0; + int wakeup_swapper; PROC_LOCK_ASSERT(p, MA_OWNED); @@ -2146,12 +2147,15 @@ * the PROCESS runnable, leave it stopped. * It may run a bit until it hits a thread_suspend_check(). */ + wakeup_swapper = 0; PROC_SLOCK(p); thread_lock(td); if (TD_ON_SLEEPQ(td) && (td->td_flags & TDF_SINTR)) - sleepq_abort(td, intrval); + wakeup_swapper = sleepq_abort(td, intrval); thread_unlock(td); PROC_SUNLOCK(p); + if (wakeup_swapper) + kick_proc0(); goto out; /* * Mutexes are short lived. Threads waiting on them will @@ -2218,7 +2222,9 @@ { struct proc *p = td->td_proc; register int prop; + int wakeup_swapper; + wakeup_swapper = 0; PROC_LOCK_ASSERT(p, MA_OWNED); prop = sigprop(sig); @@ -2262,7 +2268,7 @@ if (td->td_priority > PUSER) sched_prio(td, PUSER); - sleepq_abort(td, intrval); + wakeup_swapper = sleepq_abort(td, intrval); } else { /* * Other states do nothing with the signal immediately, @@ -2277,6 +2283,8 @@ out: PROC_SUNLOCK(p); thread_unlock(td); + if (wakeup_swapper) + kick_proc0(); } static void --- //depot/projects/smpng/sys/kern/kern_sx.c 2008/03/13 21:35:00 +++ //depot/user/jhb/lock/kern/kern_sx.c 2008/05/21 00:13:31 @@ -360,6 +360,7 @@ _sx_downgrade(struct sx *sx, const char *file, int line) { uintptr_t x; + int wakeup_swapper; KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); @@ -400,15 +401,19 @@ * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single * shared lock. If there are any shared waiters, wake them up. */ + wakeup_swapper = 0; x = sx->sx_lock; atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | (x & SX_LOCK_EXCLUSIVE_WAITERS)); if (x & SX_LOCK_SHARED_WAITERS) - sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, - SQ_SHARED_QUEUE); + wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, + 0, SQ_SHARED_QUEUE); sleepq_release(&sx->lock_object); LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); + + if (wakeup_swapper) + kick_proc0(); } /* @@ -588,7 +593,7 @@ _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) { uintptr_t x; - int queue; + int queue, wakeup_swapper; MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); @@ -626,8 +631,11 @@ __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" : "exclusive"); atomic_store_rel_ptr(&sx->sx_lock, x); - sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, queue); + wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, + queue); sleepq_release(&sx->lock_object); + if (wakeup_swapper) + kick_proc0(); } /* @@ -794,6 +802,7 @@ _sx_sunlock_hard(struct sx *sx, const char *file, int line) { uintptr_t x; + int wakeup_swapper; for (;;) { x = sx->sx_lock; @@ -861,9 +870,11 @@ if (LOCK_LOG_TEST(&sx->lock_object, 0)) CTR2(KTR_LOCK, "%s: %p waking up all thread on" "exclusive queue", __func__, sx); - sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, - SQ_EXCLUSIVE_QUEUE); + wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, + 0, SQ_EXCLUSIVE_QUEUE); sleepq_release(&sx->lock_object); + if (wakeup_swapper) + kick_proc0(); break; } } --- //depot/projects/smpng/sys/kern/kern_synch.c 2008/03/18 12:54:14 +++ //depot/user/jhb/lock/kern/kern_synch.c 2008/05/21 00:13:31 @@ -326,10 +326,13 @@ void wakeup(void *ident) { + int wakeup_swapper; sleepq_lock(ident); - sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0); + wakeup_swapper = sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0); sleepq_release(ident); + if (wakeup_swapper) + kick_proc0(); } /* @@ -340,10 +343,13 @@ void wakeup_one(void *ident) { + int wakeup_swapper; sleepq_lock(ident); - sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0); + wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0); sleepq_release(ident); + if (wakeup_swapper) + kick_proc0(); } static void @@ -436,11 +442,11 @@ } /* - * Change process state to be runnable, - * placing it on the run queue if it is in memory, - * and awakening the swapper if it isn't in memory. + * Change thread state to be runnable, placing it on the run queue if + * it is in memory. If it is swapped out, return true so our caller + * will know to awaken the swapper. */ -void +int setrunnable(struct thread *td) { @@ -450,15 +456,15 @@ switch (td->td_state) { case TDS_RUNNING: case TDS_RUNQ: - return; + return (0); case TDS_INHIBITED: /* * If we are only inhibited because we are swapped out * then arange to swap in this process. Otherwise just return. */ if (td->td_inhibitors != TDI_SWAPPED) - return; - /* XXX: intentional fall-through ? */ + return (0); + /* FALLTHROUGH */ case TDS_CAN_RUN: break; default: @@ -468,15 +474,11 @@ if ((td->td_flags & TDF_INMEM) == 0) { if ((td->td_flags & TDF_SWAPINREQ) == 0) { td->td_flags |= TDF_SWAPINREQ; - /* - * due to a LOR between the thread lock and - * the sleepqueue chain locks, use - * lower level scheduling functions. - */ - kick_proc0(); + return (1); } } else sched_wakeup(td); + return (0); } /* --- //depot/projects/smpng/sys/kern/kern_thread.c 2008/03/24 19:59:34 +++ //depot/user/jhb/lock/kern/kern_thread.c 2008/05/21 00:13:31 @@ -508,7 +508,7 @@ struct thread *td; struct thread *td2; struct proc *p; - int remaining; + int remaining, wakeup_swapper; td = curthread; p = td->td_proc; @@ -545,6 +545,7 @@ while (remaining != 1) { if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) goto stopme; + wakeup_swapper = 0; FOREACH_THREAD_IN_PROC(p, td2) { if (td2 == td) continue; @@ -559,7 +560,8 @@ thread_unsuspend_one(td2); if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) - sleepq_abort(td2, EINTR); + wakeup_swapper = + sleepq_abort(td2, EINTR); break; case SINGLE_BOUNDARY: break; @@ -585,6 +587,8 @@ #endif thread_unlock(td2); } + if (wakeup_swapper) + kick_proc0(); if (mode == SINGLE_EXIT) remaining = p->p_numthreads; else if (mode == SINGLE_BOUNDARY) @@ -787,7 +791,11 @@ KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); TD_CLR_SUSPENDED(td); p->p_suspcount--; - setrunnable(td); + if (setrunnable(td)) { +#ifdef INVARIANTS + panic("not waking up swapper"); +#endif + } } /* --- //depot/projects/smpng/sys/kern/subr_sleepqueue.c 2008/04/04 19:26:14 +++ //depot/user/jhb/lock/kern/subr_sleepqueue.c 2008/05/21 00:13:31 @@ -160,7 +160,7 @@ static void sleepq_dtor(void *mem, int size, void *arg); #endif static int sleepq_init(void *mem, int size, int flags); -static void sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, +static int sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri); static void sleepq_switch(void *wchan, int pri); static void sleepq_timeout(void *arg); @@ -424,20 +424,29 @@ PROC_UNLOCK(p); thread_lock(td); PROC_SUNLOCK(p); - if (ret == 0) { + + if (ret != 0) { + /* + * There were pending signals and this thread is still + * on the sleep queue, remove it from the sleep queue. + */ + if (TD_ON_SLEEPQ(td)) { + sq = sleepq_lookup(wchan); + if (sleepq_resume_thread(sq, td, 0)) { +#ifdef INVARIANTS + /* + * This thread hasn't gone to sleep + * yet, so it should not be swapped + * out. + */ + panic("not waking up swapper"); +#endif + } + } + mtx_unlock_spin(&sc->sc_lock); + MPASS(td->td_lock != &sc->sc_lock); + } else sleepq_switch(wchan, pri); - return (0); - } - /* - * There were pending signals and this thread is still - * on the sleep queue, remove it from the sleep queue. - */ - if (TD_ON_SLEEPQ(td)) { - sq = sleepq_lookup(wchan); - sleepq_resume_thread(sq, td, 0); - } - mtx_unlock_spin(&sc->sc_lock); - MPASS(td->td_lock != &sc->sc_lock); return (ret); } @@ -474,7 +483,15 @@ if (td->td_flags & TDF_TIMEOUT) { MPASS(TD_ON_SLEEPQ(td)); sq = sleepq_lookup(wchan); - sleepq_resume_thread(sq, td, 0); + if (sleepq_resume_thread(sq, td, 0)) { +#ifdef INVARIANTS + /* + * This thread hasn't gone to sleep yet, so it + * should not be swapped out. + */ + panic("not waking up swapper"); +#endif + } mtx_unlock_spin(&sc->sc_lock); return; } @@ -633,7 +650,7 @@ * Removes a thread from a sleep queue and makes it * runnable. */ -static void +static int sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri) { struct sleepqueue_chain *sc; @@ -670,22 +687,25 @@ td->td_wchan = NULL; td->td_flags &= ~TDF_SINTR; - /* - * Note that thread td might not be sleeping if it is running - * sleepq_catch_signals() on another CPU or is blocked on - * its proc lock to check signals. It doesn't hurt to clear - * the sleeping flag if it isn't set though, so we just always - * do it. However, we can't assert that it is set. - */ CTR3(KTR_PROC, "sleepq_wakeup: thread %p (pid %ld, %s)", (void *)td, (long)td->td_proc->p_pid, td->td_name); - TD_CLR_SLEEPING(td); /* Adjust priority if requested. */ MPASS(pri == 0 || (pri >= PRI_MIN && pri <= PRI_MAX)); if (pri != 0 && td->td_priority > pri) sched_prio(td, pri); - setrunnable(td); + + /* + * Note that thread td might not be sleeping if it is running + * sleepq_catch_signals() on another CPU or is blocked on its + * proc lock to check signals. There's no need to mark the + * thread runnable in that case. + */ + if (TD_IS_SLEEPING(td)) { + TD_CLR_SLEEPING(td); + return (setrunnable(td)); + } + return (0); } #ifdef INVARIANTS @@ -724,18 +744,19 @@ /* * Find the highest priority thread sleeping on a wait channel and resume it. */ -void +int sleepq_signal(void *wchan, int flags, int pri, int queue) { struct sleepqueue *sq; struct thread *td, *besttd; + int wakeup_swapper; CTR2(KTR_PROC, "sleepq_signal(%p, %d)", wchan, flags); KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__)); MPASS((queue >= 0) && (queue < NR_SLEEPQS)); sq = sleepq_lookup(wchan); if (sq == NULL) - return; + return (0); KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE), ("%s: mismatch between sleep/wakeup and cv_*", __func__)); @@ -752,35 +773,40 @@ } MPASS(besttd != NULL); thread_lock(besttd); - sleepq_resume_thread(sq, besttd, pri); + wakeup_swapper = sleepq_resume_thread(sq, besttd, pri); thread_unlock(besttd); + return (wakeup_swapper); } /* * Resume all threads sleeping on a specified wait channel. */ -void +int sleepq_broadcast(void *wchan, int flags, int pri, int queue) { struct sleepqueue *sq; struct thread *td; + int wakeup_swapper; CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags); KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__)); MPASS((queue >= 0) && (queue < NR_SLEEPQS)); sq = sleepq_lookup(wchan); if (sq == NULL) - return; + return (0); KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE), ("%s: mismatch between sleep/wakeup and cv_*", __func__)); /* Resume all blocked threads on the sleep queue. */ + wakeup_swapper = 0; while (!TAILQ_EMPTY(&sq->sq_blocked[queue])) { td = TAILQ_FIRST(&sq->sq_blocked[queue]); thread_lock(td); - sleepq_resume_thread(sq, td, pri); + if (sleepq_resume_thread(sq, td, pri)) + wakeup_swapper = 1; thread_unlock(td); } + return (wakeup_swapper); } /* @@ -794,8 +820,10 @@ struct sleepqueue *sq; struct thread *td; void *wchan; + int wakeup_swapper; td = arg; + wakeup_swapper = 0; CTR3(KTR_PROC, "sleepq_timeout: thread %p (pid %ld, %s)", (void *)td, (long)td->td_proc->p_pid, (void *)td->td_name); @@ -811,8 +839,10 @@ sq = sleepq_lookup(wchan); MPASS(sq != NULL); td->td_flags |= TDF_TIMEOUT; - sleepq_resume_thread(sq, td, 0); + wakeup_swapper = sleepq_resume_thread(sq, td, 0); thread_unlock(td); + if (wakeup_swapper) + kick_proc0(); return; } @@ -841,10 +871,12 @@ MPASS(TD_IS_SLEEPING(td)); td->td_flags &= ~TDF_TIMEOUT; TD_CLR_SLEEPING(td); - setrunnable(td); + wakeup_swapper = setrunnable(td); } else td->td_flags |= TDF_TIMOFAIL; thread_unlock(td); + if (wakeup_swapper) + kick_proc0(); } /* @@ -855,6 +887,7 @@ sleepq_remove(struct thread *td, void *wchan) { struct sleepqueue *sq; + int wakeup_swapper; /* * Look up the sleep queue for this wait channel, then re-check @@ -878,16 +911,18 @@ thread_lock(td); MPASS(sq != NULL); MPASS(td->td_wchan == wchan); - sleepq_resume_thread(sq, td, 0); + wakeup_swapper = sleepq_resume_thread(sq, td, 0); thread_unlock(td); sleepq_release(wchan); + if (wakeup_swapper) + kick_proc0(); } /* * Abort a thread as if an interrupt had occurred. Only abort * interruptible waits (unfortunately it isn't safe to abort others). */ -void +int sleepq_abort(struct thread *td, int intrval) { struct sleepqueue *sq; @@ -903,7 +938,7 @@ * timeout is scheduled anyhow. */ if (td->td_flags & TDF_TIMEOUT) - return; + return (0); CTR3(KTR_PROC, "sleepq_abort: thread %p (pid %ld, %s)", (void *)td, (long)td->td_proc->p_pid, (void *)td->td_name); @@ -915,14 +950,14 @@ * we have to do it here. */ if (!TD_IS_SLEEPING(td)) - return; + return (0); wchan = td->td_wchan; MPASS(wchan != NULL); sq = sleepq_lookup(wchan); MPASS(sq != NULL); /* Thread is asleep on sleep queue sq, so wake it up. */ - sleepq_resume_thread(sq, td, 0); + return (sleepq_resume_thread(sq, td, 0)); } #ifdef SLEEPQUEUE_PROFILING --- //depot/projects/smpng/sys/sys/proc.h 2008/03/24 19:59:34 +++ //depot/user/jhb/lock/sys/proc.h 2008/05/21 00:13:31 @@ -785,7 +785,7 @@ int securelevel_ge(struct ucred *cr, int level); int securelevel_gt(struct ucred *cr, int level); void sessrele(struct session *); -void setrunnable(struct thread *); +int setrunnable(struct thread *); void setsugid(struct proc *p); int sigonstack(size_t sp); void sleepinit(void); --- //depot/projects/smpng/sys/sys/sleepqueue.h 2008/03/13 21:35:00 +++ //depot/user/jhb/lock/sys/sleepqueue.h 2008/05/21 00:13:31 @@ -90,17 +90,17 @@ #define SLEEPQ_INTERRUPTIBLE 0x100 /* Sleep is interruptible. */ void init_sleepqueues(void); -void sleepq_abort(struct thread *td, int intrval); +int sleepq_abort(struct thread *td, int intrval); void sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags, int queue); struct sleepqueue *sleepq_alloc(void); -void sleepq_broadcast(void *wchan, int flags, int pri, int queue); +int sleepq_broadcast(void *wchan, int flags, int pri, int queue); void sleepq_free(struct sleepqueue *sq); void sleepq_lock(void *wchan); struct sleepqueue *sleepq_lookup(void *wchan); void sleepq_release(void *wchan); void sleepq_remove(struct thread *td, void *wchan); -void sleepq_signal(void *wchan, int flags, int pri, int queue); +int sleepq_signal(void *wchan, int flags, int pri, int queue); void sleepq_set_timeout(void *wchan, int timo); int sleepq_timedwait(void *wchan, int pri); int sleepq_timedwait_sig(void *wchan, int pri); --- //depot/projects/smpng/sys/vm/vm_glue.c 2008/03/19 21:22:58 +++ //depot/user/jhb/lock/vm/vm_glue.c 2008/05/21 00:13:31 @@ -116,10 +116,6 @@ static void swapclear(struct proc *); #endif - -static volatile int proc0_rescan; - - /* * MPSAFE * @@ -683,9 +679,6 @@ loop: if (vm_page_count_min()) { VM_WAIT; - thread_lock(&thread0); - proc0_rescan = 0; - thread_unlock(&thread0); goto loop; } @@ -732,13 +725,7 @@ * Nothing to do, back to sleep. */ if ((p = pp) == NULL) { - thread_lock(&thread0); - if (!proc0_rescan) { - TD_SET_IWAIT(&thread0); - mi_switch(SW_VOL, NULL); - } - proc0_rescan = 0; - thread_unlock(&thread0); + tsleep(&proc0, PVM, "sched", maxslp * hz / 2); goto loop; } PROC_LOCK(p); @@ -750,9 +737,6 @@ */ if (p->p_flag & (P_INMEM | P_SWAPPINGOUT | P_SWAPPINGIN)) { PROC_UNLOCK(p); - thread_lock(&thread0); - proc0_rescan = 0; - thread_unlock(&thread0); goto loop; } @@ -762,32 +746,16 @@ */ faultin(p); PROC_UNLOCK(p); - thread_lock(&thread0); - proc0_rescan = 0; - thread_unlock(&thread0); goto loop; } -void kick_proc0(void) +void +kick_proc0(void) { - struct thread *td = &thread0; - /* XXX This will probably cause a LOR in some cases */ - thread_lock(td); - if (TD_AWAITING_INTR(td)) { - CTR2(KTR_INTR, "%s: sched_add %d", __func__, 0); - TD_CLR_IWAIT(td); - sched_add(td, SRQ_INTR); - } else { - proc0_rescan = 1; - CTR2(KTR_INTR, "%s: state %d", - __func__, td->td_state); - } - thread_unlock(td); - + wakeup(&proc0); } - #ifndef NO_SWAPPING /* @@ -980,7 +948,16 @@ td->td_flags &= ~TDF_SWAPINREQ; TD_CLR_SWAPPED(td); if (TD_CAN_RUN(td)) - setrunnable(td); + if (setrunnable(td)) { +#ifdef INVARIANTS + /* + * XXX: We just cleared TDI_SWAPPED + * above and set TDF_INMEM, so this + * should never happen. + */ + panic("not waking up swapper"); +#endif + } thread_unlock(td); } p->p_flag &= ~(P_SWAPPINGIN|P_SWAPPINGOUT);