Index: sys/kern/kern_mutex.c =================================================================== --- sys/kern/kern_mutex.c (revision 225025) +++ sys/kern/kern_mutex.c (working copy) @@ -197,6 +197,8 @@ _mtx_lock_flags(struct mtx *m, int opts, const cha KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, file, line)); + if (SCHEDULER_STOPPED()) + return; WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); @@ -216,6 +218,8 @@ _mtx_unlock_flags(struct mtx *m, int opts, const c KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, file, line)); + if (SCHEDULER_STOPPED()) + return; curthread->td_locks--; WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, @@ -237,6 +241,8 @@ _mtx_lock_spin_flags(struct mtx *m, int opts, cons KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, ("mtx_lock_spin() of sleep mutex %s @ %s:%d", m->lock_object.lo_name, file, line)); + if (SCHEDULER_STOPPED()) + return; if (mtx_owned(m)) KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n", @@ -259,6 +265,8 @@ _mtx_unlock_spin_flags(struct mtx *m, int opts, co KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", m->lock_object.lo_name, file, line)); + if (SCHEDULER_STOPPED()) + return; WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, line); @@ -287,6 +295,8 @@ _mtx_trylock(struct mtx *m, int opts, const char * KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, file, line)); + if (SCHEDULER_STOPPED()) + return (1); if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) { m->mtx_recurse++; @@ -337,6 +347,8 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int int64_t sleep_time = 0; #endif + if (SCHEDULER_STOPPED()) + return; if (mtx_owned(m)) { KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", @@ -507,6 +519,8 @@ _mtx_lock_spin(struct mtx *m, uintptr_t tid, int o uint64_t waittime = 0; #endif + if (SCHEDULER_STOPPED()) + return; if (LOCK_LOG_TEST(&m->lock_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); @@ -552,6 +566,9 @@ _thread_lock_flags(struct thread *td, int opts, co uint64_t spin_cnt = 0; #endif + if (SCHEDULER_STOPPED()) + return; + i = 0; tid = (uintptr_t)curthread; for (;;) { @@ -577,6 +594,7 @@ retry: m->mtx_recurse++; break; } + lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); /* Give interrupts a chance while we spin. */ @@ -655,6 +673,8 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const c { struct turnstile *ts; + if (SCHEDULER_STOPPED()) + return; if (mtx_recursed(m)) { if (--(m->mtx_recurse) == 0) atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); Index: sys/kern/kern_shutdown.c =================================================================== --- sys/kern/kern_shutdown.c (revision 225025) +++ sys/kern/kern_shutdown.c (working copy) @@ -121,6 +121,11 @@ SYSCTL_INT(_kern, OID_AUTO, sync_on_panic, CTLFLAG &sync_on_panic, 0, "Do a sync before rebooting from a panic"); TUNABLE_INT("kern.sync_on_panic", &sync_on_panic); +int stop_scheduler_on_panic = 1; +SYSCTL_INT(_kern, OID_AUTO, stop_scheduler_on_panic, CTLFLAG_RW | CTLFLAG_TUN, + &stop_scheduler_on_panic, 0, "stop scheduler upon entering panic"); +TUNABLE_INT("kern.stop_scheduler_on_panic", &stop_scheduler_on_panic); + SYSCTL_NODE(_kern, OID_AUTO, shutdown, CTLFLAG_RW, 0, "Shutdown environment"); /* @@ -284,10 +289,12 @@ boot(int howto) * systems don't shutdown properly (i.e., ACPI power off) if we * run on another processor. */ - thread_lock(curthread); - sched_bind(curthread, 0); - thread_unlock(curthread); - KASSERT(PCPU_GET(cpuid) == 0, ("boot: not running on cpu 0")); + if (!SCHEDULER_STOPPED()) { + thread_lock(curthread); + sched_bind(curthread, 0); + thread_unlock(curthread); + KASSERT(PCPU_GET(cpuid) == 0, ("boot: not running on cpu 0")); + } #endif /* We're in the process of rebooting. */ rebooting = 1; @@ -539,7 +546,11 @@ panic(const char *fmt, ...) va_list ap; static char buf[256]; - critical_enter(); + if (stop_scheduler_on_panic) + spinlock_enter(); + else + critical_enter(); + #ifdef SMP /* * We don't want multiple CPU's to panic at the same time, so we @@ -552,11 +563,15 @@ panic(const char *fmt, ...) PCPU_GET(cpuid)) == 0) while (panic_cpu != NOCPU) ; /* nothing */ + if (stop_scheduler_on_panic) { + if (panicstr == NULL && !kdb_active) + stop_cpus_hard(PCPU_GET(other_cpus)); + } #endif bootopt = RB_AUTOBOOT | RB_DUMP; newpanic = 0; - if (panicstr) + if (panicstr != NULL) bootopt |= RB_NOSYNC; else { panicstr = fmt; @@ -593,12 +618,17 @@ panic(const char *fmt, ...) } #endif #endif + /*thread_lock(td); */ td->td_flags |= TDF_INPANIC; /* thread_unlock(td); */ + if (!sync_on_panic) bootopt |= RB_NOSYNC; - critical_exit(); + + if (!stop_scheduler_on_panic) + critical_exit(); + boot(bootopt); } Index: sys/kern/subr_kdb.c =================================================================== --- sys/kern/subr_kdb.c (revision 225025) +++ sys/kern/subr_kdb.c (working copy) @@ -226,10 +226,7 @@ kdb_sysctl_trap_code(SYSCTL_HANDLER_ARGS) void kdb_panic(const char *msg) { - -#ifdef SMP - stop_cpus_hard(PCPU_GET(other_cpus)); -#endif + printf("KDB: panic\n"); panic("%s", msg); } @@ -530,8 +527,11 @@ kdb_trap(int type, int code, struct trapframe *tf) intr = intr_disable(); #ifdef SMP - if ((did_stop_cpus = kdb_stop_cpus) != 0) + if (kdb_stop_cpus && !kdb_active && panicstr == NULL) { stop_cpus_hard(PCPU_GET(other_cpus)); + did_stop_cpus = 1; + } else + did_stop_cpus = 0; #endif kdb_active++; Index: sys/kern/kern_synch.c =================================================================== --- sys/kern/kern_synch.c (revision 225025) +++ sys/kern/kern_synch.c (working copy) @@ -158,7 +158,7 @@ _sleep(void *ident, struct lock_object *lock, int else class = NULL; - if (cold) { + if (cold || SCHEDULER_STOPPED()) { /* * During autoconfiguration, just return; * don't run any other threads or panic below, @@ -260,7 +260,7 @@ msleep_spin(void *ident, struct mtx *mtx, const ch KASSERT(p != NULL, ("msleep1")); KASSERT(ident != NULL && TD_IS_RUNNING(td), ("msleep")); - if (cold) { + if (cold || SCHEDULER_STOPPED()) { /* * During autoconfiguration, just return; * don't run any other threads or panic below, @@ -411,6 +411,8 @@ mi_switch(int flags, struct thread *newtd) */ if (kdb_active) kdb_switch(); + if (SCHEDULER_STOPPED()) + return; if (flags & SW_VOL) td->td_ru.ru_nvcsw++; else Index: sys/kern/kern_rwlock.c =================================================================== --- sys/kern/kern_rwlock.c (revision 225025) +++ sys/kern/kern_rwlock.c (working copy) @@ -320,6 +320,9 @@ _rw_rlock(struct rwlock *rw, const char *file, int rw->lock_object.lo_name, file, line)); WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); + if (SCHEDULER_STOPPED()) + return; + for (;;) { #ifdef KDTRACE_HOOKS spin_cnt++; @@ -529,6 +532,9 @@ _rw_runlock(struct rwlock *rw, const char *file, i WITNESS_UNLOCK(&rw->lock_object, 0, file, line); LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line); + if (SCHEDULER_STOPPED()) + return; + /* TODO: drop "owner of record" here. */ for (;;) { @@ -656,6 +662,9 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, c return; } + if (SCHEDULER_STOPPED()) + return; + if (LOCK_LOG_TEST(&rw->lock_object, 0)) CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); @@ -817,6 +826,9 @@ _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, return; } + if (SCHEDULER_STOPPED()) + return; + KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), ("%s: neither of the waiter flags are set", __func__)); Index: sys/kern/kern_sx.c =================================================================== --- sys/kern/kern_sx.c (revision 225025) +++ sys/kern/kern_sx.c (working copy) @@ -241,6 +241,8 @@ _sx_slock(struct sx *sx, int opts, const char *fil MPASS(curthread != NULL); KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_slock() of destroyed sx @ %s:%d", file, line)); + if (SCHEDULER_STOPPED()) + return (0); WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL); error = __sx_slock(sx, opts, file, line); if (!error) { @@ -257,6 +259,8 @@ _sx_try_slock(struct sx *sx, const char *file, int { uintptr_t x; + if (SCHEDULER_STOPPED()) + return (1); for (;;) { x = sx->sx_lock; KASSERT(x != SX_LOCK_DESTROYED, @@ -283,6 +287,8 @@ _sx_xlock(struct sx *sx, int opts, const char *fil MPASS(curthread != NULL); KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_xlock() of destroyed sx @ %s:%d", file, line)); + if (SCHEDULER_STOPPED()) + return (0); WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); error = __sx_xlock(sx, curthread, opts, file, line); @@ -305,6 +311,8 @@ _sx_try_xlock(struct sx *sx, const char *file, int KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_try_xlock() of destroyed sx @ %s:%d", file, line)); + if (SCHEDULER_STOPPED()) + return (1); if (sx_xlocked(sx) && (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) { sx->sx_recurse++; @@ -330,6 +338,8 @@ _sx_sunlock(struct sx *sx, const char *file, int l MPASS(curthread != NULL); KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); + if (SCHEDULER_STOPPED()) + return; _sx_assert(sx, SA_SLOCKED, file, line); curthread->td_locks--; WITNESS_UNLOCK(&sx->lock_object, 0, file, line); @@ -345,6 +355,8 @@ _sx_xunlock(struct sx *sx, const char *file, int l MPASS(curthread != NULL); KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_xunlock() of destroyed sx @ %s:%d", file, line)); + if (SCHEDULER_STOPPED()) + return; _sx_assert(sx, SA_XLOCKED, file, line); curthread->td_locks--; WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); @@ -368,6 +380,8 @@ _sx_try_upgrade(struct sx *sx, const char *file, i KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line)); + if (SCHEDULER_STOPPED()) + return (1); _sx_assert(sx, SA_SLOCKED, file, line); /* @@ -398,6 +412,8 @@ _sx_downgrade(struct sx *sx, const char *file, int KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); + if (SCHEDULER_STOPPED()) + return; _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line); #ifndef INVARIANTS if (sx_recursed(sx)) @@ -478,6 +494,9 @@ _sx_xlock_hard(struct sx *sx, uintptr_t tid, int o int64_t sleep_time = 0; #endif + if (SCHEDULER_STOPPED()) + return (0); + /* If we already hold an exclusive lock, then recurse. */ if (sx_xlocked(sx)) { KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0, @@ -678,6 +697,9 @@ _sx_xunlock_hard(struct sx *sx, uintptr_t tid, con uintptr_t x; int queue, wakeup_swapper; + if (SCHEDULER_STOPPED()) + return; + MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); /* If the lock is recursed, then unrecurse one level. */ @@ -688,6 +710,7 @@ _sx_xunlock_hard(struct sx *sx, uintptr_t tid, con CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx); return; } + MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)); if (LOCK_LOG_TEST(&sx->lock_object, 0)) @@ -750,6 +773,9 @@ _sx_slock_hard(struct sx *sx, int opts, const char int64_t sleep_time = 0; #endif + if (SCHEDULER_STOPPED()) + return (0); + /* * As with rwlocks, we don't make any attempt to try to block * shared locks once there is an exclusive waiter. @@ -916,6 +942,9 @@ _sx_sunlock_hard(struct sx *sx, const char *file, uintptr_t x; int wakeup_swapper; + if (SCHEDULER_STOPPED()) + return; + for (;;) { x = sx->sx_lock; Index: sys/kern/kern_lock.c =================================================================== --- sys/kern/kern_lock.c (revision 225025) +++ sys/kern/kern_lock.c (working copy) @@ -444,6 +444,9 @@ __lockmgr_args(struct lock *lk, u_int flags, struc ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d", __func__, file, line)); + if (SCHEDULER_STOPPED()) + return (0); + class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL; if (panicstr != NULL) { if (flags & LK_INTERLOCK) @@ -1207,6 +1210,9 @@ _lockmgr_disown(struct lock *lk, const char *file, { uintptr_t tid, x; + if (SCHEDULER_STOPPED()) + return; + tid = (uintptr_t)curthread; _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line); Index: sys/kern/kern_condvar.c =================================================================== --- sys/kern/kern_condvar.c (revision 225025) +++ sys/kern/kern_condvar.c (working copy) @@ -110,7 +110,7 @@ _cv_wait(struct cv *cvp, struct lock_object *lock) "Waiting on \"%s\"", cvp->cv_description); class = LOCK_CLASS(lock); - if (cold || panicstr) { + if (cold || SCHEDULER_STOPPED()) { /* * During autoconfiguration, just give interrupts * a chance, then just return. Don't run any other @@ -171,7 +171,7 @@ _cv_wait_unlock(struct cv *cvp, struct lock_object ("cv_wait_unlock cannot be used with Giant")); class = LOCK_CLASS(lock); - if (cold || panicstr) { + if (cold || SCHEDULER_STOPPED()) { /* * During autoconfiguration, just give interrupts * a chance, then just return. Don't run any other @@ -227,7 +227,7 @@ _cv_wait_sig(struct cv *cvp, struct lock_object *l "Waiting on \"%s\"", cvp->cv_description); class = LOCK_CLASS(lock); - if (cold || panicstr) { + if (cold || SCHEDULER_STOPPED()) { /* * After a panic, or during autoconfiguration, just give * interrupts a chance, then just return; don't run any other @@ -293,7 +293,7 @@ _cv_timedwait(struct cv *cvp, struct lock_object * "Waiting on \"%s\"", cvp->cv_description); class = LOCK_CLASS(lock); - if (cold || panicstr) { + if (cold || SCHEDULER_STOPPED()) { /* * After a panic, or during autoconfiguration, just give * interrupts a chance, then just return; don't run any other @@ -360,7 +360,7 @@ _cv_timedwait_sig(struct cv *cvp, struct lock_obje "Waiting on \"%s\"", cvp->cv_description); class = LOCK_CLASS(lock); - if (cold || panicstr) { + if (cold || SCHEDULER_STOPPED()) { /* * After a panic, or during autoconfiguration, just give * interrupts a chance, then just return; don't run any other Index: sys/kern/kern_rmlock.c =================================================================== --- sys/kern/kern_rmlock.c (revision 225025) +++ sys/kern/kern_rmlock.c (working copy) @@ -315,6 +315,9 @@ _rm_rlock(struct rmlock *rm, struct rm_priotracker struct thread *td = curthread; struct pcpu *pc; + if (SCHEDULER_STOPPED()) + return; + tracker->rmp_flags = 0; tracker->rmp_thread = td; tracker->rmp_rmlock = rm; @@ -383,6 +386,9 @@ _rm_runlock(struct rmlock *rm, struct rm_priotrack struct pcpu *pc; struct thread *td = tracker->rmp_thread; + if (SCHEDULER_STOPPED()) + return; + td->td_critnest++; /* critical_enter(); */ pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ rm_tracker_remove(pc, tracker); @@ -401,6 +407,9 @@ _rm_wlock(struct rmlock *rm) struct rm_priotracker *prio; struct turnstile *ts; + if (SCHEDULER_STOPPED()) + return; + mtx_lock(&rm->rm_lock); if (rm->rm_noreadtoken == 0) { @@ -447,6 +456,9 @@ _rm_wunlock(struct rmlock *rm) void _rm_wlock_debug(struct rmlock *rm, const char *file, int line) { + if (SCHEDULER_STOPPED()) + return; + WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); @@ -464,6 +476,9 @@ void _rm_wunlock_debug(struct rmlock *rm, const char *file, int line) { + if (SCHEDULER_STOPPED()) + return; + curthread->td_locks--; WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line); @@ -475,6 +490,9 @@ _rm_rlock_debug(struct rmlock *rm, struct rm_priot const char *file, int line) { + if (SCHEDULER_STOPPED()) + return; + WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line, NULL); _rm_rlock(rm, tracker); @@ -491,6 +509,9 @@ _rm_runlock_debug(struct rmlock *rm, struct rm_pri const char *file, int line) { + if (SCHEDULER_STOPPED()) + return; + curthread->td_locks--; WITNESS_UNLOCK(&rm->lock_object, 0, file, line); LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line); --- sys/sys/mutex.h +++ sys/sys/mutex.h @@ -344,7 +344,8 @@ do { \ \ if (mtx_owned(&Giant)) { \ WITNESS_SAVE(&Giant.lock_object, Giant); \ - for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++) \ + for (_giantcnt = 0; !SCHEDULER_STOPPED() && \ + mtx_owned(&Giant); _giantcnt++) \ mtx_unlock(&Giant); \ } Index: sys/sys/systm.h =================================================================== --- sys/sys/systm.h (revision 225025) +++ sys/sys/systm.h (working copy) @@ -47,6 +47,7 @@ extern int cold; /* nonzero if we are doing a cold boot */ extern int rebooting; /* boot() has been called. */ +extern int stop_scheduler_on_panic; /* only one thread runs after panic */ extern const char *panicstr; /* panic message */ extern char version[]; /* system version */ extern char copyright[]; /* system copyright */ @@ -111,6 +112,14 @@ enum VM_GUEST { VM_GUEST_NO = 0, VM_GUEST_VM, VM_G ((uintptr_t)&(var) & (sizeof(void *) - 1)) == 0, msg) /* + * If we have already panic'd and this is the thread that called + * panic(), then don't block on any mutexes but silently succeed. + * Otherwise, the kernel will deadlock since the scheduler isn't + * going to run the thread that holds any lock we need. + */ +#define SCHEDULER_STOPPED() (panicstr != NULL && stop_scheduler_on_panic) + +/* * XXX the hints declarations are even more misplaced than most declarations * in this file, since they are needed in one file (per arch) and only used * in two files.