Index: sys/sparc64/pci/schizo.c =================================================================== --- sys/sparc64/pci/schizo.c (revision 241446) +++ sys/sparc64/pci/schizo.c (working copy) @@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include Index: sys/sparc64/pci/fire.c =================================================================== --- sys/sparc64/pci/fire.c (revision 241446) +++ sys/sparc64/pci/fire.c (working copy) @@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include Index: sys/i386/i386/i686_mem.c =================================================================== --- sys/i386/i386/i686_mem.c (revision 241446) +++ sys/i386/i386/i686_mem.c (working copy) @@ -32,6 +32,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include Index: sys/i386/pci/pci_cfgreg.c =================================================================== --- sys/i386/pci/pci_cfgreg.c (revision 241446) +++ sys/i386/pci/pci_cfgreg.c (working copy) @@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include Index: sys/amd64/amd64/amd64_mem.c =================================================================== --- sys/amd64/amd64/amd64_mem.c (revision 241446) +++ sys/amd64/amd64/amd64_mem.c (working copy) @@ -32,6 +32,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include Index: sys/sys/buf_ring.h =================================================================== --- sys/sys/buf_ring.h (revision 241446) +++ sys/sys/buf_ring.h (working copy) @@ -30,6 +30,8 @@ #ifndef _SYS_BUF_RING_H_ #define _SYS_BUF_RING_H_ +#include + #include #if defined(INVARIANTS) && !defined(DEBUG_BUFRING) Index: sys/sys/systm.h =================================================================== --- sys/sys/systm.h (revision 241446) +++ sys/sys/systm.h (working copy) @@ -171,11 +171,37 @@ void g_waitidle(void); void panic(const char *, ...) __dead2 __printflike(1, 2); +void _critical_enter(const char *, int); +void _critical_exit(const char *, int); +void _critical_exit_hard(struct thread *); + +#define _critical_enter_fast() do { \ + curthread->td_critnest++; \ + __compiler_membar(); \ +} while (0) + +#define _critical_exit_fast() do { \ + struct thread *__td; \ + \ + __td = curthread; \ + __compiler_membar(); \ + if (__td->td_critnest != 1) \ + __td->td_critnest--; \ + else \ + _critical_exit_hard(__td); \ +} while (0) + +#if defined(INVARIANT_SUPPORT) || defined(INVARIANTS) || defined(KTR) +#define critical_enter() _critical_enter(__FILE__, __LINE__) +#define critical_exit() _critical_exit(__FILE__, __LINE__) +#else +#define critical_enter() _critical_enter_fast() +#define critical_exit() _critical_exit_fast() +#endif + void cpu_boot(int); void cpu_flush_dcache(void *, size_t); void cpu_rootconf(void); -void critical_enter(void); -void critical_exit(void); void init_param1(void); void init_param2(long physpages); void init_static_kenv(char *, size_t); Index: sys/kern/kern_switch.c =================================================================== --- sys/kern/kern_switch.c (revision 241446) +++ sys/kern/kern_switch.c (working copy) @@ -178,47 +178,56 @@ retry: * regions of code in which preemptions are not allowed. */ void -critical_enter(void) +_critical_enter(const char *file, int line) { struct thread *td; td = curthread; td->td_critnest++; - CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, - (long)td->td_proc->p_pid, td->td_name, td->td_critnest); + CTR6(KTR_CRITICAL, "critical_enter: thread %p (%ld, %s) to %d @ %s:%d", + td, (long)td->td_proc->p_pid, td->td_name, td->td_critnest, file, + line); } void -critical_exit(void) +_critical_exit(const char *file, int line) { struct thread *td; - int flags; td = curthread; KASSERT(td->td_critnest != 0, ("critical_exit: td_critnest == 0")); - if (td->td_critnest == 1) { - td->td_critnest = 0; - if (td->td_owepreempt && !kdb_active) { - td->td_critnest = 1; - thread_lock(td); - td->td_critnest--; - flags = SW_INVOL | SW_PREEMPT; - if (TD_IS_IDLETHREAD(td)) - flags |= SWT_IDLE; - else - flags |= SWT_OWEPREEMPT; - mi_switch(flags, NULL); - thread_unlock(td); - } - } else + if (td->td_critnest != 1) td->td_critnest--; + else + _critical_exit_hard(td); - CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td, - (long)td->td_proc->p_pid, td->td_name, td->td_critnest); + CTR6(KTR_CRITICAL, "critical_exit: thread %p (%ld, %s) to %d @ %s:%d", + td, (long)td->td_proc->p_pid, td->td_name, td->td_critnest, + file, line); } +void +_critical_exit_hard(struct thread *td) +{ + int flags; + + td->td_critnest = 0; + if (td->td_owepreempt && !kdb_active) { + td->td_critnest = 1; + thread_lock(td); + td->td_critnest--; + flags = SW_INVOL | SW_PREEMPT; + if (TD_IS_IDLETHREAD(td)) + flags |= SWT_IDLE; + else + flags |= SWT_OWEPREEMPT; + mi_switch(flags, NULL); + thread_unlock(td); + } +} + /************************************************************************ * SYSTEM RUN QUEUE manipulations and tests * ************************************************************************/ Index: sys/kern/subr_smp.c =================================================================== --- sys/kern/subr_smp.c (revision 241446) +++ sys/kern/subr_smp.c (working copy) @@ -368,7 +368,7 @@ smp_rendezvous_action(void) * during this routine. */ td = curthread; - td->td_critnest++; + critical_enter(); #ifdef INVARIANTS owepreempt = td->td_owepreempt; #endif Index: sys/kern/kern_rmlock.c =================================================================== --- sys/kern/kern_rmlock.c (revision 241446) +++ sys/kern/kern_rmlock.c (working copy) @@ -347,10 +347,8 @@ _rm_rlock(struct rmlock *rm, struct rm_priotracker tracker->rmp_thread = td; tracker->rmp_rmlock = rm; - td->td_critnest++; /* critical_enter(); */ + critical_enter(); - __compiler_membar(); - pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ rm_tracker_add(pc, tracker); @@ -415,7 +413,7 @@ _rm_runlock(struct rmlock *rm, struct rm_priotrack if (SCHEDULER_STOPPED()) return; - td->td_critnest++; /* critical_enter(); */ + critical_enter(); pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ rm_tracker_remove(pc, tracker); td->td_critnest--; Index: sys/mips/nlm/tick.c =================================================================== --- sys/mips/nlm/tick.c (revision 241446) +++ sys/mips/nlm/tick.c (working copy) @@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include Index: sys/mips/nlm/intr_machdep.c =================================================================== --- sys/mips/nlm/intr_machdep.c (revision 241446) +++ sys/mips/nlm/intr_machdep.c (working copy) @@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include Index: sys/mips/mips/tick.c =================================================================== --- sys/mips/mips/tick.c (revision 241446) +++ sys/mips/mips/tick.c (working copy) @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include Index: sys/dev/scd/scd.c =================================================================== --- sys/dev/scd/scd.c (revision 241446) +++ sys/dev/scd/scd.c (working copy) @@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include Index: sys/dev/mcd/mcd.c =================================================================== --- sys/dev/mcd/mcd.c (revision 241446) +++ sys/dev/mcd/mcd.c (working copy) @@ -55,6 +55,7 @@ static const char COPYRIGHT[] = "mcd-driver (C)199 #include #include #include +#include #include #include Index: sys/dev/glxiic/glxiic.c =================================================================== --- sys/dev/glxiic/glxiic.c (revision 241446) +++ sys/dev/glxiic/glxiic.c (working copy) @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #ifdef GLXIIC_DEBUG #include