Index: sys/amd64/vmm/intel/ept.c =================================================================== --- sys/amd64/vmm/intel/ept.c (revision 331013) +++ sys/amd64/vmm/intel/ept.c (working copy) @@ -33,6 +33,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include Index: sys/amd64/vmm/intel/vmcs.c =================================================================== --- sys/amd64/vmm/intel/vmcs.c (revision 331013) +++ sys/amd64/vmm/intel/vmcs.c (working copy) @@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$"); #include +#include #include #include #include Index: sys/amd64/vmm/intel/vmx_genassym.c =================================================================== --- sys/amd64/vmm/intel/vmx_genassym.c (revision 331013) +++ sys/amd64/vmm/intel/vmx_genassym.c (working copy) @@ -32,6 +32,7 @@ __FBSDID("$FreeBSD$"); #include +#include #include #include #include Index: sys/dev/cxgbe/t4_mp_ring.c =================================================================== --- sys/dev/cxgbe/t4_mp_ring.c (revision 331013) +++ sys/dev/cxgbe/t4_mp_ring.c (working copy) @@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include "t4_mp_ring.h" Index: sys/dev/hyperv/vmbus/vmbus.c =================================================================== --- sys/dev/hyperv/vmbus/vmbus.c (revision 331013) +++ sys/dev/hyperv/vmbus/vmbus.c (working copy) @@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include Index: sys/dev/ow/owc_gpiobus.c =================================================================== --- sys/dev/ow/owc_gpiobus.c (revision 331013) +++ sys/dev/ow/owc_gpiobus.c (working copy) @@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #ifdef FDT #include Index: sys/kern/kern_switch.c =================================================================== --- sys/kern/kern_switch.c (revision 331013) +++ sys/kern/kern_switch.c (working copy) @@ -188,71 +188,29 @@ choosethread(void) return (td); } -/* - * Kernel thread preemption implementation. Critical sections mark - * regions of code in which preemptions are not allowed. - * - * It might seem a good idea to inline critical_enter() but, in order - * to prevent instructions reordering by the compiler, a __compiler_membar() - * would have to be used here (the same as sched_pin()). The performance - * penalty imposed by the membar could, then, produce slower code than - * the function call itself, for most cases. - */ void -critical_enter(void) +critical_exit_owepreempt(struct thread *td) { - struct thread *td; - - td = curthread; - td->td_critnest++; - CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, - (long)td->td_proc->p_pid, td->td_name, td->td_critnest); -} - -void -critical_exit(void) -{ - struct thread *td; int flags; - td = curthread; - KASSERT(td->td_critnest != 0, - ("critical_exit: td_critnest == 0")); + if (kdb_active) + return; - if (td->td_critnest == 1) { - td->td_critnest = 0; - - /* - * Interrupt handlers execute critical_exit() on - * leave, and td_owepreempt may be left set by an - * interrupt handler only when td_critnest > 0. If we - * are decrementing td_critnest from 1 to 0, read - * td_owepreempt after decrementing, to not miss the - * preempt. Disallow compiler to reorder operations. - */ - __compiler_membar(); - if (td->td_owepreempt && !kdb_active) { - /* - * Microoptimization: we committed to switch, - * disable preemption in interrupt handlers - * while spinning for the thread lock. - */ - td->td_critnest = 1; - thread_lock(td); - td->td_critnest--; - flags = SW_INVOL | SW_PREEMPT; - if (TD_IS_IDLETHREAD(td)) - flags |= SWT_IDLE; - else - flags |= SWT_OWEPREEMPT; - mi_switch(flags, NULL); - thread_unlock(td); - } - } else - td->td_critnest--; - - CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td, - (long)td->td_proc->p_pid, td->td_name, td->td_critnest); + /* + * Microoptimization: we committed to switch, + * disable preemption in interrupt handlers + * while spinning for the thread lock. + */ + td->td_critnest = 1; + thread_lock(td); + td->td_critnest--; + flags = SW_INVOL | SW_PREEMPT; + if (TD_IS_IDLETHREAD(td)) + flags |= SWT_IDLE; + else + flags |= SWT_OWEPREEMPT; + mi_switch(flags, NULL); + thread_unlock(td); } /************************************************************************ Index: sys/net/if_var.h =================================================================== --- sys/net/if_var.h (revision 331013) +++ sys/net/if_var.h (working copy) @@ -73,6 +73,7 @@ struct netmap_adapter; #ifdef _KERNEL #include /* ifqueue only? */ +#include #include #include #endif /* _KERNEL */ Index: sys/net/mp_ring.c =================================================================== --- sys/net/mp_ring.c (revision 331013) +++ sys/net/mp_ring.c (working copy) @@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #if defined(__powerpc__) || defined(__mips__) Index: sys/sys/buf_ring.h =================================================================== --- sys/sys/buf_ring.h (revision 331013) +++ sys/sys/buf_ring.h (working copy) @@ -33,6 +33,7 @@ #define _SYS_BUF_RING_H_ #include +#include #if defined(INVARIANTS) && !defined(DEBUG_BUFRING) #define DEBUG_BUFRING 1 Index: sys/sys/proc.h =================================================================== --- sys/sys/proc.h (revision 331013) +++ sys/sys/proc.h (working copy) @@ -48,6 +48,7 @@ #endif #include #include +#include #include #include #include @@ -1107,7 +1108,6 @@ void thread_unlink(struct thread *td); void thread_unsuspend(struct proc *p); void thread_wait(struct proc *p); struct thread *thread_find(struct proc *p, lwpid_t tid); - void stop_all_proc(void); void resume_all_proc(void); @@ -1146,6 +1146,64 @@ td_softdep_cleanup(struct thread *td) softdep_ast_cleanup(td); } +/* + * Kernel thread preemption implementation. Critical sections mark + * regions of code in which preemptions are not allowed. + */ + +void critical_exit_owepreempt(struct thread *td); + +static __inline void +critical_enter(void) +{ + struct thread *td; + + td = curthread; + td->td_critnest++; + __compiler_membar(); + + CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, + (long)td->td_proc->p_pid, td->td_name, td->td_critnest); +} + +static __inline void +critical_exit(void) +{ + struct thread *td; + + td = curthread; + KASSERT(td->td_critnest != 0, + ("critical_exit: td_critnest == 0")); + + if (td->td_critnest == 1) { + td->td_critnest = 0; + + /* + * Interrupt handlers execute critical_exit() on + * leave, and td_owepreempt may be left set by an + * interrupt handler only when td_critnest > 0. If we + * are decrementing td_critnest from 1 to 0, read + * td_owepreempt after decrementing, to not miss the + * preempt. Disallow compiler to reorder operations. + */ + __compiler_membar(); +#if 1 + if (td->td_owepreempt) + critical_exit_owepreempt(td); +#endif + } else + td->td_critnest--; + + CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td, + (long)td->td_proc->p_pid, td->td_name, td->td_critnest); +} + +/* + * Assert that a thread is in critical(9) section. + */ +#define CRITICAL_ASSERT(td) \ + KASSERT((td)->td_critnest >= 1, ("Not in critical section")); + #endif /* _KERNEL */ #endif /* !_SYS_PROC_H_ */ Index: sys/sys/systm.h =================================================================== --- sys/sys/systm.h (revision 331013) +++ sys/sys/systm.h (working copy) @@ -119,12 +119,6 @@ void kassert_panic(const char *fmt, ...) __printf ((uintptr_t)&(var) & (sizeof(void *) - 1)) == 0, msg) /* - * Assert that a thread is in critical(9) section. - */ -#define CRITICAL_ASSERT(td) \ - KASSERT((td)->td_critnest >= 1, ("Not in critical section")); - -/* * If we have already panic'd and this is the thread that called * panic(), then don't block on any mutexes but silently succeed. * Otherwise, the kernel will deadlock since the scheduler isn't @@ -214,8 +208,6 @@ void vpanic(const char *, __va_list) __dead2 __pri void cpu_boot(int); void cpu_flush_dcache(void *, size_t); void cpu_rootconf(void); -void critical_enter(void); -void critical_exit(void); void init_param1(void); void init_param2(long physpages); void init_static_kenv(char *, size_t); Index: sys/x86/x86/x86_mem.c =================================================================== --- sys/x86/x86/x86_mem.c (revision 331013) +++ sys/x86/x86/x86_mem.c (working copy) @@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include Index: sys/x86/xen/xen_intr.c =================================================================== --- sys/x86/xen/xen_intr.c (revision 331013) +++ sys/x86/xen/xen_intr.c (working copy) @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include