--- //depot/vendor/freebsd/src/sys/amd64/amd64/machdep.c 2012-02-15 21:35:20.000000000 0000 +++ //depot/user/jhb/cr8/amd64/amd64/machdep.c 2012-02-23 21:04:48.000000000 0000 @@ -182,6 +182,10 @@ long Maxmem = 0; long realmem = 0; +#ifdef DEV_ATPIC +int use_tpr; +#endif + /* * The number of PHYSMAP entries must be one less than the number of * PHYSSEG entries because the PHYSMAP entry that spans the largest @@ -1896,13 +1900,18 @@ spinlock_enter(void) { struct thread *td; - register_t flags; + register_t reg; td = curthread; if (td->td_md.md_spinlock_count == 0) { - flags = intr_disable(); +#ifdef DEV_ATPIC + if (!use_tpr) + reg = intr_disable(); + else +#endif + reg = raise_tpr(IDT_CRITICAL); td->td_md.md_spinlock_count = 1; - td->td_md.md_saved_flags = flags; + td->td_md.md_saved_reg = reg; } else td->td_md.md_spinlock_count++; critical_enter(); @@ -1912,14 +1921,20 @@ spinlock_exit(void) { struct thread *td; - register_t flags; + register_t reg; td = curthread; critical_exit(); - flags = td->td_md.md_saved_flags; + reg = td->td_md.md_saved_reg; td->td_md.md_spinlock_count--; - if (td->td_md.md_spinlock_count == 0) - intr_restore(flags); + if (td->td_md.md_spinlock_count == 0) { +#ifdef DEV_ATPIC + if (!use_tpr) + intr_restore(reg); + else +#endif + restore_tpr(reg); + } } /* --- //depot/vendor/freebsd/src/sys/amd64/amd64/mp_machdep.c 2012-02-15 23:35:16.000000000 0000 +++ //depot/user/jhb/cr8/amd64/amd64/mp_machdep.c 2012-02-23 21:04:48.000000000 0000 @@ -731,6 +731,10 @@ /* Init local apic for irq's */ lapic_setup(1); + /* Raise TPR and enable interrupts. */ + raise_tpr(IDT_CRITICAL); + enable_intr(); + /* Set memory range attributes for this CPU to match the BSP */ mem_range_AP_init(); @@ -1106,6 +1110,10 @@ lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); } +/* CR8 assumes the low 4 bits are always zero. */ +CTASSERT((IDT_CRITICAL & 0x0f) == 0); +CTASSERT(IPI_RENDEZVOUS < IDT_CRITICAL && IPI_INVLTLB >= IDT_CRITICAL); + /* * Flush the TLB on all other CPU's */ @@ -1256,8 +1264,11 @@ struct trapframe *oldframe; struct thread *td; int cpu = PCPU_GET(cpuid); + register_t tpr; u_int ipi_bitmap; + tpr = raise_tpr(IDT_CRITICAL); + enable_intr(); critical_enter(); td = curthread; td->td_intr_nesting_level++; @@ -1285,6 +1296,8 @@ td->td_intr_frame = oldframe; td->td_intr_nesting_level--; critical_exit(); + disable_intr(); + restore_tpr(tpr); } /* --- //depot/vendor/freebsd/src/sys/amd64/amd64/trap.c 2012-01-21 17:50:18.000000000 0000 +++ //depot/user/jhb/cr8/amd64/amd64/trap.c 2012-02-14 21:50:39.000000000 0000 @@ -44,6 +44,7 @@ * AMD64 Trap and System call handling */ +#include "opt_atpic.h" #include "opt_clock.h" #include "opt_cpu.h" #include "opt_hwpmc_hooks.h" @@ -290,12 +291,13 @@ */ printf("kernel trap %d with interrupts disabled\n", type); - +#ifdef DEV_ATPIC /* * We shouldn't enable interrupts while holding a * spin lock. */ - if (td->td_md.md_spinlock_count == 0) + if (use_tpr || td->td_md.md_spinlock_count == 0) +#endif enable_intr(); } } --- //depot/vendor/freebsd/src/sys/amd64/amd64/vm_machdep.c 2012-02-10 21:30:19.000000000 0000 +++ //depot/user/jhb/cr8/amd64/amd64/vm_machdep.c 2012-02-23 21:04:48.000000000 0000 @@ -43,9 +43,10 @@ #include __FBSDID("$FreeBSD: src/sys/amd64/amd64/vm_machdep.c,v 1.279 2012/02/10 21:26:25 kib Exp $"); +#include "opt_atpic.h" +#include "opt_compat.h" +#include "opt_cpu.h" #include "opt_isa.h" -#include "opt_cpu.h" -#include "opt_compat.h" #include #include @@ -231,7 +232,12 @@ /* Setup to release spin count in fork_exit(). */ td2->td_md.md_spinlock_count = 1; - td2->td_md.md_saved_flags = PSL_KERNEL | PSL_I; +#ifdef DEV_ATPIC + if (!use_tpr) + td2->td_md.md_saved_reg = PSL_KERNEL | PSL_I; + else +#endif + td2->td_md.md_saved_reg = 0; /* As an i386, do not copy io permission bitmap. */ pcb2->pcb_tssp = NULL; @@ -477,8 +483,43 @@ /* Setup to release spin count in fork_exit(). */ td->td_md.md_spinlock_count = 1; - td->td_md.md_saved_flags = PSL_KERNEL | PSL_I; +#ifdef DEV_ATPIC + if (!use_tpr) + td->td_md.md_saved_reg = PSL_KERNEL | PSL_I; + else +#endif + td->td_md.md_saved_reg = 0; +} + +#ifdef DEV_ATPIC +void +enable_tpr(void) +{ + struct proc *p; + struct thread *td; + + use_tpr = 1; + + /* + * When this is called at SI_SUB_INTR, some threads have + * already been created (init and idle threads), but not + * started. Adjust their saved spinlock state accordingly. + */ + FOREACH_PROC_IN_SYSTEM(p) { + FOREACH_THREAD_IN_PROC(p, td) { + if (td != curthread) { + MPASS(td->td_md.md_spinlock_count == 1); + MPASS(td->td_state == TDS_INACTIVE || + td->td_state == TDS_RUNQ || + td->td_state == TDS_CAN_RUN); + MPASS(td->td_md.md_saved_reg == + (PSL_KERNEL | PSL_I)); + td->td_md.md_saved_reg = 0; + } + } + } } +#endif /* * Set that machine state for performing an upcall that has to --- //depot/vendor/freebsd/src/sys/amd64/include/apicvar.h 2010-09-13 07:30:15.000000000 0000 +++ //depot/user/jhb/cr8/amd64/include/apicvar.h 2012-02-27 19:04:39.000000000 0000 @@ -42,11 +42,11 @@ * Layout of local APIC interrupt vectors: * * 0xff (255) +-------------+ - * | | 15 (Spurious / IPIs / Local Interrupts) + * | | 15 (Spurious / Critical IPIs ) * 0xf0 (240) +-------------+ - * | | 14 (I/O Interrupts / Timer) + * | | 14 (Local Interrupts / IPIs ) * 0xe0 (224) +-------------+ - * | | 13 (I/O Interrupts) + * | | 13 (I/O Interrupts / Timer ) * 0xd0 (208) +-------------+ * | | 12 (I/O Interrupts) * 0xc0 (192) +-------------+ @@ -84,7 +84,7 @@ /* I/O Interrupts are used for external devices such as ISA, PCI, etc. */ #define APIC_IO_INTS (IDT_IO_INTS + 16) -#define APIC_NUM_IOINTS 191 +#define APIC_NUM_IOINTS 175 /* The timer interrupt is used for clock handling and drives hardclock, etc. */ #define APIC_TIMER_INT (APIC_IO_INTS + APIC_NUM_IOINTS) @@ -105,19 +105,14 @@ */ /* Interrupts for local APIC LVT entries other than the timer. */ -#define APIC_LOCAL_INTS 240 +#define APIC_LOCAL_INTS 224 #define APIC_ERROR_INT APIC_LOCAL_INTS #define APIC_THERMAL_INT (APIC_LOCAL_INTS + 1) #define APIC_CMC_INT (APIC_LOCAL_INTS + 2) +#define APIC_IPI_INTS (APIC_LOCAL_INTS + 3) -#define APIC_IPI_INTS (APIC_LOCAL_INTS + 3) -#define IPI_RENDEZVOUS (APIC_IPI_INTS) /* Inter-CPU rendezvous. */ -#define IPI_INVLTLB (APIC_IPI_INTS + 1) /* TLB Shootdown IPIs */ -#define IPI_INVLPG (APIC_IPI_INTS + 2) -#define IPI_INVLRNG (APIC_IPI_INTS + 3) -#define IPI_INVLCACHE (APIC_IPI_INTS + 4) /* Vector to handle bitmap based IPIs */ -#define IPI_BITMAP_VECTOR (APIC_IPI_INTS + 6) +#define IPI_BITMAP_VECTOR (APIC_IPI_INTS) /* IPIs handled by IPI_BITMAPED_VECTOR (XXX ups is there a better place?) */ #define IPI_AST 0 /* Generate software trap. */ @@ -126,9 +121,17 @@ #define IPI_BITMAP_LAST IPI_HARDCLOCK #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST) -#define IPI_STOP (APIC_IPI_INTS + 7) /* Stop CPU until restarted. */ -#define IPI_SUSPEND (APIC_IPI_INTS + 8) /* Suspend CPU until restarted. */ -#define IPI_STOP_HARD (APIC_IPI_INTS + 9) /* Stop CPU with a NMI. */ +#define IPI_STOP (APIC_IPI_INTS + 1) /* Stop CPU until restarted. */ +#define IPI_SUSPEND (APIC_IPI_INTS + 2) /* Suspend CPU until restarted. */ +#define IPI_RENDEZVOUS (APIC_IPI_INTS + 3) /* Inter-CPU rendezvous. */ + +/* IPIs above this point are not blocked by spinlocks. */ +#define IPI_INVLTLB (IDT_CRITICAL) /* TLB Shootdown IPIs */ +#define IPI_INVLPG (IDT_CRITICAL + 1) +#define IPI_INVLRNG (IDT_CRITICAL + 2) +#define IPI_INVLCACHE (IDT_CRITICAL + 3) + +#define IPI_STOP_HARD (NIDT) /* Stop CPU with a NMI. */ /* * The spurious interrupt can share the priority class with the IPIs since @@ -225,8 +228,25 @@ enum intr_polarity pol); int lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, enum intr_trigger trigger); -void lapic_set_tpr(u_int vector); void lapic_setup(int boot); +static __inline register_t +raise_tpr(u_int level) +{ + register_t cr8; + + cr8 = rcr8(); + if (cr8 < (level >> 4)) + load_cr8(level >> 4); + return (cr8); +} + +static __inline void +restore_tpr(register_t cr8) +{ + + load_cr8(cr8); +} + #endif /* !LOCORE */ #endif /* _MACHINE_APICVAR_H_ */ --- //depot/vendor/freebsd/src/sys/amd64/include/cpufunc.h 2012-02-27 17:30:15.000000000 0000 +++ //depot/user/jhb/cr8/amd64/include/cpufunc.h 2012-02-27 17:35:27.000000000 0000 @@ -409,6 +409,22 @@ return (data); } +static __inline void +load_cr8(u_long data) +{ + + __asm __volatile("movq %0,%%cr8" : : "r" (data)); +} + +static __inline u_long +rcr8(void) +{ + u_long data; + + __asm __volatile("movq %%cr8,%0" : "=r" (data)); + return (data); +} + /* * Global TLB flush (except for thise for pages marked PG_G) */ @@ -695,6 +711,7 @@ void load_cr0(u_long cr0); void load_cr3(u_long cr3); void load_cr4(u_long cr4); +void load_cr8(u_long cr4); void load_dr0(uint64_t dr0); void load_dr1(uint64_t dr1); void load_dr2(uint64_t dr2); @@ -716,6 +733,7 @@ u_long rcr2(void); u_long rcr3(void); u_long rcr4(void); +u_long rcr8(void); uint64_t rdmsr(u_int msr); uint64_t rdpmc(u_int pmc); uint64_t rdr0(void); --- //depot/vendor/freebsd/src/sys/amd64/include/md_var.h 2012-01-21 17:50:18.000000000 0000 +++ //depot/user/jhb/cr8/amd64/include/md_var.h 2012-02-14 21:50:39.000000000 0000 @@ -68,6 +68,7 @@ extern int _ucode32sel; extern int _ufssel; extern int _ugssel; +extern int use_tpr; extern int use_xsave; extern uint64_t xsave_mask; @@ -101,6 +102,7 @@ void gsbase_load_fault(void) __asm(__STRING(gsbase_load_fault)); void dump_add_page(vm_paddr_t); void dump_drop_page(vm_paddr_t); +void enable_tpr(void); void initializecpu(void); void initializecpucache(void); void fillw(int /*u_short*/ pat, void *base, size_t cnt); --- //depot/vendor/freebsd/src/sys/amd64/include/proc.h 2011-10-07 16:10:16.000000000 0000 +++ //depot/user/jhb/cr8/amd64/include/proc.h 2012-02-08 15:48:05.000000000 0000 @@ -45,7 +45,7 @@ */ struct mdthread { int md_spinlock_count; /* (k) */ - register_t md_saved_flags; /* (k) */ + register_t md_saved_reg; /* (k) */ }; struct mdproc { --- //depot/vendor/freebsd/src/sys/amd64/include/segments.h 2012-02-27 17:30:15.000000000 0000 +++ //depot/user/jhb/cr8/amd64/include/segments.h 2012-02-27 17:35:27.000000000 0000 @@ -215,6 +215,7 @@ #define IDT_IO_INTS NRSVIDT /* Base of IDT entries for I/O interrupts. */ #define IDT_DTRACE_RET 0x20 /* DTrace pid provider Interrupt Vector */ #define IDT_SYSCALL 0x80 /* System Call Interrupt Vector */ +#define IDT_CRITICAL 0xf0 /* First interrupt not blocked by spin locks */ /* * Entries in the Global Descriptor Table (GDT) --- //depot/vendor/freebsd/src/sys/i386/acpica/acpi_wakeup.c 2010-11-12 21:00:27.000000000 0000 +++ //depot/user/jhb/cr8/i386/acpica/acpi_wakeup.c 2012-02-27 19:04:39.000000000 0000 @@ -25,6 +25,9 @@ * SUCH DAMAGE. */ +#include "opt_apic.h" +#include "opt_atpic.h" + #include __FBSDID("$FreeBSD: src/sys/i386/acpica/acpi_wakeup.c,v 1.53 2010/11/12 20:55:14 jkim Exp $"); @@ -45,7 +48,9 @@ #include #include #include +#include #include +#include #include #include @@ -198,6 +203,9 @@ struct pmap *pm; int ret; uint32_t cr3; +#ifdef DEV_APIC + uint32_t tpr; +#endif u_long ef; ret = 0; @@ -207,6 +215,14 @@ AcpiSetFirmwareWakingVector(sc->acpi_wakephys); ef = read_eflags(); +#ifdef DEV_APIC +#ifdef DEV_ATPIC + if (!use_tpr) + tpr = 0; + else +#endif + tpr = lapic_get_tpr(); +#endif /* * Temporarily switch to the kernel pmap because it provides an @@ -285,6 +301,12 @@ out: load_cr3(cr3); +#ifdef DEV_APIC +#ifdef DEV_ATPIC + if (use_tpr) +#endif + lapic_set_tpr(tpr); +#endif write_eflags(ef); /* If we beeped, turn it off after a delay. */ --- //depot/vendor/freebsd/src/sys/i386/i386/machdep.c 2012-01-21 17:50:18.000000000 0000 +++ //depot/user/jhb/cr8/i386/i386/machdep.c 2012-02-27 19:04:39.000000000 0000 @@ -40,7 +40,9 @@ #include __FBSDID("$FreeBSD: src/sys/i386/i386/machdep.c,v 1.741 2012/01/21 17:45:27 kib Exp $"); +#include "opt_apic.h" #include "opt_atalk.h" +#include "opt_atpic.h" #include "opt_compat.h" #include "opt_cpu.h" #include "opt_ddb.h" @@ -116,6 +118,8 @@ #include #include #include +#include +#include #include #include #include @@ -216,6 +220,10 @@ long Maxmem = 0; long realmem = 0; +#if defined(DEV_ATPIC) && defined(DEV_APIC) +int use_tpr; +#endif + #ifdef PAE FEATURE(pae, "Physical Address Extensions"); #endif @@ -3081,13 +3089,22 @@ spinlock_enter(void) { struct thread *td; - register_t flags; + register_t reg; td = curthread; if (td->td_md.md_spinlock_count == 0) { - flags = intr_disable(); +#if defined(DEV_APIC) || defined(DEV_ATPIC) + if (use_tpr) + reg = raise_tpr(IDT_CRITICAL); + else + reg = intr_disable(); +#elif defined(DEV_APIC) + reg = raise_tpr(IDT_CRITICAL); +#else + reg = intr_disable(); +#endif td->td_md.md_spinlock_count = 1; - td->td_md.md_saved_flags = flags; + td->td_md.md_saved_reg = reg; } else td->td_md.md_spinlock_count++; critical_enter(); @@ -3097,14 +3114,24 @@ spinlock_exit(void) { struct thread *td; - register_t flags; + register_t reg; td = curthread; critical_exit(); - flags = td->td_md.md_saved_flags; + reg = td->td_md.md_saved_reg; td->td_md.md_spinlock_count--; - if (td->td_md.md_spinlock_count == 0) - intr_restore(flags); + if (td->td_md.md_spinlock_count == 0) { +#if defined(DEV_APIC) || defined(DEV_ATPIC) + if (use_tpr) + restore_tpr(reg); + else + intr_restore(reg); +#elif defined(DEV_APIC) + restore_tpr(reg); +#else + intr_restore(reg); +#endif + } } #if defined(I586_CPU) && !defined(NO_F00F_HACK) --- //depot/vendor/freebsd/src/sys/i386/i386/mp_machdep.c 2011-12-15 17:55:17.000000000 0000 +++ //depot/user/jhb/cr8/i386/i386/mp_machdep.c 2012-02-10 12:24:57.000000000 0000 @@ -773,6 +773,10 @@ /* Init local apic for irq's */ lapic_setup(1); + /* Raise TPR and enable interrupts. */ + raise_tpr(IDT_CRITICAL); + enable_intr(); + /* Set memory range attributes for this CPU to match the BSP */ mem_range_AP_init(); @@ -1201,6 +1205,10 @@ lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); } +/* TPR works best if the low 4 bits are always zero. */ +CTASSERT((IDT_CRITICAL & 0x0f) == 0); +CTASSERT(IPI_RENDEZVOUS < IDT_CRITICAL && IPI_INVLTLB >= IDT_CRITICAL); + /* * Flush the TLB on all other CPU's */ @@ -1351,8 +1359,11 @@ struct trapframe *oldframe; struct thread *td; int cpu = PCPU_GET(cpuid); + register_t tpr; u_int ipi_bitmap; + tpr = raise_tpr(IDT_CRITICAL); + enable_intr(); critical_enter(); td = curthread; td->td_intr_nesting_level++; @@ -1380,6 +1391,8 @@ td->td_intr_frame = oldframe; td->td_intr_nesting_level--; critical_exit(); + disable_intr(); + restore_tpr(tpr); } /* --- //depot/vendor/freebsd/src/sys/i386/i386/trap.c 2012-02-27 17:35:20.000000000 0000 +++ //depot/user/jhb/cr8/i386/i386/trap.c 2012-02-27 18:13:48.000000000 0000 @@ -44,6 +44,8 @@ * 386 Trap and System call handling */ +#include "opt_apic.h" +#include "opt_atpic.h" #include "opt_clock.h" #include "opt_cpu.h" #include "opt_hwpmc_hooks.h" @@ -318,8 +320,13 @@ * and we shouldn't enable interrupts while holding * a spin lock. */ - if (type != T_PAGEFLT && - td->td_md.md_spinlock_count == 0) + if (type != T_PAGEFLT +#if defined(DEV_APIC) && defined(DEV_ATPIC) + && (use_tpr || td->td_md.md_spinlock_count == 0) +#elif defined(DEV_ATPIC) + && td->td_md.md_spinlock_count == 0 +#endif + ) enable_intr(); } } --- //depot/vendor/freebsd/src/sys/i386/i386/vm_machdep.c 2011-07-04 12:10:17.000000000 0000 +++ //depot/user/jhb/cr8/i386/i386/vm_machdep.c 2012-02-14 21:58:44.000000000 0000 @@ -43,10 +43,12 @@ #include __FBSDID("$FreeBSD: src/sys/i386/i386/vm_machdep.c,v 1.307 2011/07/04 12:04:52 attilio Exp $"); +#include "opt_apic.h" +#include "opt_atpic.h" +#include "opt_cpu.h" #include "opt_isa.h" #include "opt_npx.h" #include "opt_reset.h" -#include "opt_cpu.h" #include "opt_xbox.h" #include @@ -271,7 +273,16 @@ /* * XXX XEN need to check on PSL_USER is handled */ - td2->td_md.md_saved_flags = PSL_KERNEL | PSL_I; +#if defined(DEV_APIC) && defined(DEV_ATPIC) + if (use_tpr) + td2->td_md.md_saved_reg = 0; + else + td2->td_md.md_saved_reg = PSL_KERNEL | PSL_I; +#elif defined(DEV_APIC) + td2->td_md.md_saved_reg = 0; +#else + td2->td_md.md_saved_reg = PSL_KERNEL | PSL_I; +#endif /* * Now, cpu_switch() can schedule the new process. * pcb_esp is loaded pointing to the cpu_switch() stack frame @@ -482,8 +493,47 @@ /* Setup to release spin count in fork_exit(). */ td->td_md.md_spinlock_count = 1; - td->td_md.md_saved_flags = PSL_KERNEL | PSL_I; +#if defined(DEV_APIC) && defined(DEV_ATPIC) + if (use_tpr) + td->td_md.md_saved_reg = 0; + else + td->td_md.md_saved_reg = PSL_KERNEL | PSL_I; +#elif defined(DEV_APIC) + td->td_md.md_saved_reg = 0; +#else + td->td_md.md_saved_reg = PSL_KERNEL | PSL_I; +#endif +} + +#if defined(DEV_APIC) && defined(DEV_ATPIC) +void +enable_tpr(void) +{ + struct proc *p; + struct thread *td; + + use_tpr = 1; + + /* + * When this is called at SI_SUB_INTR, some threads have + * already been created (init and idle threads), but not + * started. Adjust their saved spinlock state accordingly. + */ + FOREACH_PROC_IN_SYSTEM(p) { + FOREACH_THREAD_IN_PROC(p, td) { + if (td != curthread) { + MPASS(td->td_md.md_spinlock_count == 1); + MPASS(td->td_state == TDS_INACTIVE || + td->td_state == TDS_RUNQ || + td->td_state == TDS_CAN_RUN); + MPASS(td->td_md.md_saved_reg == + (PSL_KERNEL | PSL_I)); + td->td_md.md_saved_reg = 0; + } + } + } } +#endif /* * Set that machine state for performing an upcall that has to --- //depot/vendor/freebsd/src/sys/i386/include/apicvar.h 2012-02-27 17:35:20.000000000 0000 +++ //depot/user/jhb/cr8/i386/include/apicvar.h 2012-02-27 19:04:39.000000000 0000 @@ -40,11 +40,11 @@ * Layout of local APIC interrupt vectors: * * 0xff (255) +-------------+ - * | | 15 (Spurious / IPIs / Local Interrupts) + * | | 15 (Spurious / Critical IPIs ) * 0xf0 (240) +-------------+ - * | | 14 (I/O Interrupts / Timer) + * | | 14 (Local Interrupts / IPIs ) * 0xe0 (224) +-------------+ - * | | 13 (I/O Interrupts) + * | | 13 (I/O Interrupts / Timer ) * 0xd0 (208) +-------------+ * | | 12 (I/O Interrupts) * 0xc0 (192) +-------------+ @@ -82,7 +82,7 @@ /* I/O Interrupts are used for external devices such as ISA, PCI, etc. */ #define APIC_IO_INTS (IDT_IO_INTS + 16) -#define APIC_NUM_IOINTS 191 +#define APIC_NUM_IOINTS 175 /* The timer interrupt is used for clock handling and drives hardclock, etc. */ #define APIC_TIMER_INT (APIC_IO_INTS + APIC_NUM_IOINTS) @@ -103,30 +103,36 @@ */ /* Interrupts for local APIC LVT entries other than the timer. */ -#define APIC_LOCAL_INTS 240 +#define APIC_LOCAL_INTS 224 #define APIC_ERROR_INT APIC_LOCAL_INTS #define APIC_THERMAL_INT (APIC_LOCAL_INTS + 1) #define APIC_CMC_INT (APIC_LOCAL_INTS + 2) #define APIC_IPI_INTS (APIC_LOCAL_INTS + 3) -#define IPI_RENDEZVOUS (APIC_IPI_INTS) /* Inter-CPU rendezvous. */ -#define IPI_INVLTLB (APIC_IPI_INTS + 1) /* TLB Shootdown IPIs */ -#define IPI_INVLPG (APIC_IPI_INTS + 2) -#define IPI_INVLRNG (APIC_IPI_INTS + 3) -#define IPI_INVLCACHE (APIC_IPI_INTS + 4) -#define IPI_LAZYPMAP (APIC_IPI_INTS + 5) /* Lazy pmap release. */ /* Vector to handle bitmap based IPIs */ -#define IPI_BITMAP_VECTOR (APIC_IPI_INTS + 6) +#define IPI_BITMAP_VECTOR (APIC_IPI_INTS) /* IPIs handled by IPI_BITMAPED_VECTOR (XXX ups is there a better place?) */ #define IPI_AST 0 /* Generate software trap. */ #define IPI_PREEMPT 1 -#define IPI_HARDCLOCK 2 +#define IPI_HARDCLOCK 2 #define IPI_BITMAP_LAST IPI_HARDCLOCK #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST) -#define IPI_STOP (APIC_IPI_INTS + 7) /* Stop CPU until restarted. */ -#define IPI_STOP_HARD (APIC_IPI_INTS + 8) /* Stop CPU with a NMI. */ +#define IPI_STOP (APIC_IPI_INTS + 1) /* Stop CPU until restarted. */ +#ifdef notyet +#define IPI_SUSPEND (APIC_IPI_INTS + 2) /* Suspend CPU until restarted. */ +#endif +#define IPI_RENDEZVOUS (APIC_IPI_INTS + 3) /* Inter-CPU rendezvous. */ + +/* IPIs above this point are not blocked by spinlocks. */ +#define IPI_INVLTLB (IDT_CRITICAL) /* TLB Shootdown IPIs */ +#define IPI_INVLPG (IDT_CRITICAL + 1) +#define IPI_INVLRNG (IDT_CRITICAL + 2) +#define IPI_INVLCACHE (IDT_CRITICAL + 3) +#define IPI_LAZYPMAP (IDT_CRITICAL + 4) /* Lazy pmap release. */ + +#define IPI_STOP_HARD (NIDT) /* Stop CPU with a NMI. */ /* * The spurious interrupt can share the priority class with the IPIs since @@ -205,6 +211,7 @@ void lapic_enable_cmc(void); int lapic_enable_pmc(void); void lapic_eoi(void); +u_int lapic_get_tpr(void); int lapic_id(void); void lapic_init(vm_paddr_t addr); int lapic_intr_pending(u_int vector); @@ -223,8 +230,33 @@ enum intr_polarity pol); int lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, enum intr_trigger trigger); -void lapic_set_tpr(u_int vector); +void lapic_set_tpr(u_int tpr); void lapic_setup(int boot); +#ifdef APIC_TPR_PRIO +static __inline register_t +raise_tpr(u_int level) +{ + u_int tpr; + + tpr = lapic_get_tpr(); +#ifdef CHEAP_TPR + if (tpr < level) + lapic_set_tpr(level); +#else + if ((tpr & APIC_TPR_PRIO) < level) + lapic_set_tpr(level | (tpr & ~APIC_TPR_PRIO)); +#endif + return (tpr); +} + +static __inline void +restore_tpr(register_t tpr) +{ + + lapic_set_tpr(tpr); +} +#endif + #endif /* !LOCORE */ #endif /* _MACHINE_APICVAR_H_ */ --- //depot/vendor/freebsd/src/sys/i386/include/md_var.h 2010-06-23 10:45:13.000000000 0000 +++ //depot/user/jhb/cr8/i386/include/md_var.h 2012-02-10 12:24:57.000000000 0000 @@ -66,6 +66,7 @@ #ifdef COMPAT_43 extern int szosigcode; #endif +extern int use_tpr; extern uint32_t *vm_page_dump; extern int vm_page_dump_size; extern int workaround_erratum383; @@ -92,6 +93,7 @@ void dump_add_page(vm_paddr_t); void dump_drop_page(vm_paddr_t); void enable_sse(void); +void enable_tpr(void); void fillw(int /*u_short*/ pat, void *base, size_t cnt); void i686_pagezero(void *addr); void sse2_pagezero(void *addr); --- //depot/vendor/freebsd/src/sys/i386/include/proc.h 2011-10-07 16:10:16.000000000 0000 +++ //depot/user/jhb/cr8/i386/include/proc.h 2012-02-10 12:24:57.000000000 0000 @@ -50,7 +50,7 @@ */ struct mdthread { int md_spinlock_count; /* (k) */ - register_t md_saved_flags; /* (k) */ + register_t md_saved_reg; /* (k) */ }; struct mdproc { --- //depot/vendor/freebsd/src/sys/i386/include/segments.h 2012-02-27 17:30:15.000000000 0000 +++ //depot/user/jhb/cr8/i386/include/segments.h 2012-02-27 17:35:27.000000000 0000 @@ -208,6 +208,7 @@ #define IDT_IO_INTS NRSVIDT /* Base of IDT entries for I/O interrupts. */ #define IDT_DTRACE_RET 0x20 /* DTrace pid provider Interrupt Vector */ #define IDT_SYSCALL 0x80 /* System Call Interrupt Vector */ +#define IDT_CRITICAL 0xf0 /* First interrupt not blocked by spin locks */ /* * Entries in the Global Descriptor Table (GDT) --- //depot/vendor/freebsd/src/sys/kern/kern_fork.c 2012-02-23 11:55:23.000000000 0000 +++ //depot/user/jhb/cr8/kern/kern_fork.c 2012-02-23 21:04:48.000000000 0000 @@ -37,6 +37,10 @@ #include __FBSDID("$FreeBSD: src/sys/kern/kern_fork.c,v 1.337 2012/02/23 11:50:23 kib Exp $"); +#ifdef __i386__ +#include "opt_apic.h" +#endif +#include "opt_atpic.h" #include "opt_kdtrace.h" #include "opt_ktrace.h" #include "opt_kstack_pages.h" @@ -74,6 +78,12 @@ #include #include +#include +#include +#include +#include +#include + #include #include @@ -975,6 +985,52 @@ CTR4(KTR_PROC, "fork_exit: new thread %p (td_sched %p, pid %d, %s)", td, td->td_sched, p->p_pid, td->td_name); +#ifdef __amd64__ + CTR2(KTR_PROC, "fork_exit: rflags %lx, cr8 %lx", read_rflags(), + rcr8()); + /* XXX: Temporary debugging */ +#ifdef DEV_ATPIC + if (!use_tpr) { + KASSERT(!(read_rflags() & PSL_I), + ("fork_exit() with interrupts enabled")); + } else +#endif + { + KASSERT(rcr8() == IDT_CRITICAL >> 4, + ("fork_exit() with bad TPR")); + KASSERT(read_rflags() & PSL_I, + ("fork_exit() with interrupts disabled")); + } +#endif +#ifdef __i386__ +#ifdef DEV_APIC + if (use_tpr) + CTR2(KTR_PROC, "fork_exit: eflags %x, tpr %x", read_eflags(), + lapic_get_tpr()); + else +#else + CTR1(KTR_PROC, "fork_exit: eflags %x", read_eflags()); +#endif + /* XXX: Temporary debugging */ +#if defined(DEV_APIC) && defined(DEV_ATPIC) + if (use_tpr) { + KASSERT(lapic_get_tpr() == IDT_CRITICAL, + ("fork_exit() with bad TPR")); + KASSERT(read_eflags() & PSL_I, + ("fork_exit() with interrupts disabled")); + } else + KASSERT(!(read_eflags() & PSL_I), + ("fork_exit() with interrupts enabled")); +#elif defined(DEV_APIC) + KASSERT(lapic_get_tpr() == IDT_CRITICAL, + ("fork_exit() with bad TPR")); + KASSERT(read_eflags() & PSL_I, + ("fork_exit() with interrupts disabled")); +#else + KASSERT(!(read_eflags() & PSL_I), + ("fork_exit() with interrupts enabled")); +#endif +#endif sched_fork_exit(td); /* * Processes normally resume in mi_switch() after being @@ -987,6 +1043,32 @@ } thread_unlock(td); +#ifdef __amd64__ +#ifdef DEV_ATPIC + if (!use_tpr) + KASSERT(read_rflags() & PSL_I, + ("fork_exit() with no locks, interrupts disabled")); + else +#endif + KASSERT(rcr8() == 0, + ("fork_exit() with no locks, bad TPR")); +#endif +#ifdef __i386__ +#if defined(DEV_APIC) && defined(DEV_ATPIC) + if (use_tpr) + KASSERT(lapic_get_tpr() == 0, + ("fork_exit() with no locks, bad TPR")); + else + KASSERT(read_eflags() & PSL_I, + ("fork_exit() with no locks, interrupts disabled")); +#elif defined(DEV_APIC) + KASSERT(lapic_get_tpr() == 0, + ("fork_exit() with no locks, bad TPR")); +#else + KASSERT(read_eflags() & PSL_I, + ("fork_exit() with no locks, interrupts disabled")); +#endif +#endif /* * cpu_set_fork_handler intercepts this function call to * have this call a non-return function to stay in kernel mode. --- //depot/vendor/freebsd/src/sys/kern/kern_synch.c 2011-12-11 21:05:23.000000000 0000 +++ //depot/user/jhb/cr8/kern/kern_synch.c 2012-02-14 21:50:39.000000000 0000 @@ -37,6 +37,10 @@ #include __FBSDID("$FreeBSD: src/sys/kern/kern_synch.c,v 1.330 2011/12/11 21:02:01 avg Exp $"); +#ifdef __i386__ +#include "opt_apic.h" +#endif +#include "opt_atpic.h" #include "opt_ktrace.h" #include "opt_sched.h" @@ -63,6 +67,10 @@ #include #endif +#include +#include +#include +#include #include #ifdef XEN @@ -410,6 +418,52 @@ struct thread *td; struct proc *p; +#ifdef __amd64__ + CTR2(KTR_PROC, "mi_switch: suspending rflags %lx, cr8 %lx", + read_rflags(), rcr8()); + /* XXX: Temporary debugging */ +#ifdef DEV_ATPIC + if (!use_tpr) { + KASSERT(!(read_rflags() & PSL_I), + ("mi_switch() suspending with interrupts enabled")); + } else +#endif + { + KASSERT(read_rflags() & PSL_I, + ("mi_switch() suspending with interrupts disabled")); + KASSERT(rcr8() == IDT_CRITICAL >> 4, + ("mi_switch() suspending with bad TPR")); + } +#endif +#ifdef __i386__ +#ifdef DEV_APIC + if (use_tpr) + CTR2(KTR_PROC, "mi_switch: suspending eflags %x, tpr %x", + read_eflags(), lapic_get_tpr()); + else +#endif + CTR1(KTR_PROC, "mi_switch: suspending eflags %x", + read_eflags()); + /* XXX: Temporary debugging */ +#if defined(DEV_APIC) || defined(DEV_ATPIC) + if (use_tpr) { + KASSERT(read_eflags() & PSL_I, + ("mi_switch() suspending with interrupts disabled")); + KASSERT(lapic_get_tpr() == IDT_CRITICAL, + ("mi_switch() suspending with bad TPR")); + } else + KASSERT(!(read_eflags() & PSL_I), + ("mi_switch() suspending with interrupts enabled")); +#elif defined(DEV_APIC) + KASSERT(read_eflags() & PSL_I, + ("mi_switch() suspending with interrupts disabled")); + KASSERT(lapic_get_tpr() == IDT_CRITICAL, + ("mi_switch() suspending with bad TPR")); +#else + KASSERT(!(read_eflags() & PSL_I), + ("mi_switch() suspending with interrupts enabled")); +#endif +#endif td = curthread; /* XXX */ THREAD_LOCK_ASSERT(td, MA_OWNED | MA_NOTRECURSED); p = td->td_proc; /* XXX */ @@ -472,6 +526,53 @@ CTR4(KTR_PROC, "mi_switch: new thread %ld (td_sched %p, pid %ld, %s)", td->td_tid, td->td_sched, p->p_pid, td->td_name); +#ifdef __amd64__ + CTR2(KTR_PROC, "mi_switch: resuming rflags %lx, cr8 %lx", + read_rflags(), rcr8()); + /* XXX: Temporary debugging */ +#ifdef DEV_ATPIC + if (!use_tpr) { + KASSERT(!(read_rflags() & PSL_I), + ("mi_switch() resuming with interrupts enabled")); + } else +#endif + { + KASSERT(read_rflags() & PSL_I, + ("mi_switch() resuming with interrupts disabled")); + KASSERT(rcr8() == IDT_CRITICAL >> 4, + ("mi_switch() resuming with bad TPR")); + } +#endif +#ifdef __i386__ +#ifdef DEV_APIC + if (use_tpr) + CTR2(KTR_PROC, "mi_switch: resuming eflags %x, tpr %x", + read_eflags(), lapic_get_tpr()); + else +#endif + CTR1(KTR_PROC, "mi_switch: resuming eflags %x", + read_eflags()); + /* XXX: Temporary debugging */ +#if defined(DEV_APIC) || defined(DEV_ATPIC) + if (use_tpr) { + KASSERT(read_eflags() & PSL_I, + ("mi_switch() resuming with interrupts disabled")); + KASSERT(lapic_get_tpr() == IDT_CRITICAL, + ("mi_switch() resuming with bad TPR")); + } else + KASSERT(!(read_eflags() & PSL_I), + ("mi_switch() resuming with interrupts enabled")); +#elif defined(DEV_APIC) + KASSERT(read_eflags() & PSL_I, + ("mi_switch() resuming with interrupts disabled")); + KASSERT(lapic_get_tpr() == IDT_CRITICAL, + ("mi_switch() resuming with bad TPR")); +#else + KASSERT(!(read_eflags() & PSL_I), + ("mi_switch() resuming with interrupts enabled")); +#endif +#endif + /* * If the last thread was exiting, finish cleaning it up. */ --- //depot/vendor/freebsd/src/sys/x86/x86/local_apic.c 2012-02-27 17:35:20.000000000 0000 +++ //depot/user/jhb/cr8/x86/x86/local_apic.c 2012-02-27 19:04:39.000000000 0000 @@ -84,7 +84,6 @@ /* Sanity checks on IDT vectors. */ CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT); CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS); -CTASSERT(APIC_LOCAL_INTS == 240); CTASSERT(IPI_STOP < APIC_SPURIOUS_INT); /* Magic IRQ values for the timer and syscalls. */ @@ -348,6 +347,9 @@ struct lapic *la; u_int32_t maxlvt; register_t saveintr; +#ifdef __i386__ + u_int tpr; +#endif char buf[MAXCOMLEN + 1]; la = &lapics[lapic_id()]; @@ -356,7 +358,13 @@ maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; /* Initialize the TPR to allow all interrupts. */ - lapic_set_tpr(0); +#ifdef __amd64__ + load_cr8(0); +#else + tpr = lapic_get_tpr(); + tpr &= ~APIC_TPR_PRIO; + lapic_set_tpr(tpr); +#endif /* Setup spurious vector and enable the local APIC. */ lapic_enable(); @@ -747,23 +755,20 @@ return (0); } -/* - * Adjust the TPR of the current CPU so that it blocks all interrupts below - * the passed in vector. - */ +#ifdef __i386__ +u_int +lapic_get_tpr(void) +{ + return (lapic->tpr); +} + void -lapic_set_tpr(u_int vector) +lapic_set_tpr(u_int tpr) { -#ifdef CHEAP_TPR - lapic->tpr = vector; -#else - u_int32_t tpr; - tpr = lapic->tpr & ~APIC_TPR_PRIO; - tpr |= vector; lapic->tpr = tpr; +} #endif -} void lapic_eoi(void) @@ -776,10 +781,28 @@ lapic_handle_intr(int vector, struct trapframe *frame) { struct intsrc *isrc; + register_t tpr; + + /* + * Raise TPR and enable interrupts to permit critical + * interrupts while processing device interrupts. + */ + tpr = raise_tpr(IDT_CRITICAL); +#ifdef __amd64__ + KASSERT(tpr < IDT_CRITICAL >> 4, + ("device interrupt with critical cr8")); +#else + KASSERT((tpr & APIC_TPR_PRIO) < IDT_CRITICAL, + ("device interrupt with critical tpr")); +#endif + enable_intr(); isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id), vector)); intr_execute_handlers(isrc, frame); + + disable_intr(); + restore_tpr(tpr); } void @@ -788,6 +811,7 @@ struct lapic *la; struct trapframe *oldframe; struct thread *td; + register_t tpr; /* Send EOI first thing. */ lapic_eoi(); @@ -808,6 +832,20 @@ return; #endif + /* + * Raise TPR and enable interrupts to permit critical + * interrupts while processing timer interrupts. + */ + tpr = raise_tpr(IDT_CRITICAL); +#ifdef __amd64__ + KASSERT(tpr < IDT_CRITICAL >> 4, + ("timer interrupt with critical cr8")); +#else + KASSERT((tpr & APIC_TPR_PRIO) < IDT_CRITICAL, + ("timer interrupt with critical tpr")); +#endif + enable_intr(); + /* Look up our local APIC structure for the tick counters. */ la = &lapics[PCPU_GET(apic_id)]; (*la->la_timer_count)++; @@ -822,6 +860,9 @@ td->td_intr_nesting_level--; } critical_exit(); + + disable_intr(); + restore_tpr(tpr); } static void @@ -876,9 +917,16 @@ void lapic_handle_cmc(void) { + register_t tpr; + tpr = raise_tpr(IDT_CRITICAL); + enable_intr(); + lapic_eoi(); cmc_intr(); + + disable_intr(); + restore_tpr(tpr); } /* @@ -1359,11 +1407,19 @@ if (retval != 0) printf("%s: Failed to setup I/O APICs: returned %d\n", best_enum->apic_name, retval); - #ifdef XEN return; #endif + +#ifdef DEV_ATPIC /* + * Enable use of the TPR for interrupt blocking. The TPR is + * enabled by default if device atpic isn't present. + */ + enable_tpr(); +#endif + + /* * Finish setting up the local APIC on the BSP once we know how to * properly program the LINT pins. */ @@ -1445,7 +1501,7 @@ { register_t icrlo, destfield; - KASSERT((vector & ~APIC_VECTOR_MASK) == 0, + KASSERT((vector & ~APIC_VECTOR_MASK) == 0 || vector == IPI_STOP_HARD, ("%s: invalid vector %d", __func__, vector)); icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE;