Index: conf/options.i386 =========================================================================== --- conf/options.i386 2003/04/02 14:40:12 #38 +++ conf/options.i386 2003/04/02 14:40:12 *************** *** 6,11 **** --- 6,13 ---- DISABLE_PSE opt_pmap.h PMAP_SHPGPERPROC opt_pmap.h DISABLE_PG_G opt_pmap.h + LAZY_SWITCH opt_swtch.h + SWTCH_OPTIM_STATS opt_swtch.h PPC_PROBE_CHIPSET opt_ppc.h PPC_DEBUG opt_ppc.h MAXMEM Index: i386/i386/machdep.c =========================================================================== --- i386/i386/machdep.c 2003/04/02 14:40:12 #98 +++ i386/i386/machdep.c 2003/04/02 14:40:12 *************** *** 49,54 **** --- 49,55 ---- #include "opt_msgbuf.h" #include "opt_npx.h" #include "opt_perfmon.h" + #include "opt_swtch.h" #include "opt_kstack_pages.h" #include *************** *** 151,161 **** u_int atdevbase; #if defined(SWTCH_OPTIM_STATS) ! extern int swtch_optim_stats; SYSCTL_INT(_debug, OID_AUTO, swtch_optim_stats, ! CTLFLAG_RD, &swtch_optim_stats, 0, ""); SYSCTL_INT(_debug, OID_AUTO, tlb_flush_count, ! CTLFLAG_RD, &tlb_flush_count, 0, ""); #endif int cold = 1; --- 152,191 ---- u_int atdevbase; #if defined(SWTCH_OPTIM_STATS) ! int stupid_switch; ! SYSCTL_INT(_debug, OID_AUTO, stupid_switch, ! CTLFLAG_RW, &stupid_switch, 0, ""); ! int swtch_optim_stats; SYSCTL_INT(_debug, OID_AUTO, swtch_optim_stats, ! CTLFLAG_RW, &swtch_optim_stats, 0, ""); ! int tlb_flush_count; SYSCTL_INT(_debug, OID_AUTO, tlb_flush_count, ! CTLFLAG_RW, &tlb_flush_count, 0, ""); ! int lazy_flush_count; ! SYSCTL_INT(_debug, OID_AUTO, lazy_flush_count, ! CTLFLAG_RW, &lazy_flush_count, 0, ""); ! int lazy_flush_fixup; ! SYSCTL_INT(_debug, OID_AUTO, lazy_flush_fixup, ! CTLFLAG_RW, &lazy_flush_fixup, 0, ""); ! #ifdef SMP ! int lazy_flush_smpfixup; ! SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpfixup, ! CTLFLAG_RW, &lazy_flush_smpfixup, 0, ""); ! int lazy_flush_smpipi; ! SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpipi, ! CTLFLAG_RW, &lazy_flush_smpipi, 0, ""); ! int lazy_flush_smpbadcr3; ! SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpbadcr3, ! CTLFLAG_RW, &lazy_flush_smpbadcr3, 0, ""); ! int lazy_flush_smpmiss; ! SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpmiss, ! CTLFLAG_RW, &lazy_flush_smpmiss, 0, ""); ! #endif ! #endif ! #ifdef LAZY_SWITCH ! int lazy_flush_enable = 1; ! SYSCTL_INT(_debug, OID_AUTO, lazy_flush_enable, ! CTLFLAG_RW, &lazy_flush_enable, 0, ""); #endif int cold = 1; Index: i386/i386/mp_machdep.c =========================================================================== --- i386/i386/mp_machdep.c 2003/04/02 14:40:12 #47 +++ i386/i386/mp_machdep.c 2003/04/02 14:40:12 *************** *** 27,32 **** --- 27,33 ---- #include "opt_cpu.h" #include "opt_kstack_pages.h" + #include "opt_swtch.h" #ifdef SMP #include *************** *** 634,639 **** --- 635,646 ---- setidt(XSTATCLOCK_OFFSET, Xstatclock, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + #ifdef LAZY_SWITCH + /* install an inter-CPU IPI for lazy pmap release */ + setidt(XLAZYPMAP_OFFSET, Xlazypmap, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + #endif + /* install an inter-CPU IPI for all-CPU rendezvous */ setidt(XRENDEZVOUS_OFFSET, Xrendezvous, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); *************** *** 2598,2604 **** binuptime(PCPU_PTR(switchtime)); PCPU_SET(switchticks, ticks); ! cpu_throw(); /* doesn't return */ panic("scheduler returned us to %s", __func__); } --- 2605,2611 ---- binuptime(PCPU_PTR(switchtime)); PCPU_SET(switchticks, ticks); ! cpu_throw(NULL, choosethread()); /* doesn't return */ panic("scheduler returned us to %s", __func__); } Index: i386/i386/pmap.c =========================================================================== --- i386/i386/pmap.c 2003/04/02 14:40:12 #115 +++ i386/i386/pmap.c 2003/04/02 14:40:12 *************** *** 102,107 **** --- 102,108 ---- #include "opt_pmap.h" #include "opt_msgbuf.h" #include "opt_kstack_pages.h" + #include "opt_swtch.h" #include #include *************** *** 184,189 **** --- 185,193 ---- LIST_HEAD(pmaplist, pmap); static struct pmaplist allpmaps; static struct mtx allpmaps_lock; + #if defined(SMP) && defined(LAZY_SWITCH) + static struct mtx lazypmap_lock; + #endif vm_paddr_t avail_start; /* PA of first available physical page */ vm_paddr_t avail_end; /* PA of last available physical page */ *************** *** 336,341 **** --- 340,348 ---- kernel_pmap->pm_active = -1; /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvlist); LIST_INIT(&allpmaps); + #if defined(SMP) && defined(LAZY_SWITCH) + mtx_init(&lazypmap_lock, "lazypmap", NULL, MTX_SPIN); + #endif mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN); mtx_lock_spin(&allpmaps_lock); LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list); *************** *** 1486,1492 **** --- 1493,1614 ---- * Pmap allocation/deallocation routines. ***************************************************/ + #ifdef LAZY_SWITCH + #ifdef SMP /* + * Deal with a SMP shootdown of other users of the pmap that we are + * trying to dispose of. This can be a bit hairy. + */ + static u_int *lazymask; + static u_int lazyptd; + static volatile u_int lazywait; + + void pmap_lazyfix_action(void); + + void + pmap_lazyfix_action(void) + { + u_int mymask = PCPU_GET(cpumask); + + if (rcr3() == lazyptd) { + load_cr3(PCPU_GET(curpcb)->pcb_cr3); + #ifdef SWTCH_OPTIM_STATS + atomic_add_int(&lazy_flush_smpfixup, 1); + } else { + if (*lazymask & mymask) + lazy_flush_smpbadcr3++; + else + lazy_flush_smpmiss++; + #endif + } + atomic_clear_int(lazymask, mymask); + atomic_store_rel_int(&lazywait, 1); + } + + static void + pmap_lazyfix_self(u_int mymask) + { + + if (rcr3() == lazyptd) { + load_cr3(PCPU_GET(curpcb)->pcb_cr3); + #ifdef SWTCH_OPTIM_STATS + lazy_flush_fixup++; + } else { + if (*lazymask & mymask) + lazy_flush_smpbadcr3++; + else + lazy_flush_smpmiss++; + #endif + } + atomic_clear_int(lazymask, mymask); + } + + + static void + pmap_lazyfix(pmap_t pmap) + { + u_int mymask = PCPU_GET(cpumask); + u_int mask; + register u_int spins; + + while ((mask = pmap->pm_active) != 0) { + spins = 50000000; + mask = mask & -mask; /* Find least significant set bit */ + mtx_lock_spin(&lazypmap_lock); + #ifdef PAE + lazyptd = vtophys(pmap->pm_pdpt); + #else + lazyptd = vtophys(pmap->pm_pdir); + #endif + if (mask == mymask) { + lazymask = &pmap->pm_active; + pmap_lazyfix_self(mymask); + } else { + atomic_store_rel_int((u_int *)&lazymask, + (u_int)&pmap->pm_active); + atomic_store_rel_int(&lazywait, 0); + ipi_selected(mask, IPI_LAZYPMAP); + while (lazywait == 0) { + ia32_pause(); + if (--spins == 0) + break; + } + #ifdef SWTCH_OPTIM_STATS + lazy_flush_smpipi++; + #endif + } + mtx_unlock_spin(&lazypmap_lock); + if (spins == 0) + printf("pmap_lazyfix: spun for 50000000\n"); + } + } + + #else /* SMP */ + + /* + * Cleaning up on uniprocessor is easy. For various reasons, we're + * unlikely to have to even execute this code, including the fact + * that the cleanup is deferred until the parent does a wait(2), which + * means that another userland process has run. + */ + static void + pmap_lazyfix(pmap_t pmap) + { + u_int cr3; + + cr3 = vtophys(pmap->pm_pdir); + if (cr3 == rcr3()) { + load_cr3(PCPU_GET(curpcb)->pcb_cr3); + pmap->pm_active &= ~(PCPU_GET(cpumask)); + #ifdef SWTCH_OPTIM_STATS + lazy_flush_fixup++; + #endif + } + } + #endif /* SMP */ + #endif /* LAZY_SWITCH */ + + /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. *************** *** 1507,1512 **** --- 1629,1637 ---- ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); + #ifdef LAZY_SWITCH + pmap_lazyfix(pmap); + #endif mtx_lock_spin(&allpmaps_lock); LIST_REMOVE(pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); *************** *** 3321,3329 **** pmap_t pmap; u_int32_t cr3; pmap = vmspace_pmap(td->td_proc->p_vmspace); #if defined(SMP) ! pmap->pm_active |= PCPU_GET(cpumask); #else pmap->pm_active |= 1; #endif --- 3446,3455 ---- pmap_t pmap; u_int32_t cr3; + critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); #if defined(SMP) ! atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask)); #else pmap->pm_active |= 1; #endif *************** *** 3348,3353 **** --- 3474,3480 ---- #ifdef SWTCH_OPTIM_STATS tlb_flush_count++; #endif + critical_exit(); } vm_offset_t Index: i386/i386/swtch.s =========================================================================== --- i386/i386/swtch.s 2003/04/02 14:40:12 #19 +++ i386/i386/swtch.s 2003/04/02 14:40:12 *************** *** 37,66 **** */ #include "opt_npx.h" #include - #ifdef SMP - #include - #include /* CHEAP_TPR, GRAB_LOPRIO */ - #endif - #include "assym.s" /*****************************************************************************/ /* Scheduling */ /*****************************************************************************/ - .data - - .globl panic - - #ifdef SWTCH_OPTIM_STATS - .globl swtch_optim_stats, tlb_flush_count - swtch_optim_stats: .long 0 /* number of _swtch_optims */ - tlb_flush_count: .long 0 - #endif - .text /* --- 37,52 ---- */ #include "opt_npx.h" + #include "opt_swtch.h" #include #include "assym.s" /*****************************************************************************/ /* Scheduling */ /*****************************************************************************/ .text /* *************** *** 68,97 **** * * This is the second half of cpu_swtch(). It is used when the current * thread is either a dummy or slated to die, and we no longer care ! * about its state. */ ENTRY(cpu_throw) jmp sw1 /* ! * cpu_switch() * * Save the current thread state, then select the next thread to run * and load its state. */ ENTRY(cpu_switch) ! /* Switch to new thread. First, save context as needed. */ ! movl PCPU(CURTHREAD),%ecx ! /* If no thread to save, don't save it (XXX shouldn't happen). */ ! testl %ecx,%ecx ! jz sw1 ! ! movl TD_PROC(%ecx), %eax ! movl P_VMSPACE(%eax), %edx ! movl PCPU(CPUID), %eax ! btrl %eax, VM_PMAP+PM_ACTIVE(%edx) movl TD_PCB(%ecx),%edx --- 54,113 ---- * * This is the second half of cpu_swtch(). It is used when the current * thread is either a dummy or slated to die, and we no longer care ! * about its state. This is only a slight optimization and is probably ! * not worth it anymore. Note that we need to clear the pm_active bits so ! * we do need the old proc if it still exists. ! * 0(%esp) = ret ! * 4(%esp) = oldtd ! * 8(%esp) = newtd */ ENTRY(cpu_throw) + movl PCPU(CPUID), %esi + movl 4(%esp),%ecx /* Old thread */ + testl %ecx,%ecx /* no thread? */ + jz 1f + /* release bit from old pm_active */ + movl TD_PROC(%ecx), %eax /* thread->td_proc */ + movl P_VMSPACE(%eax), %ebx /* proc->p_vmspace */ + #ifdef SMP + lock + #endif + btrl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* clear old */ + 1: + movl 8(%esp),%ecx /* New thread */ + movl TD_PCB(%ecx),%edx + #ifdef SWTCH_OPTIM_STATS + incl tlb_flush_count + #endif + movl PCB_CR3(%edx),%eax + movl %eax,%cr3 /* new address space */ + /* set bit in new pm_active */ + movl TD_PROC(%ecx),%eax + movl P_VMSPACE(%eax), %ebx + #ifdef SMP + lock + #endif + btsl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* set new */ jmp sw1 /* ! * cpu_switch(old, new) * * Save the current thread state, then select the next thread to run * and load its state. + * 0(%esp) = ret + * 4(%esp) = oldtd + * 8(%esp) = newtd */ ENTRY(cpu_switch) ! /* Switch to new thread. First, save context. */ ! movl 4(%esp),%ecx ! #ifdef INVARIANTS ! testl %ecx,%ecx /* no thread? */ ! jz badsw2 /* no, panic */ ! #endif movl TD_PCB(%ecx),%edx *************** *** 125,134 **** movl %eax,PCB_DR0(%edx) 1: - #ifdef SMP - /* XXX FIXME: we should be saving the local APIC TPR */ - #endif - #ifdef DEV_NPX /* have we used fp, and need a save? */ cmpl %ecx,PCPU(FPCURTHREAD) --- 141,146 ---- *************** *** 140,195 **** 1: #endif ! /* Save is done. Now choose a new thread. */ ! /* XXX still trashing space above the old "Top Of Stack". */ ! sw1: ! #ifdef SMP ! /* ! * Stop scheduling if smp_active has become zero (for rebooting) and ! * we are not the BSP. ! */ ! cmpl $0,smp_active ! jne 1f ! cmpl $0,PCPU(CPUID) je 1f ! movl PCPU(IDLETHREAD), %eax ! jmp sw1b 1: #endif ! /* ! * Choose a new thread to schedule. choosethread() returns idlethread ! * if it cannot find another thread to run. ! */ ! call choosethread /* Trash ecx, edx; ret eax. */ ! #ifdef INVARIANTS ! testl %eax,%eax /* no thread? */ ! jz badsw3 /* no, panic */ #endif ! sw1b: ! movl %eax,%ecx ! movl TD_PCB(%ecx),%edx #ifdef SWTCH_OPTIM_STATS incl swtch_optim_stats #endif - - /* switch address space */ - movl %cr3,%ebx /* The same address space? */ - cmpl PCB_CR3(%edx),%ebx - je 4f /* Yes, skip all that cruft */ - #ifdef SWTCH_OPTIM_STATS - decl swtch_optim_stats - incl tlb_flush_count #endif - movl PCB_CR3(%edx),%ebx /* Tell the CPU about the */ - movl %ebx,%cr3 /* new address space */ - 4: ! movl PCPU(CPUID), %esi cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ je 1f /* If not, use the default */ btsl %esi, private_tss /* mark use of private tss */ --- 152,227 ---- 1: #endif ! /* Save is done. Now fire up new thread. Leave old vmspace. */ ! movl %ecx,%edi ! movl 8(%esp),%ecx /* New thread */ ! #ifdef INVARIANTS ! testl %ecx,%ecx /* no thread? */ ! jz badsw3 /* no, panic */ ! #endif ! movl TD_PCB(%ecx),%edx ! movl PCPU(CPUID), %esi ! /* switch address space */ ! movl PCB_CR3(%edx),%eax ! #ifdef LAZY_SWITCH ! cmpl $0,lazy_flush_enable je 1f ! cmpl %eax,IdlePTD /* Kernel address space? */ ! #ifdef SWTCH_OPTIM_STATS ! je 3f ! #else ! je sw1 ! #endif 1: + movl %cr3,%ebx /* The same address space? */ + cmpl %ebx,%eax + #ifdef SWTCH_OPTIM_STATS + je 2f /* Yes, skip all that cruft */ + #else + je sw1 + #endif #endif ! #ifdef SWTCH_OPTIM_STATS ! incl tlb_flush_count ! #endif ! movl %eax,%cr3 /* new address space */ ! /* Release bit from old pmap->pm_active */ ! movl TD_PROC(%edi), %eax /* oldproc */ ! movl P_VMSPACE(%eax), %ebx ! #ifdef SMP ! lock #endif + btrl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* clear old */ ! /* Set bit in new pmap->pm_active */ ! movl TD_PROC(%ecx),%eax /* newproc */ ! movl P_VMSPACE(%eax), %ebx ! #ifdef SMP ! lock ! #endif ! btsl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* set new */ + #ifdef LAZY_SWITCH #ifdef SWTCH_OPTIM_STATS + jmp sw1 + + 2: /* same address space */ incl swtch_optim_stats + jmp sw1 + + 3: /* kernel address space */ + incl lazy_flush_count #endif #endif ! sw1: ! /* ! * At this point, we've switched address spaces and are ready ! * to load up the rest of the next context. ! */ cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ je 1f /* If not, use the default */ btsl %esi, private_tss /* mark use of private tss */ *************** *** 221,231 **** movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ ltr %si 3: - /* Note in vmspace that this cpu is using it. */ - movl TD_PROC(%ecx),%eax - movl P_VMSPACE(%eax), %ebx - movl PCPU(CPUID), %eax - btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) /* Restore context. */ movl PCB_EBX(%edx),%ebx --- 253,258 ---- *************** *** 241,250 **** movl %edx, PCPU(CURPCB) movl %ecx, PCPU(CURTHREAD) /* into next thread */ - #ifdef SMP - /* XXX FIXME: we should be restoring the local APIC TPR */ - #endif - /* * Determine the LDT to use and load it if is the default one and * that is not the current one. --- 268,273 ---- *************** *** 301,312 **** ret #ifdef INVARIANTS badsw3: pushal pushl $sw0_3 call panic ! ! sw0_3: .asciz "cpu_switch: choosethread returned NULL" #endif /* --- 324,346 ---- ret #ifdef INVARIANTS + badsw1: + pushal + pushl $sw0_1 + call panic + sw0_1: .asciz "cpu_throw: no newthread supplied" + + badsw2: + pushal + pushl $sw0_2 + call panic + sw0_2: .asciz "cpu_switch: no curthread supplied" + badsw3: pushal pushl $sw0_3 call panic ! sw0_3: .asciz "cpu_switch: no newthread supplied" #endif /* Index: i386/include/md_var.h =========================================================================== --- i386/include/md_var.h 2003/04/02 14:40:12 #21 +++ i386/include/md_var.h 2003/04/02 14:40:12 *************** *** 65,70 **** --- 65,86 ---- #ifdef COMPAT_43 extern int szosigcode; #endif + #ifdef SWTCH_OPTIM_STATS + extern int stupid_switch; + extern int swtch_optim_stats; + extern int tlb_flush_count; + extern int lazy_flush_count; + extern int lazy_flush_fixup; + #ifdef SMP + extern int lazy_flush_smpfixup; + extern int lazy_flush_smpipi; + extern int lazy_flush_smpbadcr3; + extern int lazy_flush_smpmiss; + #endif + #endif + #ifdef LAZY_SWITCH + extern int lazy_flush_enable; + #endif typedef void alias_for_inthand_t(u_int cs, u_int ef, u_int esp, u_int ss); struct thread; Index: i386/include/pmap.h =========================================================================== --- i386/include/pmap.h 2003/04/02 14:40:12 #24 +++ i386/include/pmap.h 2003/04/02 14:40:12 *************** *** 247,253 **** pd_entry_t *pm_pdir; /* KVA of page directory */ vm_object_t pm_pteobj; /* Container for pte's */ TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ ! int pm_active; /* active on cpus */ struct pmap_statistics pm_stats; /* pmap statistics */ LIST_ENTRY(pmap) pm_list; /* List of all pmaps */ #ifdef PAE --- 247,253 ---- pd_entry_t *pm_pdir; /* KVA of page directory */ vm_object_t pm_pteobj; /* Container for pte's */ TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ ! u_int pm_active; /* active on cpus */ struct pmap_statistics pm_stats; /* pmap statistics */ LIST_ENTRY(pmap) pm_list; /* List of all pmaps */ #ifdef PAE Index: i386/include/smp.h =========================================================================== --- i386/include/smp.h 2003/04/02 14:40:12 #11 +++ i386/include/smp.h 2003/04/02 14:40:12 *************** *** 56,61 **** --- 56,62 ---- #define IPI_INVLTLB XINVLTLB_OFFSET #define IPI_INVLPG XINVLPG_OFFSET #define IPI_INVLRNG XINVLRNG_OFFSET + #define IPI_LAZYPMAP XLAZYPMAP_OFFSET #define IPI_RENDEZVOUS XRENDEZVOUS_OFFSET #define IPI_AST XCPUAST_OFFSET #define IPI_STOP XCPUSTOP_OFFSET Index: i386/isa/apic_vector.s =========================================================================== --- i386/isa/apic_vector.s 2003/04/02 14:40:12 #16 +++ i386/isa/apic_vector.s 2003/04/02 14:40:12 *************** *** 3,8 **** --- 3,9 ---- * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.86 2003/02/03 17:53:14 jake Exp $ */ + #include "opt_swtch.h" #include #include *************** *** 648,654 **** --- 649,676 ---- POP_FRAME iret + #ifdef LAZY_SWITCH + /* + * Clean up when we lose out on the lazy context switch optimization. + * ie: when we are about to release a PTD but a cpu is still borrowing it. + */ + SUPERALIGN_TEXT + .globl Xlazypmap + Xlazypmap: + PUSH_FRAME + movl $KDSEL, %eax + mov %ax, %ds /* use KERNEL data segment */ + mov %ax, %es + movl $KPSEL, %eax + mov %ax, %fs + + call pmap_lazyfix_action + movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + POP_FRAME + iret + #endif + .data .globl apic_pin_trigger Index: i386/isa/intr_machdep.h =========================================================================== --- i386/isa/intr_machdep.h 2003/04/02 14:40:12 #13 +++ i386/isa/intr_machdep.h 2003/04/02 14:40:12 *************** *** 116,121 **** --- 116,124 ---- /* inter-CPU rendezvous */ #define XRENDEZVOUS_OFFSET (ICU_OFFSET + 122) /* 0x9A */ + /* lazy pmap release */ + #define XLAZYPMAP_OFFSET (ICU_OFFSET + 123) /* 0x9B */ + /* IPI to generate an additional software trap at the target CPU */ /* XXX in the middle of the interrupt range, overlapping IRQ48 */ #define XCPUAST_OFFSET (ICU_OFFSET + 48) /* 0x50 */ *************** *** 206,212 **** Xcpuast, /* Additional software trap on other cpu */ Xcpustop, /* CPU stops & waits for another CPU to restart it */ Xspuriousint, /* handle APIC "spurious INTs" */ ! Xrendezvous; /* handle CPU rendezvous */ #ifdef TEST_TEST1 inthand_t --- 209,216 ---- Xcpuast, /* Additional software trap on other cpu */ Xcpustop, /* CPU stops & waits for another CPU to restart it */ Xspuriousint, /* handle APIC "spurious INTs" */ ! Xrendezvous, /* handle CPU rendezvous */ ! Xlazypmap; /* handle lazy pmap release */ #ifdef TEST_TEST1 inthand_t Index: kern/kern_switch.c =========================================================================== --- kern/kern_switch.c 2003/04/02 14:40:12 #41 +++ kern/kern_switch.c 2003/04/02 14:40:12 *************** *** 98,103 **** --- 98,106 ---- #include #include #include + #if defined(SMP) && defined(__i386__) + #include + #endif #include CTASSERT((RQB_BPW * RQB_LEN) == RQ_NQS); *************** *** 122,129 **** struct thread *td; struct ksegrp *kg; retry: ! if ((ke = sched_choose())) { td = ke->ke_thread; KASSERT((td->td_kse == ke), ("kse/thread mismatch")); kg = ke->ke_ksegrp; --- 125,145 ---- struct thread *td; struct ksegrp *kg; + #if defined(SMP) && defined(__i386__) + if (smp_active == 0 && PCPU_GET(cpuid) != 0) { + /* Shutting down, run idlethread on AP's */ + td = PCPU_GET(idlethread); + ke = td->td_kse; + CTR1(KTR_RUNQ, "choosethread: td=%p (idle)", td); + ke->ke_flags |= KEF_DIDRUN; + TD_SET_RUNNING(td); + return (td); + } + #endif + retry: ! ke = sched_choose(); ! if (ke) { td = ke->ke_thread; KASSERT((td->td_kse == ke), ("kse/thread mismatch")); kg = ke->ke_ksegrp; Index: kern/kern_synch.c =========================================================================== --- kern/kern_synch.c 2003/04/02 14:40:12 #67 +++ kern/kern_synch.c 2003/04/02 14:40:12 *************** *** 41,46 **** --- 41,49 ---- #include "opt_ddb.h" #include "opt_ktrace.h" + #ifdef __i386__ + #include "opt_swtch.h" + #endif #include #include *************** *** 67,72 **** --- 70,78 ---- #endif #include + #ifdef SWTCH_OPTIM_STATS + #include + #endif static void sched_setup(void *dummy); SYSINIT(sched_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, sched_setup, NULL) *************** *** 449,460 **** mi_switch(void) { struct bintime new_switchtime; ! struct thread *td = curthread; /* XXX */ ! struct proc *p = td->td_proc; /* XXX */ u_int sched_nest; mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED); ! KASSERT(!TD_ON_RUNQ(td), ("mi_switch: called by old code")); #ifdef INVARIANTS if (!TD_ON_LOCK(td) && --- 455,470 ---- mi_switch(void) { struct bintime new_switchtime; ! struct thread *td; ! #if defined(__i386__) || defined(__sparc64__) ! struct thread *newtd; ! #endif ! struct proc *p; u_int sched_nest; mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED); ! td = curthread; /* XXX */ ! p = td->td_proc; /* XXX */ KASSERT(!TD_ON_RUNQ(td), ("mi_switch: called by old code")); #ifdef INVARIANTS if (!TD_ON_LOCK(td) && *************** *** 506,512 **** --- 516,532 ---- thread_switchout(td); sched_switchout(td); + #if defined(__i386__) || defined(__sparc64__) + newtd = choosethread(); + if (td != newtd) + cpu_switch(td, newtd); /* SHAZAM!! */ + #ifdef SWTCH_OPTIM_STATS + else + stupid_switch++; + #endif + #else cpu_switch(); /* SHAZAM!!*/ + #endif sched_lock.mtx_recurse = sched_nest; sched_lock.mtx_lock = (uintptr_t)td; Index: kern/kern_thr.c =========================================================================== --- kern/kern_thr.c 2003/04/02 14:40:12 #2 +++ kern/kern_thr.c 2003/04/02 14:40:12 *************** *** 106,112 **** --- 106,116 ---- td->td_last_kse = NULL; thread_stash(td); + #if defined(__i386__) || defined(__sparc64__) + cpu_throw(td, choosethread()); + #else cpu_throw(); + #endif } #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) Index: kern/kern_thread.c =========================================================================== --- kern/kern_thread.c 2003/04/02 14:40:12 #109 +++ kern/kern_thread.c 2003/04/02 14:40:12 *************** *** 1250,1256 **** --- 1250,1262 ---- PROC_UNLOCK(p); } /* XXX Shouldn't cpu_throw() here. */ + mtx_assert(&sched_lock, MA_OWNED); + #if defined(__i386__) || defined(__sparc64__) + cpu_throw(td, choosethread()); + #else cpu_throw(); + #endif + panic("I'm a teapot!"); /* NOTREACHED */ } Index: kern/subr_witness.c =========================================================================== --- kern/subr_witness.c 2003/04/02 14:40:12 #70 +++ kern/subr_witness.c 2003/04/02 14:40:12 *************** *** 84,89 **** --- 84,92 ---- #include "opt_ddb.h" #include "opt_witness.h" + #ifdef __i386__ + #include "opt_swtch.h" + #endif #include #include *************** *** 295,300 **** --- 298,306 ---- #if defined(__i386__) && defined(APIC_IO) { "tlb", &lock_class_mtx_spin }, #endif + #if defined(__i386__) && defined(LAZY_SWITCH) + { "lazypmap", &lock_class_mtx_spin }, + #endif #ifdef __sparc64__ { "ipi", &lock_class_mtx_spin }, #endif Index: sparc64/sparc64/mp_machdep.c =========================================================================== --- sparc64/sparc64/mp_machdep.c 2003/04/02 14:40:12 #19 +++ sparc64/sparc64/mp_machdep.c 2003/04/02 14:40:12 *************** *** 357,363 **** /* ok, now grab sched_lock and enter the scheduler */ mtx_lock_spin(&sched_lock); ! cpu_throw(); /* doesn't return */ } void --- 357,363 ---- /* ok, now grab sched_lock and enter the scheduler */ mtx_lock_spin(&sched_lock); ! cpu_throw(NULL, choosethread()); /* doesn't return */ } void Index: sparc64/sparc64/swtch.S =========================================================================== --- sparc64/sparc64/swtch.S 2003/04/02 14:40:12 #6 +++ sparc64/sparc64/swtch.S 2003/04/02 14:40:12 *************** *** 36,66 **** #include "assym.s" ENTRY(cpu_throw) save %sp, -CCFSZ, %sp - call choosethread - ldx [PCPU(CURTHREAD)], %l0 flushw ! b,a %xcc, .Lsw1 ! nop END(cpu_throw) ENTRY(cpu_switch) - /* - * Choose a new thread. If its the same as the current one, do - * nothing. - */ save %sp, -CCFSZ, %sp ! call choosethread ! ldx [PCPU(CURTHREAD)], %l0 ! cmp %l0, %o0 ! be,a,pn %xcc, 4f ! nop ! ldx [%l0 + TD_PCB], %l1 /* * If the current thread was using floating point, save its context. */ ldx [%l0 + TD_FRAME], %l2 ldx [%l2 + TF_FPRS], %l3 andcc %l3, FPRS_FEF, %g0 --- 36,64 ---- #include "assym.s" + /* + * void cpu_throw(struct thread *old, struct thread *new) + */ ENTRY(cpu_throw) save %sp, -CCFSZ, %sp flushw ! mov %i0, %l0 ! ba %xcc, .Lsw1 ! mov %i1, %o0 END(cpu_throw) + /* + * void cpu_switch(struct thread *old, struct thread *new) + */ ENTRY(cpu_switch) save %sp, -CCFSZ, %sp ! mov %i0, %l0 ! mov %i1, %o0 /* * If the current thread was using floating point, save its context. */ + ldx [%l0 + TD_PCB], %l1 ldx [%l0 + TD_FRAME], %l2 ldx [%l2 + TF_FPRS], %l3 andcc %l3, FPRS_FEF, %g0 Index: sys/proc.h =========================================================================== --- sys/proc.h 2003/04/02 14:40:12 #138 +++ sys/proc.h 2003/04/02 14:40:12 *************** *** 889,896 **** --- 889,901 ---- void sleepinit(void); void stopevent(struct proc *, u_int, u_int); void cpu_idle(void); + #if defined(__i386__) || defined(__sparc64__) + void cpu_switch(struct thread *old, struct thread *new); + void cpu_throw(struct thread *old, struct thread *new) __dead2; + #else void cpu_switch(void); void cpu_throw(void) __dead2; + #endif void unsleep(struct thread *); void userret(struct thread *, struct trapframe *, u_int);