--- //depot/vendor/freebsd/src/sys/conf/options.i386 2003/01/15 12:20:18 +++ //depot/user/peter/switch/conf/options.i386 2003/02/06 18:00:29 @@ -6,6 +6,7 @@ DISABLE_PSE opt_pmap.h PMAP_SHPGPERPROC opt_pmap.h DISABLE_PG_G opt_pmap.h +SWTCH_OPTIM_STATS opt_pmap.h PPC_PROBE_CHIPSET opt_ppc.h PPC_DEBUG opt_ppc.h MAXMEM --- //depot/vendor/freebsd/src/sys/i386/i386/machdep.c 2003/01/28 11:10:17 +++ //depot/user/peter/switch/i386/i386/machdep.c 2003/02/13 18:46:38 @@ -49,6 +49,7 @@ #include "opt_msgbuf.h" #include "opt_npx.h" #include "opt_perfmon.h" +#include "opt_pmap.h" #include "opt_kstack_pages.h" #include @@ -151,11 +152,32 @@ u_int atdevbase; #if defined(SWTCH_OPTIM_STATS) -extern int swtch_optim_stats; +int stupid_switch; +SYSCTL_INT(_debug, OID_AUTO, stupid_switch, + CTLFLAG_RW, &stupid_switch, 0, ""); +int swtch_optim_stats; SYSCTL_INT(_debug, OID_AUTO, swtch_optim_stats, - CTLFLAG_RD, &swtch_optim_stats, 0, ""); + CTLFLAG_RW, &swtch_optim_stats, 0, ""); +int tlb_flush_count; SYSCTL_INT(_debug, OID_AUTO, tlb_flush_count, - CTLFLAG_RD, &tlb_flush_count, 0, ""); + CTLFLAG_RW, &tlb_flush_count, 0, ""); +int lazy_flush_count; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_count, + CTLFLAG_RW, &lazy_flush_count, 0, ""); +int lazy_flush_fixup; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_fixup, + CTLFLAG_RW, &lazy_flush_fixup, 0, ""); +#ifdef SMP +int lazy_flush_smpfixup; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpfixup, + CTLFLAG_RW, &lazy_flush_smpfixup, 0, ""); +int lazy_flush_smpipi; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpipi, + CTLFLAG_RW, &lazy_flush_smpipi, 0, ""); +#endif +int lazy_flush_enable; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_enable, + CTLFLAG_RW, &lazy_flush_enable, 0, ""); #endif int cold = 1; --- //depot/vendor/freebsd/src/sys/i386/i386/mp_machdep.c 2003/02/03 09:55:26 +++ //depot/user/peter/switch/i386/i386/mp_machdep.c 2003/02/12 12:00:38 @@ -2595,7 +2595,7 @@ binuptime(PCPU_PTR(switchtime)); PCPU_SET(switchticks, ticks); - cpu_throw(); /* doesn't return */ + cpu_throw(NULL, choosethread()); /* doesn't return */ panic("scheduler returned us to %s", __func__); } --- //depot/vendor/freebsd/src/sys/i386/i386/pmap.c 2003/02/11 20:36:04 +++ //depot/user/peter/switch/i386/i386/pmap.c 2003/02/20 19:06:21 @@ -1482,6 +1482,31 @@ * Pmap allocation/deallocation routines. ***************************************************/ +#ifdef SMP +static void +smp_pmap_release(void *arg) +{ +#ifdef SWTCH_OPTIM_STATSx + u_int cr3; +#endif + pmap_t pmap = (pmap_t)arg; + u_int mymask = PCPU_GET(cpumask); + + if (1 || pmap->pm_active & mymask) { +#ifdef SWTCH_OPTIM_STATSx + cr3 = vtophys(pmap->pm_pdir); + if (rcr3() == cr3) + lazy_flush_smpmiss++; +#endif + load_cr3(PCPU_GET(curpcb)->pcb_cr3); + atomic_clear_int(&pmap->pm_active, mymask); +#ifdef SWTCH_OPTIM_STATS + atomic_add_int(&lazy_flush_smpfixup, 1); +#endif + } +} +#endif + /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. @@ -1493,12 +1518,48 @@ vm_page_t p,n,ptdpg; vm_object_t object = pmap->pm_pteobj; int curgeneration; + u_int cr3; #if defined(DIAGNOSTIC) if (object->ref_count != 1) panic("pmap_release: pteobj reference count != 1"); #endif + mtx_lock_spin(&sched_lock); /* protect pm_active */ + cr3 = vtophys(pmap->pm_pdir); + if (cr3 == rcr3()) { + printf("lazy free cleanup\n"); + load_cr3(PCPU_GET(curpcb)->pcb_cr3); + pmap->pm_active &= ~(PCPU_GET(cpumask)); +#ifdef SWTCH_OPTIM_STATS + lazy_flush_fixup++; +#endif + } + if (pmap->pm_active != 0) { + static time_t whined; +#ifdef SMP + smp_rendezvous(NULL, smp_pmap_release, NULL, pmap); +#ifdef SWTCH_OPTIM_STATS + lazy_flush_smpipi++; +#endif + if (pmap->pm_active != 0 && whined != time_second) { +#else + /* not a problem for SMP, remove */ + if (whined != time_second) { +#endif + u_int mask, active; + whined = time_second; + active = pmap->pm_active; + mask = PCPU_GET(cpumask); + mtx_unlock_spin(&sched_lock); /* protect pm_active */ + printf("failed to clear pm_active: 0x%x, I am 0x%x\n", active, mask); + } else { + mtx_unlock_spin(&sched_lock); /* protect pm_active */ + } + } else { + mtx_unlock_spin(&sched_lock); /* protect pm_active */ + } + ptdpg = NULL; mtx_lock_spin(&allpmaps_lock); LIST_REMOVE(pmap, pm_list); --- //depot/vendor/freebsd/src/sys/i386/i386/support.s 2002/09/21 21:50:57 +++ //depot/user/peter/switch/i386/i386/support.s 2003/02/06 18:00:29 @@ -34,6 +34,7 @@ */ #include "opt_npx.h" +#include "opt_pmap.h" #include #include --- //depot/vendor/freebsd/src/sys/i386/i386/swtch.s 2003/01/22 17:05:18 +++ //depot/user/peter/switch/i386/i386/swtch.s 2003/02/13 18:46:59 @@ -37,30 +37,16 @@ */ #include "opt_npx.h" +#include "opt_pmap.h" #include -#ifdef SMP -#include -#include /* CHEAP_TPR, GRAB_LOPRIO */ -#endif - #include "assym.s" /*****************************************************************************/ /* Scheduling */ /*****************************************************************************/ - .data - - .globl panic - -#ifdef SWTCH_OPTIM_STATS - .globl swtch_optim_stats, tlb_flush_count -swtch_optim_stats: .long 0 /* number of _swtch_optims */ -tlb_flush_count: .long 0 -#endif - .text /* @@ -71,27 +57,46 @@ * about its state. */ ENTRY(cpu_throw) + movl PCPU(CPUID), %esi + movl 4(%esp),%ecx /* Old thread */ + testl %ecx,%ecx /* no thread? */ + jz 1f + /* release bit from old pm_active */ + movl TD_PROC(%ecx), %eax /* thread->td_proc */ + movl P_VMSPACE(%eax), %ebx /* proc->p_vmspace */ + btrl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* clear old */ +1: + movl 8(%esp),%ecx /* New thread */ + movl TD_PCB(%ecx),%edx +#ifdef SWTCH_OPTIM_STATS + incl tlb_flush_count +#endif + movl PCB_CR3(%edx),%eax + movl %eax,%cr3 /* new address space */ + /* set bit in new pm_active */ + movl TD_PROC(%ecx),%eax + movl P_VMSPACE(%eax), %ebx + btsl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* set new */ jmp sw1 /* - * cpu_switch() + * cpu_switch(old, new) * * Save the current thread state, then select the next thread to run * and load its state. + * 0(%esp) = ret + * 4(%esp) = oldtd + * 8(%esp) = newtd */ ENTRY(cpu_switch) - /* Switch to new thread. First, save context as needed. */ - movl PCPU(CURTHREAD),%ecx + /* Switch to new thread. First, save context. */ + movl 4(%esp),%ecx - /* If no thread to save, don't save it (XXX shouldn't happen). */ - testl %ecx,%ecx - jz sw1 - - movl TD_PROC(%ecx), %eax - movl P_VMSPACE(%eax), %edx - movl PCPU(CPUID), %eax - btrl %eax, VM_PMAP+PM_ACTIVE(%edx) +#ifdef INVARIANTS + testl %ecx,%ecx /* no thread? */ + jz badsw2 /* no, panic */ +#endif movl TD_PCB(%ecx),%edx @@ -125,10 +130,6 @@ movl %eax,PCB_DR0(%edx) 1: -#ifdef SMP - /* XXX FIXME: we should be saving the local APIC TPR */ -#endif - #ifdef DEV_NPX /* have we used fp, and need a save? */ cmpl %ecx,PCPU(FPCURTHREAD) @@ -140,56 +141,62 @@ 1: #endif - /* Save is done. Now choose a new thread. */ - /* XXX still trashing space above the old "Top Of Stack". */ -sw1: + /* Save is done. Now fire up new thread. Leave old vmspace. */ + movl %ecx,%edi + movl 8(%esp),%ecx /* New thread */ +#ifdef INVARIANTS + testl %ecx,%ecx /* no thread? */ + jz badsw3 /* no, panic */ +#endif + movl TD_PCB(%ecx),%edx + movl PCPU(CPUID), %esi -#ifdef SMP - /* - * Stop scheduling if smp_active has become zero (for rebooting) and - * we are not the BSP. - */ - cmpl $0,smp_active - jne 1f - cmpl $0,PCPU(CPUID) + /* switch address space */ + movl PCB_CR3(%edx),%eax +#ifdef SWTCH_OPTIM_STATS + cmpl $0,lazy_flush_enable je 1f - movl PCPU(IDLETHREAD), %eax - jmp sw1b +#endif + cmpl %eax,IdlePTD /* Kernel address space? */ + je 3f 1: + movl %cr3,%ebx /* The same address space? */ + cmpl %ebx,%eax + je 2f /* Yes, skip all that cruft */ + +#ifdef SWTCH_OPTIM_STATS + incl tlb_flush_count #endif + movl %eax,%cr3 /* new address space */ - /* - * Choose a new thread to schedule. choosethread() returns idlethread - * if it cannot find another thread to run. - */ - call choosethread /* Trash ecx, edx; ret eax. */ + /* Release bit from old pmap->pm_active */ + movl TD_PROC(%edi), %eax /* oldproc */ + movl P_VMSPACE(%eax), %ebx + btrl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* clear old */ -#ifdef INVARIANTS - testl %eax,%eax /* no thread? */ - jz badsw3 /* no, panic */ -#endif + /* Set bit in new pmap->pm_active */ + movl TD_PROC(%ecx),%eax /* newproc */ + movl P_VMSPACE(%eax), %ebx + btsl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* set new */ -sw1b: - movl %eax,%ecx - movl TD_PCB(%ecx),%edx + jmp sw1 +2: /* same address space */ #ifdef SWTCH_OPTIM_STATS incl swtch_optim_stats #endif + jmp sw1 - /* switch address space */ - movl %cr3,%ebx /* The same address space? */ - cmpl PCB_CR3(%edx),%ebx - je 4f /* Yes, skip all that cruft */ +3: /* kernel address space */ #ifdef SWTCH_OPTIM_STATS - decl swtch_optim_stats - incl tlb_flush_count + incl lazy_flush_count #endif - movl PCB_CR3(%edx),%ebx /* Tell the CPU about the */ - movl %ebx,%cr3 /* new address space */ -4: - movl PCPU(CPUID), %esi +sw1: + /* + * At this point, we've switched address spaces and are ready + * to load up the rest of the next context. + */ cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ je 1f /* If not, use the default */ btsl %esi, private_tss /* mark use of private tss */ @@ -221,11 +228,6 @@ movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ ltr %si 3: - /* Note in vmspace that this cpu is using it. */ - movl TD_PROC(%ecx),%eax - movl P_VMSPACE(%eax), %ebx - movl PCPU(CPUID), %eax - btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) /* Restore context. */ movl PCB_EBX(%edx),%ebx @@ -241,10 +243,6 @@ movl %edx, PCPU(CURPCB) movl %ecx, PCPU(CURTHREAD) /* into next thread */ -#ifdef SMP - /* XXX FIXME: we should be restoring the local APIC TPR */ -#endif - /* * Determine the LDT to use and load it if is the default one and * that is not the current one. @@ -301,12 +299,23 @@ ret #ifdef INVARIANTS +badsw1: + pushal + pushl $sw0_1 + call panic +sw0_1: .asciz "cpu_throw: no newthread supplied" + +badsw2: + pushal + pushl $sw0_2 + call panic +sw0_2: .asciz "cpu_switch: no curthread supplied" + badsw3: pushal pushl $sw0_3 call panic - -sw0_3: .asciz "cpu_switch: choosethread returned NULL" +sw0_3: .asciz "cpu_switch: no newthread supplied" #endif /* --- //depot/vendor/freebsd/src/sys/i386/include/md_var.h 2003/01/22 10:20:17 +++ //depot/user/peter/switch/i386/include/md_var.h 2003/02/13 18:46:38 @@ -65,6 +65,18 @@ #ifdef COMPAT_43 extern int szosigcode; #endif +#ifdef SWTCH_OPTIM_STATS +extern int stupid_switch; +extern int swtch_optim_stats; +extern int tlb_flush_count; +extern int lazy_flush_count; +extern int lazy_flush_fixup; +#ifdef SMP +extern int lazy_flush_smpfixup; +extern int lazy_flush_smpipi; +#endif +extern int lazy_flush_enable; +#endif typedef void alias_for_inthand_t(u_int cs, u_int ef, u_int esp, u_int ss); struct thread; --- //depot/vendor/freebsd/src/sys/kern/kern_switch.c 2003/02/01 04:20:38 +++ //depot/user/peter/switch/kern/kern_switch.c 2003/02/13 16:48:10 @@ -98,6 +98,9 @@ #include #include #include +#ifdef SMP +#include +#endif #include CTASSERT((RQB_BPW * RQB_LEN) == RQ_NQS); @@ -122,6 +125,13 @@ struct thread *td; struct ksegrp *kg; +#ifdef SMP + if (smp_active == 0 && PCPU_GET(cpuid) != 0) { + /* Shutting down, run idlethread on AP's */ + goto do_idle; + } +#endif + retry: if ((ke = sched_choose())) { td = ke->ke_thread; @@ -138,6 +148,9 @@ CTR2(KTR_RUNQ, "choosethread: td=%p pri=%d", td, td->td_priority); } else { +#ifdef SMP +do_idle: +#endif /* Simulate runq_choose() having returned the idle thread */ td = PCPU_GET(idlethread); ke = td->td_kse; --- //depot/vendor/freebsd/src/sys/kern/kern_synch.c 2002/12/27 17:25:20 +++ //depot/user/peter/switch/kern/kern_synch.c 2003/02/06 18:00:29 @@ -41,6 +41,7 @@ #include "opt_ddb.h" #include "opt_ktrace.h" +#include "opt_pmap.h" #include #include @@ -67,6 +68,9 @@ #endif #include +#ifdef SWTCH_OPTIM_STATS +#include +#endif static void sched_setup(void *dummy); SYSINIT(sched_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, sched_setup, NULL) @@ -446,13 +450,18 @@ mi_switch(void) { struct bintime new_switchtime; - struct thread *td = curthread; /* XXX */ - struct proc *p = td->td_proc; /* XXX */ - struct kse *ke = td->td_kse; + struct thread *td; +#if defined(__i386__) + struct thread *newtd; +#endif + struct proc *p; + struct kse *ke; u_int sched_nest; mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED); - + td = curthread; /* XXX */ + p = td->td_proc; /* XXX */ + ke = td->td_kse; KASSERT(!TD_ON_RUNQ(td), ("mi_switch: called by old code")); #ifdef INVARIANTS if (!TD_ON_LOCK(td) && @@ -502,7 +511,28 @@ sched_nest = sched_lock.mtx_recurse; sched_switchout(td); +#if defined(__i386__) + newtd = choosethread(); + if (td == newtd) { +#if 0 + static int lastwarn; + static int hits; + + hits++; + if (lastwarn != time_second) { + printf("newtd == td, count = %d\n", hits); + lastwarn = time_second; + } +#else +#ifdef SWTCH_OPTIM_STATS + stupid_switch++; +#endif +#endif + } + cpu_switch(td, newtd); /* SHAZAM!! */ +#else cpu_switch(); /* SHAZAM!!*/ +#endif sched_lock.mtx_recurse = sched_nest; sched_lock.mtx_lock = (uintptr_t)td; --- //depot/vendor/freebsd/src/sys/kern/kern_thread.c 2003/02/01 04:20:38 +++ //depot/user/peter/switch/kern/kern_thread.c 2003/02/20 19:06:21 @@ -1051,7 +1051,13 @@ } else { PROC_UNLOCK(p); } + mtx_assert(&sched_lock, MA_OWNED); +#if defined(__i386__) + cpu_switch(td, choosethread()); + panic("I'm a teapot!"); +#else cpu_throw(); +#endif /* NOTREACHED */ } --- //depot/vendor/freebsd/src/sys/sys/proc.h 2003/02/09 00:55:02 +++ //depot/user/peter/switch/sys/proc.h 2003/02/12 12:08:37 @@ -902,8 +902,13 @@ void sleepinit(void); void stopevent(struct proc *, u_int, u_int); void cpu_idle(void); +#if defined(__i386__) +void cpu_switch(struct thread *old, struct thread *new); +void cpu_throw(struct thread *old, struct thread *new) __dead2; +#else void cpu_switch(void); void cpu_throw(void) __dead2; +#endif void unsleep(struct thread *); void userret(struct thread *, struct trapframe *, u_int);