==== //depot/projects/hammer/sys/amd64/amd64/apic_vector.S#34 - /home/src/sys/amd64/amd64/apic_vector.S ==== *************** *** 169,176 **** movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ lock ! incl smp_tlb_wait popq %rax iretq --- 169,178 ---- movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + XXX we have to use swapgs etc first!!! + movl PCPU(CPUMASK), %eax lock ! andl %eax, smp_tlb_wait popq %rax iretq *************** *** 189,196 **** movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ lock ! incl smp_tlb_wait popq %rax iretq --- 191,200 ---- movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + XXX we have to use swapgs etc first!!! + movl PCPU(CPUMASK), %eax lock ! andl %eax, smp_tlb_wait popq %rax iretq *************** *** 214,221 **** movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ lock ! incl smp_tlb_wait popq %rdx popq %rax --- 218,227 ---- movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + XXX we have to use swapgs etc first!!! + movl PCPU(CPUMASK), %eax lock ! andl %eax, smp_tlb_wait popq %rdx popq %rax ==== //depot/projects/hammer/sys/amd64/amd64/cpu_switch.S#35 - /home/src/sys/amd64/amd64/cpu_switch.S ==== *************** *** 109,115 **** --- 109,119 ---- movl %es,PCB_ES(%r8) movl %fs,PCB_FS(%r8) movl %gs,PCB_GS(%r8) + jmp 2f 1: + movq PCB_FSBASE(%r8),%r9 + movq PCB_GSBASE(%r8),%r10 + 2: /* Test if debug registers should be saved. */ testl $PCB_DBREGS,PCB_FLAGS(%r8) jz 1f /* no, skip over */ *************** *** 185,198 **** wrmsr jmp 2f 1: - /* Restore userland %fs */ movl $MSR_FSBASE,%ecx movl PCB_FSBASE(%r8),%eax movl PCB_FSBASE+4(%r8),%edx wrmsr /* Restore userland %gs */ movl $MSR_KGSBASE,%ecx movl PCB_GSBASE(%r8),%eax movl PCB_GSBASE+4(%r8),%edx --- 189,206 ---- wrmsr jmp 2f 1: /* Restore userland %fs */ + cmpq %r9,PCB_FSBASE(%r8) + jz 3f movl $MSR_FSBASE,%ecx movl PCB_FSBASE(%r8),%eax movl PCB_FSBASE+4(%r8),%edx wrmsr + 3: /* Restore userland %gs */ + cmpq %r10,PCB_GSBASE(%r8) + jz 2f movl $MSR_KGSBASE,%ecx movl PCB_GSBASE(%r8),%eax movl PCB_GSBASE+4(%r8),%edx ==== //depot/projects/hammer/sys/amd64/amd64/mp_machdep.c#111 - /home/src/sys/amd64/amd64/mp_machdep.c ==== *************** *** 585,590 **** --- 585,591 ---- binuptime(PCPU_PTR(switchtime)); PCPU_SET(switchticks, ticks); + PCPU_SET(tickin, rdtsc()); cpu_throw(NULL, choosethread()); /* doesn't return */ panic("scheduler returned us to %s", __func__); *************** *** 810,875 **** * Flush the TLB on all other CPU's */ static void smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) { ! u_int ncpu; - ncpu = mp_ncpus - 1; /* does not shootdown self */ - if (ncpu < 1) - return; /* no other cpus */ mtx_assert(&smp_ipi_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; ! atomic_store_rel_int(&smp_tlb_wait, 0); ipi_all_but_self(vector); ! while (smp_tlb_wait < ncpu) ! ia32_pause(); } static void smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) { ! int ncpu, othercpus; ! othercpus = mp_ncpus - 1; ! if (mask == (u_int)-1) { ! ncpu = othercpus; ! if (ncpu < 1) ! return; ! } else { ! mask &= ~PCPU_GET(cpumask); ! if (mask == 0) ! return; ! ncpu = bitcount32(mask); ! if (ncpu > othercpus) { ! /* XXX this should be a panic offence */ ! printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", ! ncpu, othercpus); ! ncpu = othercpus; ! } ! /* XXX should be a panic, implied by mask == 0 above */ ! if (ncpu < 1) ! return; ! } mtx_assert(&smp_ipi_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; ! atomic_store_rel_int(&smp_tlb_wait, 0); if (mask == (u_int)-1) ipi_all_but_self(vector); else ipi_selected(mask, vector); ! while (smp_tlb_wait < ncpu) ! ia32_pause(); } void smp_invltlb(void) { ! if (smp_started) { smp_tlb_shootdown(IPI_INVLTLB, 0, 0); - } } void --- 811,951 ---- * Flush the TLB on all other CPU's */ static void + tlb_check(td) + { + u_int mask = PCPU_GET(cpumask); + int vector = smp_tlb_vector; + vm_offset_t addr; + + if ((mask & smp_tlb_wait) == 0) + return; + switch (vector) { + case IPI_INVLTLB: + invltlb(); + atomic_clear(&smp_tlb_wait, mask); + break; + case IPI_INVLPG: + invlpg(smp_tlb_addr1); + atomic_clear(&smp_tlb_wait, mask); + break; + case IPI_INVLRNG: + for (addr = smp_tlb_addr1; addr < smp_tlb_addr2; addr += PAGE_SIZE) + invlpg(addr); + atomic_clear(&smp_tlb_wait, mask); + break; + case 0: + break; + default: + panic("tlb vector %d", vector); + } + return; + + } + + void + _smp_ipi_spinlock(const char *file, int line) + { + uintptr_t td = curthread; + struct mtx *m = &smp_ipi_mtx; + + MPASS(curthread != NULL); + KASSERT(m->mtx_object.lo_class == &lock_class_mtx_spin, + ("mtx_lock_spin() of sleep mutex %s @ %s:%d", + m->mtx_object.lo_name, file, line)); + WITNESS_CHECKORDER(&m->mtx_object, LOP_NEWORDER | LOP_EXCLUSIVE, + file, line); + spinlock_enter(); + if (!_obtain_lock(m, td)) { + if (m->mtx_lock == td) + m->mtx_recurse++; + else { + int i = 0; + + if (LOCK_LOG_TEST(&m->mtx_object, 0)) + CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); + + for (;;) { + if (_obtain_lock(m, td)) + break; + tlb_check(td); + + /* Give interrupts a chance while we spin. */ + spinlock_exit(); + while (m->mtx_lock != MTX_UNOWNED) { + if (i++ < 10000000) { + cpu_spinwait(); + continue; + } + if (i < 60000000) + DELAY(1); + else if (!kdb_active && !panicstr) { + printf("spin lock %s held by %p for > 5 seconds\n", + m->mtx_object.lo_name, (void *)m->mtx_lock); + #ifdef WITNESS + witness_display_spinlock(&m->mtx_object, + mtx_owner(m)); + #endif + panic("spin lock held too long"); + } + cpu_spinwait(); + } + spinlock_enter(); + } + + if (LOCK_LOG_TEST(&m->mtx_object, 0)) + CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); + } + } + LOCK_LOG_LOCK("LOCK", &m->mtx_object, 0, m->mtx_recurse, file, + line); + WITNESS_LOCK(&m->mtx_object, LOP_EXCLUSIVE, file, line); + } + + static void smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) { ! int mask; mtx_assert(&smp_ipi_mtx, MA_OWNED); + mask = PCPU_GET(other_cpus); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; ! smp_tlb_vector = vector; ! atomic_store_rel_int(&smp_tlb_wait, mask); ipi_all_but_self(vector); ! while (smp_tlb_wait != 0) ! cpu_spinwait(); } static void smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) { ! int othercpus; ! othercpus = PCPU_GET(other_cpus); ! if (mask != (u_int)-1) ! othercpus &= mask; ! if (mask == 0) ! return; mtx_assert(&smp_ipi_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; ! smp_tlb_vector = vector; ! atomic_store_rel_int(&smp_tlb_wait, mask); if (mask == (u_int)-1) ipi_all_but_self(vector); else ipi_selected(mask, vector); ! while (smp_tlb_wait != 0) ! cpu_spinwait(); } void smp_invltlb(void) { ! if (smp_started) smp_tlb_shootdown(IPI_INVLTLB, 0, 0); } void *************** *** 884,919 **** smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) { ! if (smp_started) { smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); - } } void smp_masked_invltlb(u_int mask) { ! if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); - } } void smp_masked_invlpg(u_int mask, vm_offset_t addr) { ! if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); - } } void smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2) { ! if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); - } } void --- 960,991 ---- smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) { ! if (smp_started) smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); } void smp_masked_invltlb(u_int mask) { ! if (smp_started) smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); } void smp_masked_invlpg(u_int mask, vm_offset_t addr) { ! if (smp_started) smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); } void smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2) { ! if (smp_started) smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); } void ==== //depot/projects/hammer/sys/amd64/amd64/pmap.c#130 - /home/src/sys/amd64/amd64/pmap.c ==== *************** *** 611,617 **** if (smp_started) { if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); ! mtx_lock_spin(&smp_ipi_mtx); } else critical_enter(); /* --- 611,617 ---- if (smp_started) { if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); ! smp_ipi_spinlock(); } else critical_enter(); /* *************** *** 647,653 **** if (smp_started) { if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); ! mtx_lock_spin(&smp_ipi_mtx); } else critical_enter(); /* --- 647,653 ---- if (smp_started) { if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); ! smp_ipi_spinlock(); } else critical_enter(); /* *************** *** 685,691 **** if (smp_started) { if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); ! mtx_lock_spin(&smp_ipi_mtx); } else critical_enter(); /* --- 685,691 ---- if (smp_started) { if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); ! smp_ipi_spinlock(); } else critical_enter(); /* ==== //depot/projects/hammer/sys/amd64/include/smp.h#28 - /home/src/sys/amd64/include/smp.h ==== *************** *** 67,74 **** --- 67,81 ---- int ipi_nmi_handler(void); #endif + #endif /* !LOCORE */ #endif /* SMP */ + #ifdef SMP + #define smp_ipi_spinlock() _smp_ipi_spinlock(LOCK_FILE, LOCK_LINE) + #else + #define smp_ipi_spinlock() mtx_lock_spin(&smp_ipi_mtx) + #endif + #endif /* _KERNEL */ #endif /* _MACHINE_SMP_H_ */