commit b61fc3eadbdc8b30ac6d5ae5c96704ece49010a5 Author: Andriy Gapon Date: Thu Sep 15 12:15:06 2011 +0300 [wip] implement tlb shootdowns via smp rendezvous mechanism (amd64 only) - implement tlb shootdowns via smp rendezvous mechanism, no special ipis are needed now, amd64 only (see if the code can be further simplified) - thus the smp_ipi_mtx is not needed any longer smp_ipi_mtx is kept for the benefit of i386 platforms where it is still used To do: cleanse and repeat for the mentioned platforms. diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index 96c778d..02f7431 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -130,138 +130,6 @@ IDTVEC(errorint) #ifdef SMP /* - * Global address space TLB shootdown. - */ - .text - SUPERALIGN_TEXT -IDTVEC(invltlb) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_gbl(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invltlb_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - - pushq %rax - - movq %cr3, %rax /* invalidate the TLB */ - movq %rax, %cr3 - - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ - - lock - incl smp_tlb_wait - - popq %rax - jmp doreti_iret - -/* - * Single page TLB shootdown - */ - .text - SUPERALIGN_TEXT -IDTVEC(invlpg) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_pg(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invlpg_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - - pushq %rax - - movq smp_tlb_addr1, %rax - invlpg (%rax) /* invalidate single page */ - - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ - - lock - incl smp_tlb_wait - - popq %rax - jmp doreti_iret - -/* - * Page range TLB shootdown. - */ - .text - SUPERALIGN_TEXT -IDTVEC(invlrng) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_rng(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invlrng_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - - pushq %rax - pushq %rdx - - movq smp_tlb_addr1, %rdx - movq smp_tlb_addr2, %rax -1: invlpg (%rdx) /* invalidate single page */ - addq $PAGE_SIZE, %rdx - cmpq %rax, %rdx - jb 1b - - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ - - lock - incl smp_tlb_wait - - popq %rdx - popq %rax - jmp doreti_iret - -/* - * Invalidate cache. - */ - .text - SUPERALIGN_TEXT -IDTVEC(invlcache) -#ifdef COUNT_IPIS - PUSH_FRAME - movl PCPU(CPUID), %eax - movq ipi_invlcache_counts(,%rax,8),%rax - incq (%rax) - POP_FRAME -#endif - - pushq %rax - - wbinvd - - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ - - lock - incl smp_tlb_wait - - popq %rax - jmp doreti_iret - -/* * Handler for IPIs sent via the per-cpu IPI bitmap. */ .text diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 50b52c8..52567e8 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -101,19 +101,10 @@ void *dpcpu; struct pcb stoppcbs[MAXCPU]; struct pcb **susppcbs = NULL; -/* Variables needed for SMP tlb shootdown. */ -vm_offset_t smp_tlb_addr1; -vm_offset_t smp_tlb_addr2; -volatile int smp_tlb_wait; - #ifdef COUNT_IPIS /* Interrupt counts. */ static u_long *ipi_preempt_counts[MAXCPU]; static u_long *ipi_ast_counts[MAXCPU]; -u_long *ipi_invltlb_counts[MAXCPU]; -u_long *ipi_invlrng_counts[MAXCPU]; -u_long *ipi_invlpg_counts[MAXCPU]; -u_long *ipi_invlcache_counts[MAXCPU]; u_long *ipi_rendezvous_counts[MAXCPU]; static u_long *ipi_hardclock_counts[MAXCPU]; #endif @@ -516,14 +507,6 @@ cpu_mp_start(void) cpu_ipi_pending[i] = 0; } - /* Install an inter-CPU IPI for TLB invalidation */ - setidt(IPI_INVLTLB, IDTVEC(invltlb), SDT_SYSIGT, SEL_KPL, 0); - setidt(IPI_INVLPG, IDTVEC(invlpg), SDT_SYSIGT, SEL_KPL, 0); - setidt(IPI_INVLRNG, IDTVEC(invlrng), SDT_SYSIGT, SEL_KPL, 0); - - /* Install an inter-CPU IPI for cache invalidation. */ - setidt(IPI_INVLCACHE, IDTVEC(invlcache), SDT_SYSIGT, SEL_KPL, 0); - /* Install an inter-CPU IPI for all-CPU rendezvous */ setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0); @@ -1108,63 +1091,81 @@ ipi_send_cpu(int cpu, u_int ipi) /* * Flush the TLB on all other CPU's */ +struct tlb_shootdown_params { + u_int type; + vm_offset_t addr1; + vm_offset_t addr2; +}; + static void -smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) +tlb_shootdown_action(void *arg) { - u_int ncpu; + struct tlb_shootdown_params *params; + vm_offset_t addr; +#if defined(COUNT_XINVLTLB_HITS) + int cpu = PCPU_GET(cpuid); +#endif - ncpu = mp_ncpus - 1; /* does not shootdown self */ - if (ncpu < 1) - return; /* no other cpus */ - if (!(read_rflags() & PSL_I)) - panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_ipi_mtx); - smp_tlb_addr1 = addr1; - smp_tlb_addr2 = addr2; - atomic_store_rel_int(&smp_tlb_wait, 0); - ipi_all_but_self(vector); - while (smp_tlb_wait < ncpu) - ia32_pause(); - mtx_unlock_spin(&smp_ipi_mtx); + params = (struct tlb_shootdown_params *)arg; + switch (params->type) { + case IPI_INVLCACHE: + wbinvd(); + break; + case IPI_INVLTLB: +#ifdef COUNT_XINVLTLB_HITS + xhits_gbl[cpu]++; +#endif + invltlb(); + break; + case IPI_INVLPG: +#ifdef COUNT_XINVLTLB_HITS + xhits_pg[cpu]++; +#endif + invlpg(params->addr1); + break; + case IPI_INVLRNG: +#ifdef COUNT_XINVLTLB_HITS + xhits_rng[cpu]++; +#endif + for (addr = params->addr1; addr < params->addr2; + addr += PAGE_SIZE) + invlpg(addr); + break; + default: + panic("Unknown TLB shootdown type %u", params->type); + } } static void -smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) +smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, + vm_offset_t addr1, vm_offset_t addr2) { - int cpu, ncpu, othercpus; + struct tlb_shootdown_params params; - othercpus = mp_ncpus - 1; - if (CPU_ISFULLSET(&mask)) { - if (othercpus < 1) - return; - } else { - CPU_CLR(PCPU_GET(cpuid), &mask); - if (CPU_EMPTY(&mask)) - return; - } +#if 0 if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_ipi_mtx); - smp_tlb_addr1 = addr1; - smp_tlb_addr2 = addr2; - atomic_store_rel_int(&smp_tlb_wait, 0); - if (CPU_ISFULLSET(&mask)) { - ncpu = othercpus; - ipi_all_but_self(vector); - } else { - ncpu = 0; - while ((cpu = cpusetobj_ffs(&mask)) != 0) { - cpu--; - CPU_CLR(cpu, &mask); - CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, - cpu, vector); - ipi_send_cpu(cpu, vector); - ncpu++; - } - } - while (smp_tlb_wait < ncpu) - ia32_pause(); - mtx_unlock_spin(&smp_ipi_mtx); +#endif + params.type = vector; + params.addr1 = addr1; + params.addr2 = addr2; + smp_rendezvous_cpus(mask, + smp_no_rendevous_barrier, tlb_shootdown_action, + smp_no_rendevous_barrier, ¶ms); +} + +/* + * Flush the TLB on all other CPU's + */ +static void +smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) +{ + cpuset_t mask; + + CPU_COPY(&all_cpus, &mask); + CPU_CLR(curcpu, &mask); + smp_targeted_tlb_shootdown(mask, + vector, addr1, addr2); } void @@ -1471,12 +1472,6 @@ mp_ipi_intrcnt(void *dummy) int i; CPU_FOREACH(i) { - snprintf(buf, sizeof(buf), "cpu%d:invltlb", i); - intrcnt_add(buf, &ipi_invltlb_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d:invlrng", i); - intrcnt_add(buf, &ipi_invlrng_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d:invlpg", i); - intrcnt_add(buf, &ipi_invlpg_counts[i]); snprintf(buf, sizeof(buf), "cpu%d:preempt", i); intrcnt_add(buf, &ipi_preempt_counts[i]); snprintf(buf, sizeof(buf), "cpu%d:ast", i); diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index de686b7..285a913 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -36,19 +36,11 @@ extern int boot_cpu_id; extern struct pcb stoppcbs[]; extern int cpu_apic_ids[]; #ifdef COUNT_IPIS -extern u_long *ipi_invltlb_counts[MAXCPU]; -extern u_long *ipi_invlrng_counts[MAXCPU]; -extern u_long *ipi_invlpg_counts[MAXCPU]; -extern u_long *ipi_invlcache_counts[MAXCPU]; extern u_long *ipi_rendezvous_counts[MAXCPU]; #endif /* IPI handlers */ inthand_t - IDTVEC(invltlb), /* TLB shootdowns - global */ - IDTVEC(invlpg), /* TLB shootdowns - 1 page */ - IDTVEC(invlrng), /* TLB shootdowns - page range */ - IDTVEC(invlcache), /* Write back and invalidate cache */ IDTVEC(ipi_intr_bitmap_handler), /* Bitmap based IPIs */ IDTVEC(cpustop), /* CPU stops & waits to be restarted */ IDTVEC(cpususpend), /* CPU suspends & waits to be resumed */ diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c index 2999826..0933f77 100644 --- a/sys/kern/kern_shutdown.c +++ b/sys/kern/kern_shutdown.c @@ -527,26 +527,9 @@ shutdown_reset(void *junk, int howto) printf("Rebooting...\n"); DELAY(1000000); /* wait 1 sec for printf's to complete and be read */ - /* - * Acquiring smp_ipi_mtx here has a double effect: - * - it disables interrupts avoiding CPU0 preemption - * by fast handlers (thus deadlocking against other CPUs) - * - it avoids deadlocks against smp_rendezvous() or, more - * generally, threads busy-waiting, with this spinlock held, - * and waiting for responses by threads on other CPUs - * (ie. smp_tlb_shootdown()). - * - * For the !SMP case it just needs to handle the former problem. - */ -#ifdef SMP - mtx_lock_spin(&smp_ipi_mtx); -#else spinlock_enter(); -#endif - - /* cpu_boot(howto); */ /* doesn't do anything at the moment */ cpu_reset(); - /* NOTREACHED */ /* assuming reset worked */ + /* NOTREACHED */ } /*