Index: sys/amd64/amd64/apic_vector.S =================================================================== --- sys/amd64/amd64/apic_vector.S (revision 279858) +++ sys/amd64/amd64/apic_vector.S (working copy) @@ -301,4 +301,22 @@ call smp_rendezvous_action call as_lapic_eoi jmp doreti + +/* + * IPI handler whose purpose is to interrupt the CPU with minimum overhead. + * This is used by bhyve to force a host cpu executing in guest context to + * trap into the hypervisor. + */ + .text + SUPERALIGN_TEXT +IDTVEC(justreturn) + pushq %rax + pushq %rcx + pushq %rdx + call as_lapic_eoi + popq %rdx + popq %rcx + popq %rax + jmp doreti_iret + #endif /* SMP */ Index: sys/amd64/include/smp.h =================================================================== --- sys/amd64/include/smp.h (revision 279858) +++ sys/amd64/include/smp.h (working copy) @@ -54,6 +54,7 @@ IDTVEC(ipi_intr_bitmap_handler), /* Bitmap based IPIs */ IDTVEC(cpustop), /* CPU stops & waits to be restarted */ IDTVEC(cpususpend), /* CPU suspends & waits to be resumed */ + IDTVEC(justreturn), /* interrupt CPU with minimum overhead */ IDTVEC(rendezvous); /* handle CPU rendezvous */ struct pmap; Index: sys/amd64/vmm/intel/ept.c =================================================================== --- sys/amd64/vmm/intel/ept.c (revision 279858) +++ sys/amd64/vmm/intel/ept.c (working copy) @@ -43,7 +43,6 @@ #include #include "vmx_cpufunc.h" -#include "vmm_ipi.h" #include "ept.h" #define EPT_SUPPORTS_EXEC_ONLY(cap) ((cap) & (1UL << 0)) Index: sys/amd64/vmm/intel/vmx.c =================================================================== --- sys/amd64/vmm/intel/vmx.c (revision 279858) +++ sys/amd64/vmm/intel/vmx.c (working copy) @@ -55,7 +55,6 @@ #include "vmm_lapic.h" #include "vmm_host.h" #include "vmm_ioport.h" -#include "vmm_ipi.h" #include "vmm_ktr.h" #include "vmm_stat.h" #include "vatpic.h" @@ -175,7 +174,7 @@ SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, CTLFLAG_RD, &posted_interrupts, 0, "APICv posted interrupt support"); -static int pirvec; +static int pirvec = -1; SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD, &pirvec, 0, "APICv posted interrupt vector"); @@ -485,8 +484,8 @@ vmx_cleanup(void) { - if (pirvec != 0) - vmm_ipi_free(pirvec); + if (pirvec >= 0) + lapic_ipi_free(pirvec); if (vpid_unr != NULL) { delete_unrhdr(vpid_unr); @@ -694,8 +693,8 @@ MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0, &tmp); if (error == 0) { - pirvec = vmm_ipi_alloc(); - if (pirvec == 0) { + pirvec = lapic_ipi_alloc(&IDTVEC(justreturn)); + if (pirvec < 0) { if (bootverbose) { printf("vmx_init: unable to allocate " "posted interrupt vector\n"); Index: sys/amd64/vmm/io/vlapic.c =================================================================== --- sys/amd64/vmm/io/vlapic.c (revision 279858) +++ sys/amd64/vmm/io/vlapic.c (working copy) @@ -45,7 +45,6 @@ #include -#include "vmm_ipi.h" #include "vmm_lapic.h" #include "vmm_ktr.h" #include "vmm_stat.h" Index: sys/amd64/vmm/vmm.c =================================================================== --- sys/amd64/vmm/vmm.c (revision 279858) +++ sys/amd64/vmm/vmm.c (working copy) @@ -76,7 +76,6 @@ #include "vlapic.h" #include "vpmtmr.h" #include "vrtc.h" -#include "vmm_ipi.h" #include "vmm_stat.h" #include "vmm_lapic.h" @@ -298,8 +297,8 @@ vmm_host_state_init(); - vmm_ipinum = vmm_ipi_alloc(); - if (vmm_ipinum == 0) + vmm_ipinum = lapic_ipi_alloc(&IDTVEC(justreturn)); + if (vmm_ipinum < 0) vmm_ipinum = IPI_AST; error = vmm_mem_init(); @@ -338,7 +337,7 @@ vmm_resume_p = NULL; iommu_cleanup(); if (vmm_ipinum != IPI_AST) - vmm_ipi_free(vmm_ipinum); + lapic_ipi_free(vmm_ipinum); error = VMM_CLEANUP(); /* * Something bad happened - prevent new Index: sys/amd64/vmm/vmm_ipi.c =================================================================== --- sys/amd64/vmm/vmm_ipi.c (revision 279858) +++ sys/amd64/vmm/vmm_ipi.c (working copy) @@ -1,93 +0,0 @@ -/*- - * Copyright (c) 2011 NetApp, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#include -__FBSDID("$FreeBSD$"); - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include "vmm_ipi.h" - -extern inthand_t IDTVEC(rsvd), IDTVEC(justreturn); - -CTASSERT(APIC_SPURIOUS_INT == 255); - -int -vmm_ipi_alloc(void) -{ - int idx; - uintptr_t func; - struct gate_descriptor *ip; - - /* - * Search backwards from the highest IDT vector available for use - * as our IPI vector. We install the 'justreturn' handler at that - * vector and use it to interrupt the vcpus. - * - * We do this because the IPI_AST is heavyweight and saves all - * registers in the trapframe. This is overkill for our use case - * which is simply to EOI the interrupt and return. - */ - idx = APIC_SPURIOUS_INT; - while (--idx >= APIC_IPI_INTS) { - ip = &idt[idx]; - func = ((long)ip->gd_hioffset << 16 | ip->gd_looffset); - if (func == (uintptr_t)&IDTVEC(rsvd)) { - setidt(idx , IDTVEC(justreturn), SDT_SYSIGT, - SEL_KPL, 0); - return (idx); - } - } - return (0); -} - -void -vmm_ipi_free(int ipinum) -{ - uintptr_t func; - struct gate_descriptor *ip; - - KASSERT(ipinum >= APIC_IPI_INTS && ipinum < APIC_SPURIOUS_INT, - ("invalid ipi %d", ipinum)); - - ip = &idt[ipinum]; - func = ((long)ip->gd_hioffset << 16 | ip->gd_looffset); - KASSERT(func == (uintptr_t)&IDTVEC(justreturn), - ("invalid ipi %d", ipinum)); - - setidt(ipinum, IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0); -} Index: sys/amd64/vmm/vmm_ipi.h =================================================================== --- sys/amd64/vmm/vmm_ipi.h (revision 279858) +++ sys/amd64/vmm/vmm_ipi.h (working copy) @@ -1,35 +0,0 @@ -/*- - * Copyright (c) 2011 NetApp, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#ifndef _VMM_IPI_H_ -#define _VMM_IPI_H_ - -int vmm_ipi_alloc(void); -void vmm_ipi_free(int num); - -#endif Index: sys/amd64/vmm/vmm_lapic.c =================================================================== --- sys/amd64/vmm/vmm_lapic.c (revision 279858) +++ sys/amd64/vmm/vmm_lapic.c (working copy) @@ -37,7 +37,6 @@ #include #include -#include "vmm_ipi.h" #include "vmm_ktr.h" #include "vmm_lapic.h" #include "vlapic.h" Index: sys/amd64/vmm/vmm_support.S =================================================================== --- sys/amd64/vmm/vmm_support.S (revision 279858) +++ sys/amd64/vmm/vmm_support.S (working copy) @@ -1,43 +0,0 @@ -/*- - * Copyright (c) 2011 NetApp, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#define LOCORE - -#include - - .text - SUPERALIGN_TEXT -IDTVEC(justreturn) - pushq %rdx - pushq %rax - pushq %rcx - call as_lapic_eoi - popq %rcx - popq %rax - popq %rdx - jmp doreti_iret Index: sys/modules/vmm/Makefile =================================================================== --- sys/modules/vmm/Makefile (revision 279858) +++ sys/modules/vmm/Makefile (working copy) @@ -19,13 +19,11 @@ vmm_host.c \ vmm_instruction_emul.c \ vmm_ioport.c \ - vmm_ipi.c \ vmm_lapic.c \ vmm_mem.c \ vmm_stat.c \ vmm_util.c \ - x86.c \ - vmm_support.S + x86.c .PATH: ${.CURDIR}/../../amd64/vmm/io SRCS+= iommu.c \ Index: sys/x86/include/apicvar.h =================================================================== --- sys/x86/include/apicvar.h (revision 279858) +++ sys/x86/include/apicvar.h (working copy) @@ -111,11 +111,8 @@ #define IPI_INVLPG (APIC_IPI_INTS + 2) #define IPI_INVLRNG (APIC_IPI_INTS + 3) #define IPI_INVLCACHE (APIC_IPI_INTS + 4) -#ifdef __i386__ -#define IPI_LAZYPMAP (APIC_IPI_INTS + 5) /* Lazy pmap release. */ -#endif /* Vector to handle bitmap based IPIs */ -#define IPI_BITMAP_VECTOR (APIC_IPI_INTS + 6) +#define IPI_BITMAP_VECTOR (APIC_IPI_INTS + 5) /* IPIs handled by IPI_BITMAP_VECTOR */ #define IPI_AST 0 /* Generate software trap. */ @@ -124,8 +121,15 @@ #define IPI_BITMAP_LAST IPI_HARDCLOCK #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST) -#define IPI_STOP (APIC_IPI_INTS + 7) /* Stop CPU until restarted. */ -#define IPI_SUSPEND (APIC_IPI_INTS + 8) /* Suspend CPU until restarted. */ +#define IPI_STOP (APIC_IPI_INTS + 6) /* Stop CPU until restarted. */ +#define IPI_SUSPEND (APIC_IPI_INTS + 7) /* Suspend CPU until restarted. */ +#ifdef __i386__ +#define IPI_LAZYPMAP (APIC_IPI_INTS + 8) /* Lazy pmap release. */ +#define IPI_DYN_FIRST (APIC_IPI_INTS + 9) +#else +#define IPI_DYN_FIRST (APIC_IPI_INTS + 8) +#endif +#define IPI_DYN_LAST (254) /* IPIs allocated at runtime */ /* * IPI_STOP_HARD does not need to occupy a slot in the IPI vector space since @@ -224,6 +228,8 @@ void (*ipi_raw)(register_t, u_int); void (*ipi_vectored)(u_int, int); int (*ipi_wait)(int); + int (*ipi_alloc)(inthand_t *ipifunc); + void (*ipi_free)(int vector); /* LVT */ int (*set_lvt_mask)(u_int, u_int, u_char); @@ -397,6 +403,20 @@ } static inline int +lapic_ipi_alloc(inthand_t *ipifunc) +{ + + return (apic_ops.ipi_alloc(ipifunc)); +} + +static inline void +lapic_ipi_free(int vector) +{ + + return (apic_ops.ipi_free(vector)); +} + +static inline int lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) { Index: sys/x86/x86/local_apic.c =================================================================== --- sys/x86/x86/local_apic.c (revision 279858) +++ sys/x86/x86/local_apic.c (working copy) @@ -303,6 +303,8 @@ enum intr_polarity pol); static int native_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, enum intr_trigger trigger); +static int native_lapic_ipi_alloc(inthand_t *ipifunc); +static void native_lapic_ipi_free(int vector); struct apic_ops apic_ops = { .create = native_lapic_create, @@ -329,6 +331,8 @@ .ipi_raw = native_lapic_ipi_raw, .ipi_vectored = native_lapic_ipi_vectored, .ipi_wait = native_lapic_ipi_wait, + .ipi_alloc = native_lapic_ipi_alloc, + .ipi_free = native_lapic_ipi_free, #endif .set_lvt_mask = native_lapic_set_lvt_mask, .set_lvt_mode = native_lapic_set_lvt_mode, @@ -1761,4 +1765,60 @@ } #endif /* DETECT_DEADLOCK */ } + +/* + * Since the IDT is shared by all CPUs the IPI slot update needs to be globally + * visible. + * + * Consider the case where an IPI is generated immediately after allocation: + * vector = lapic_ipi_alloc(ipifunc); + * ipi_selected(other_cpus, vector); + * + * In xAPIC mode a write to ICR_LO has serializing semantics because the + * APIC page is mapped as an uncached region. In x2APIC mode there is an + * explicit 'mfence' before the ICR MSR is written. Therefore in both cases + * the IDT slot update is globally visible before the IPI is delivered. + */ +static int +native_lapic_ipi_alloc(inthand_t *ipifunc) +{ + struct gate_descriptor *ip; + long func; + int idx, vector; + + KASSERT(ipifunc != &IDTVEC(rsvd), ("invalid ipifunc %p", ipifunc)); + + vector = -1; + mtx_lock_spin(&icu_lock); + for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) { + ip = &idt[idx]; + func = (ip->gd_hioffset << 16) | ip->gd_looffset; + if (func == (uintptr_t)&IDTVEC(rsvd)) { + vector = idx; + setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC); + break; + } + } + mtx_unlock_spin(&icu_lock); + return (vector); +} + +static void +native_lapic_ipi_free(int vector) +{ + struct gate_descriptor *ip; + long func; + + KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST, + ("%s: invalid vector %d", __func__, vector)); + + mtx_lock_spin(&icu_lock); + ip = &idt[vector]; + func = (ip->gd_hioffset << 16) | ip->gd_looffset; + KASSERT(func != (uintptr_t)&IDTVEC(rsvd), + ("invalid idtfunc %#lx", func)); + setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC); + mtx_unlock_spin(&icu_lock); +} + #endif /* SMP */ Index: sys/x86/xen/xen_apic.c =================================================================== --- sys/x86/xen/xen_apic.c (revision 279858) +++ sys/x86/xen/xen_apic.c (working copy) @@ -311,9 +311,24 @@ XEN_APIC_UNSUPPORTED; return (0); } -#endif static int +xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) +{ + + XEN_APIC_UNSUPPORTED; + return (-1); +} + +static void +xen_pv_lapic_ipi_free(int vector) +{ + + XEN_APIC_UNSUPPORTED; +} +#endif /* SMP */ + +static int xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) { @@ -372,6 +387,8 @@ .ipi_raw = xen_pv_lapic_ipi_raw, .ipi_vectored = xen_pv_lapic_ipi_vectored, .ipi_wait = xen_pv_lapic_ipi_wait, + .ipi_alloc = xen_pv_lapic_ipi_alloc, + .ipi_free = xen_pv_lapic_ipi_free, #endif .set_lvt_mask = xen_pv_lapic_set_lvt_mask, .set_lvt_mode = xen_pv_lapic_set_lvt_mode,