Index: sys/amd64/include/vmm.h =================================================================== --- sys/amd64/include/vmm.h (revision 264519) +++ sys/amd64/include/vmm.h (working copy) @@ -286,7 +286,6 @@ struct { uint64_t gpa; int fault_type; - int protection; } paging; struct { uint64_t gpa; @@ -299,9 +298,19 @@ * exitcode to represent the VM-exit. */ struct { - int error; /* vmx inst error */ + int status; /* vmx inst status */ + /* + * 'exit_reason' and 'exit_qualification' are valid + * only if 'status' is zero. + */ uint32_t exit_reason; uint64_t exit_qualification; + /* + * 'inst_error' and 'inst_type' are valid + * only if 'status' is non-zero. + */ + int inst_type; + int inst_error; } vmx; struct { uint32_t code; /* ecx value */ Index: sys/amd64/vmm/intel/vmx.c =================================================================== --- sys/amd64/vmm/intel/vmx.c (revision 264519) +++ sys/amd64/vmm/intel/vmx.c (working copy) @@ -286,82 +286,6 @@ return (reasonbuf); } } - -#ifdef SETJMP_TRACE -static const char * -vmx_setjmp_rc2str(int rc) -{ - switch (rc) { - case VMX_RETURN_DIRECT: - return "direct"; - case VMX_RETURN_LONGJMP: - return "longjmp"; - case VMX_RETURN_VMRESUME: - return "vmresume"; - case VMX_RETURN_VMLAUNCH: - return "vmlaunch"; - case VMX_RETURN_AST: - return "ast"; - default: - return "unknown"; - } -} - -#define SETJMP_TRACE(vmx, vcpu, vmxctx, regname) \ - VCPU_CTR1((vmx)->vm, (vcpu), "setjmp trace " #regname " 0x%016lx", \ - (vmxctx)->regname) - -static void -vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc) -{ - uint64_t host_rip, host_rsp; - - if (vmxctx != &vmx->ctx[vcpu]) - panic("vmx_setjmp_trace: invalid vmxctx %p; should be %p", - vmxctx, &vmx->ctx[vcpu]); - - VCPU_CTR1((vmx)->vm, (vcpu), "vmxctx = %p", vmxctx); - VCPU_CTR2((vmx)->vm, (vcpu), "setjmp return code %s(%d)", - vmx_setjmp_rc2str(rc), rc); - - host_rip = vmcs_read(VMCS_HOST_RIP); - host_rsp = vmcs_read(VMCS_HOST_RSP); - VCPU_CTR2((vmx)->vm, (vcpu), "vmcs host_rip 0x%016lx, host_rsp %#lx", - host_rip, host_rsp); - - SETJMP_TRACE(vmx, vcpu, vmxctx, host_r15); - SETJMP_TRACE(vmx, vcpu, vmxctx, host_r14); - SETJMP_TRACE(vmx, vcpu, vmxctx, host_r13); - SETJMP_TRACE(vmx, vcpu, vmxctx, host_r12); - SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbp); - SETJMP_TRACE(vmx, vcpu, vmxctx, host_rsp); - SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbx); - SETJMP_TRACE(vmx, vcpu, vmxctx, host_rip); - - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdi); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rsi); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdx); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rcx); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r8); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r9); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rax); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbx); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbp); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r10); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r11); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r12); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r13); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r14); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r15); - SETJMP_TRACE(vmx, vcpu, vmxctx, guest_cr2); -} -#endif -#else -static void __inline -vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc) -{ - return; -} #endif /* KTR */ u_long @@ -825,7 +749,7 @@ } error = vmcs_set_defaults(&vmx->vmcs[i], - (u_long)vmx_longjmp, + (u_long)vmx_exit_guest, (u_long)&vmx->ctx[i], vmx->eptp, pinbased_ctls, @@ -1222,21 +1146,6 @@ return (fault_type); } -static int -ept_protection(uint64_t ept_qual) -{ - int prot = 0; - - if (ept_qual & EPT_VIOLATION_GPA_READABLE) - prot |= VM_PROT_READ; - if (ept_qual & EPT_VIOLATION_GPA_WRITEABLE) - prot |= VM_PROT_WRITE; - if (ept_qual & EPT_VIOLATION_GPA_EXECUTABLE) - prot |= VM_PROT_EXECUTE; - - return (prot); -} - static boolean_t ept_emulation_fault(uint64_t ept_qual) { @@ -1269,7 +1178,6 @@ vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) { int error, handled; - struct vmcs *vmcs; struct vmxctx *vmxctx; uint32_t eax, ecx, edx, idtvec_info, idtvec_err, reason; uint64_t qual, gpa; @@ -1276,8 +1184,8 @@ bool retu; handled = 0; - vmcs = &vmx->vmcs[vcpu]; vmxctx = &vmx->ctx[vcpu]; + qual = vmexit->u.vmx.exit_qualification; reason = vmexit->u.vmx.exit_reason; vmexit->exitcode = VM_EXITCODE_BOGUS; @@ -1421,7 +1329,6 @@ vmexit->exitcode = VM_EXITCODE_PAGING; vmexit->u.paging.gpa = gpa; vmexit->u.paging.fault_type = ept_fault_type(qual); - vmexit->u.paging.protection = ept_protection(qual); } else if (ept_emulation_fault(qual)) { vmexit->exitcode = VM_EXITCODE_INST_EMUL; vmexit->u.inst_emul.gpa = gpa; @@ -1455,7 +1362,7 @@ * treat it as a generic VMX exit. */ vmexit->exitcode = VM_EXITCODE_VMX; - vmexit->u.vmx.error = 0; + vmexit->u.vmx.status = VM_SUCCESS; } else { /* * The exitcode and collateral have been populated. @@ -1466,23 +1373,63 @@ return (handled); } +static __inline int +vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) +{ + + vmexit->rip = vmcs_guest_rip(); + vmexit->inst_length = 0; + vmexit->exitcode = VM_EXITCODE_BOGUS; + vmx_astpending_trace(vmx, vcpu, vmexit->rip); + vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1); + + return (HANDLED); +} + +static __inline int +vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit) +{ + + KASSERT(vmxctx->inst_fail_status != VM_SUCCESS, + ("vmx_exit_inst_error: invalid inst_fail_status %d", + vmxctx->inst_fail_status)); + + vmexit->inst_length = 0; + vmexit->exitcode = VM_EXITCODE_VMX; + vmexit->u.vmx.status = vmxctx->inst_fail_status; + vmexit->u.vmx.inst_error = vmcs_instruction_error(); + vmexit->u.vmx.exit_reason = ~0; + vmexit->u.vmx.exit_qualification = ~0; + + switch (rc) { + case VMX_VMRESUME_ERROR: + case VMX_VMLAUNCH_ERROR: + case VMX_INVEPT_ERROR: + vmexit->u.vmx.inst_type = rc; + break; + default: + panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc); + } + + return (UNHANDLED); +} + static int -vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap) +vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap) { - int vie, rc, handled, astpending; - uint32_t exit_reason; + int rc, handled, launched; struct vmx *vmx; struct vmxctx *vmxctx; struct vmcs *vmcs; struct vm_exit *vmexit; + uint64_t rip; + uint32_t exit_reason; vmx = arg; vmcs = &vmx->vmcs[vcpu]; vmxctx = &vmx->ctx[vcpu]; - vmxctx->launched = 0; - - astpending = 0; vmexit = vm_exitinfo(vmx->vm, vcpu); + launched = 0; KASSERT(vmxctx->pmap == pmap, ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap)); @@ -1489,9 +1436,6 @@ KASSERT(vmxctx->eptp == vmx->eptp, ("eptp %p different than ctx eptp %#lx", eptp, vmxctx->eptp)); - /* - * XXX Can we avoid doing this every time we do a vm run? - */ VMPTRLD(vmcs); /* @@ -1503,76 +1447,55 @@ * of a single process we could do this once in vmcs_set_defaults(). */ vmcs_write(VMCS_HOST_CR3, rcr3()); - vmcs_write(VMCS_GUEST_RIP, rip); + + vmcs_write(VMCS_GUEST_RIP, startrip); vmx_set_pcpu_defaults(vmx, vcpu); + do { + /* + * Interrupts are disabled from this point on until the + * guest starts executing. This is done for the following + * reasons: + * + * If an AST is asserted on this thread after the check below, + * then the IPI_AST notification will not be lost, because it + * will cause a VM exit due to external interrupt as soon as + * the guest state is loaded. + * + * A posted interrupt after 'vmx_inject_interrupts()' will + * not be "lost" because it will be held pending in the host + * APIC because interrupts are disabled. The pending interrupt + * will be recognized as soon as the guest state is loaded. + * + * The same reasoning applies to the IPI generated by + * pmap_invalidate_ept(). + */ + disable_intr(); + if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) { + enable_intr(); + handled = vmx_exit_astpending(vmx, vcpu, vmexit); + break; + } - do { vmx_inject_interrupts(vmx, vcpu); vmx_run_trace(vmx, vcpu); - rc = vmx_setjmp(vmxctx); -#ifdef SETJMP_TRACE - vmx_setjmp_trace(vmx, vcpu, vmxctx, rc); -#endif - switch (rc) { - case VMX_RETURN_DIRECT: - if (vmxctx->launched == 0) { - vmxctx->launched = 1; - vmx_launch(vmxctx); - } else - vmx_resume(vmxctx); - panic("vmx_launch/resume should not return"); - break; - case VMX_RETURN_LONGJMP: - break; /* vm exit */ - case VMX_RETURN_AST: - astpending = 1; - break; - case VMX_RETURN_VMRESUME: - vie = vmcs_instruction_error(); - if (vmxctx->launch_error == VM_FAIL_INVALID || - vie != VMRESUME_WITH_NON_LAUNCHED_VMCS) { - printf("vmresume error %d vmcs inst error %d\n", - vmxctx->launch_error, vie); - goto err_exit; - } - vmx_launch(vmxctx); /* try to launch the guest */ - panic("vmx_launch should not return"); - break; - case VMX_RETURN_VMLAUNCH: - vie = vmcs_instruction_error(); -#if 1 - printf("vmlaunch error %d vmcs inst error %d\n", - vmxctx->launch_error, vie); -#endif - goto err_exit; - case VMX_RETURN_INVEPT: - panic("vm %s:%d invept error %d", - vm_name(vmx->vm), vcpu, vmxctx->launch_error); - default: - panic("vmx_setjmp returned %d", rc); - } - - /* enable interrupts */ + rc = vmx_enter_guest(vmxctx, launched); + enable_intr(); - /* collect some basic information for VM exit processing */ + /* Collect some information for VM exit processing */ vmexit->rip = rip = vmcs_guest_rip(); vmexit->inst_length = vmexit_instruction_length(); vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(); vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); - if (astpending) { - handled = 1; - vmexit->inst_length = 0; - vmexit->exitcode = VM_EXITCODE_BOGUS; - vmx_astpending_trace(vmx, vcpu, rip); - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1); - break; + if (rc == VMX_GUEST_VMEXIT) { + launched = 1; + handled = vmx_exit_process(vmx, vcpu, vmexit); + } else { + handled = vmx_exit_inst_error(vmxctx, rc, vmexit); } - handled = vmx_exit_process(vmx, vcpu, vmexit); vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled); - } while (handled); /* @@ -1588,26 +1511,11 @@ if (!handled) vmm_stat_incr(vmx->vm, vcpu, VMEXIT_USERSPACE, 1); - VCPU_CTR1(vmx->vm, vcpu, "goto userland: exitcode %d",vmexit->exitcode); + VCPU_CTR1(vmx->vm, vcpu, "returning from vmx_run: exitcode %d", + vmexit->exitcode); - /* - * XXX - * We need to do this to ensure that any VMCS state cached by the - * processor is flushed to memory. We need to do this in case the - * VM moves to a different cpu the next time it runs. - * - * Can we avoid doing this? - */ VMCLEAR(vmcs); return (0); - -err_exit: - vmexit->exitcode = VM_EXITCODE_VMX; - vmexit->u.vmx.exit_reason = (uint32_t)-1; - vmexit->u.vmx.exit_qualification = (uint32_t)-1; - vmexit->u.vmx.error = vie; - VMCLEAR(vmcs); - return (ENOEXEC); } static void Index: sys/amd64/vmm/intel/vmx.h =================================================================== --- sys/amd64/vmm/intel/vmx.h (revision 264519) +++ sys/amd64/vmm/intel/vmx.h (working copy) @@ -36,9 +36,6 @@ #define GUEST_MSR_MAX_ENTRIES 64 /* arbitrary */ struct vmxctx { - register_t tmpstk[32]; /* vmx_return() stack */ - register_t tmpstktop; - register_t guest_rdi; /* Guest state */ register_t guest_rsi; register_t guest_rdx; @@ -68,8 +65,7 @@ * XXX todo debug registers and fpu state */ - int launched; /* vmcs launch state */ - int launch_error; + int inst_fail_status; long eptgen[MAXCPU]; /* cached pmap->pm_eptgen */ @@ -107,25 +103,12 @@ CTASSERT((offsetof(struct vmx, msr_bitmap) & PAGE_MASK) == 0); CTASSERT((offsetof(struct vmx, guest_msrs) & 15) == 0); -#define VMX_RETURN_DIRECT 0 -#define VMX_RETURN_LONGJMP 1 -#define VMX_RETURN_VMRESUME 2 -#define VMX_RETURN_VMLAUNCH 3 -#define VMX_RETURN_AST 4 -#define VMX_RETURN_INVEPT 5 -/* - * vmx_setjmp() returns: - * - 0 when it returns directly - * - 1 when it returns from vmx_longjmp - * - 2 when it returns from vmx_resume (which would only be in the error case) - * - 3 when it returns from vmx_launch (which would only be in the error case) - * - 4 when it returns from vmx_resume or vmx_launch because of AST pending - * - 5 when it returns from vmx_launch/vmx_resume because of invept error - */ -int vmx_setjmp(struct vmxctx *ctx); -void vmx_longjmp(void); /* returns via vmx_setjmp */ -void vmx_launch(struct vmxctx *ctx) __dead2; /* may return via vmx_setjmp */ -void vmx_resume(struct vmxctx *ctx) __dead2; /* may return via vmx_setjmp */ +#define VMX_GUEST_VMEXIT 0 +#define VMX_VMRESUME_ERROR 1 +#define VMX_VMLAUNCH_ERROR 2 +#define VMX_INVEPT_ERROR 3 +int vmx_enter_guest(struct vmxctx *ctx, int launched); +void vmx_exit_guest(void); u_long vmx_fix_cr0(u_long cr0); u_long vmx_fix_cr4(u_long cr4); Index: sys/amd64/vmm/intel/vmx_genassym.c =================================================================== --- sys/amd64/vmm/intel/vmx_genassym.c (revision 264519) +++ sys/amd64/vmm/intel/vmx_genassym.c (working copy) @@ -31,7 +31,6 @@ #include #include -#include #include #include @@ -42,7 +41,6 @@ #include "vmx_cpufunc.h" #include "vmx.h" -ASSYM(VMXCTX_TMPSTKTOP, offsetof(struct vmxctx, tmpstktop)); ASSYM(VMXCTX_GUEST_RDI, offsetof(struct vmxctx, guest_rdi)); ASSYM(VMXCTX_GUEST_RSI, offsetof(struct vmxctx, guest_rsi)); ASSYM(VMXCTX_GUEST_RDX, offsetof(struct vmxctx, guest_rdx)); @@ -69,27 +67,19 @@ ASSYM(VMXCTX_HOST_RBX, offsetof(struct vmxctx, host_rbx)); ASSYM(VMXCTX_HOST_RIP, offsetof(struct vmxctx, host_rip)); -ASSYM(VMXCTX_LAUNCH_ERROR, offsetof(struct vmxctx, launch_error)); +ASSYM(VMXCTX_INST_FAIL_STATUS, offsetof(struct vmxctx, inst_fail_status)); ASSYM(VMXCTX_EPTGEN, offsetof(struct vmxctx, eptgen)); ASSYM(VMXCTX_PMAP, offsetof(struct vmxctx, pmap)); ASSYM(VMXCTX_EPTP, offsetof(struct vmxctx, eptp)); -ASSYM(VM_SUCCESS, VM_SUCCESS); ASSYM(VM_FAIL_INVALID, VM_FAIL_INVALID); ASSYM(VM_FAIL_VALID, VM_FAIL_VALID); +ASSYM(VMX_GUEST_VMEXIT, VMX_GUEST_VMEXIT); +ASSYM(VMX_VMRESUME_ERROR, VMX_VMRESUME_ERROR); +ASSYM(VMX_VMLAUNCH_ERROR, VMX_VMLAUNCH_ERROR); +ASSYM(VMX_INVEPT_ERROR, VMX_INVEPT_ERROR); -ASSYM(VMX_RETURN_DIRECT, VMX_RETURN_DIRECT); -ASSYM(VMX_RETURN_LONGJMP, VMX_RETURN_LONGJMP); -ASSYM(VMX_RETURN_VMRESUME, VMX_RETURN_VMRESUME); -ASSYM(VMX_RETURN_VMLAUNCH, VMX_RETURN_VMLAUNCH); -ASSYM(VMX_RETURN_AST, VMX_RETURN_AST); -ASSYM(VMX_RETURN_INVEPT, VMX_RETURN_INVEPT); - -ASSYM(TDF_ASTPENDING, TDF_ASTPENDING); -ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED); -ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); -ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread)); ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active)); Index: sys/amd64/vmm/intel/vmx_support.S =================================================================== --- sys/amd64/vmm/intel/vmx_support.S (revision 264519) +++ sys/amd64/vmm/intel/vmx_support.S (working copy) @@ -1,5 +1,6 @@ /*- * Copyright (c) 2011 NetApp, Inc. + * Copyright (c) 2013 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,32 +38,6 @@ #endif /* - * Disable interrupts before updating %rsp in VMX_CHECK_AST or - * VMX_GUEST_RESTORE. - * - * The location that %rsp points to is a 'vmxctx' and not a - * real stack so we don't want an interrupt handler to trash it - */ -#define VMX_DISABLE_INTERRUPTS cli - -/* - * If the thread hosting the vcpu has an ast pending then take care of it - * by returning from vmx_setjmp() with a return value of VMX_RETURN_AST. - * - * Assumes that %rdi holds a pointer to the 'vmxctx' and that interrupts - * are disabled. - */ -#define VMX_CHECK_AST \ - movq PCPU(CURTHREAD),%rax; \ - testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax); \ - je 9f; \ - movq $VMX_RETURN_AST,%rsi; \ - movq %rdi,%rsp; \ - addq $VMXCTX_TMPSTKTOP,%rsp; \ - callq vmx_return; \ -9: - -/* * Assumes that %rdi holds a pointer to the 'vmxctx'. * * On "return" all registers are updated to reflect guest state. The two @@ -93,132 +68,132 @@ movq VMXCTX_GUEST_RDI(%rdi),%rdi; /* restore rdi the last */ /* - * Check for an error after executing a VMX instruction. - * 'errreg' will be zero on success and non-zero otherwise. - * 'ctxreg' points to the 'struct vmxctx' associated with the vcpu. + * Save and restore the host context. + * + * Assumes that %rdi holds a pointer to the 'vmxctx'. */ -#define VM_INSTRUCTION_ERROR(errreg, ctxreg) \ - jnc 1f; \ - movl $VM_FAIL_INVALID,errreg; /* CF is set */ \ - jmp 3f; \ -1: jnz 2f; \ - movl $VM_FAIL_VALID,errreg; /* ZF is set */ \ - jmp 3f; \ -2: movl $VM_SUCCESS,errreg; \ -3: movl errreg,VMXCTX_LAUNCH_ERROR(ctxreg) +#define VMX_HOST_SAVE(tmpreg) \ + movq (%rsp), tmpreg; /* return address */ \ + movq %r15, VMXCTX_HOST_R15(%rdi); \ + movq %r14, VMXCTX_HOST_R14(%rdi); \ + movq %r13, VMXCTX_HOST_R13(%rdi); \ + movq %r12, VMXCTX_HOST_R12(%rdi); \ + movq %rbp, VMXCTX_HOST_RBP(%rdi); \ + movq %rsp, VMXCTX_HOST_RSP(%rdi); \ + movq %rbx, VMXCTX_HOST_RBX(%rdi); \ + movq tmpreg, VMXCTX_HOST_RIP(%rdi) +#define VMX_HOST_RESTORE(tmpreg) \ + movq VMXCTX_HOST_R15(%rdi), %r15; \ + movq VMXCTX_HOST_R14(%rdi), %r14; \ + movq VMXCTX_HOST_R13(%rdi), %r13; \ + movq VMXCTX_HOST_R12(%rdi), %r12; \ + movq VMXCTX_HOST_RBP(%rdi), %rbp; \ + movq VMXCTX_HOST_RSP(%rdi), %rsp; \ + movq VMXCTX_HOST_RBX(%rdi), %rbx; \ + movq VMXCTX_HOST_RIP(%rdi), tmpreg; \ + movq tmpreg, (%rsp) /* return address */ + /* - * set or clear the appropriate bit in 'pm_active' - * %rdi = vmxctx - * %rax, %r11 = scratch registers + * vmx_enter_guest(struct vmxctx *vmxctx, int launched) + * %rdi: pointer to the 'vmxctx' + * %esi: launch state of the VMCS + * Interrupts must be disabled on entry. */ -#define VMX_SET_PM_ACTIVE \ - movq VMXCTX_PMAP(%rdi), %r11; \ - movl PCPU(CPUID), %eax; \ +ENTRY(vmx_enter_guest) + /* + * Save host state before doing anything else. + */ + VMX_HOST_SAVE(%r10) + + /* + * Activate guest pmap on this cpu. + */ + movq VMXCTX_PMAP(%rdi), %r11 + movl PCPU(CPUID), %eax LK btsl %eax, PM_ACTIVE(%r11) -#define VMX_CLEAR_PM_ACTIVE \ - movq VMXCTX_PMAP(%rdi), %r11; \ - movl PCPU(CPUID), %eax; \ - LK btrl %eax, PM_ACTIVE(%r11) + /* + * If 'vmxctx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen' + * then we must invalidate all mappings associated with this EPTP. + */ + movq PM_EPTGEN(%r11), %r10 + cmpq %r10, VMXCTX_EPTGEN(%rdi, %rax, 8) + je guest_restore -/* - * If 'vmxctx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen' - * then we must invalidate all mappings associated with this eptp. - * - * %rdi = vmxctx - * %rax, %rbx, %r11 = scratch registers - */ -#define VMX_CHECK_EPTGEN \ - movl PCPU(CPUID), %ebx; \ - movq VMXCTX_PMAP(%rdi), %r11; \ - movq PM_EPTGEN(%r11), %rax; \ - cmpq %rax, VMXCTX_EPTGEN(%rdi, %rbx, 8); \ - je 9f; \ - \ - /* Refresh 'vmxctx->eptgen[curcpu]' */ \ - movq %rax, VMXCTX_EPTGEN(%rdi, %rbx, 8); \ - \ - /* Setup the invept descriptor at the top of tmpstk */ \ - mov %rdi, %r11; \ - addq $VMXCTX_TMPSTKTOP, %r11; \ - movq VMXCTX_EPTP(%rdi), %rax; \ - movq %rax, -16(%r11); \ - movq $0x0, -8(%r11); \ - mov $0x1, %eax; /* Single context invalidate */ \ - invept -16(%r11), %rax; \ - \ - /* Check for invept error */ \ - VM_INSTRUCTION_ERROR(%eax, %rdi); \ - testl %eax, %eax; \ - jz 9f; \ - \ - /* Return via vmx_setjmp with retval of VMX_RETURN_INVEPT */ \ - movq $VMX_RETURN_INVEPT, %rsi; \ - movq %rdi,%rsp; \ - addq $VMXCTX_TMPSTKTOP, %rsp; \ - callq vmx_return; \ -9: ; + /* Refresh 'vmxctx->eptgen[curcpu]' */ + movq %r10, VMXCTX_EPTGEN(%rdi, %rax, 8) - .text -/* - * int vmx_setjmp(ctxp) - * %rdi = ctxp - * - * Return value is '0' when it returns directly from here. - * Return value is '1' when it returns after a vm exit through vmx_longjmp. - */ -ENTRY(vmx_setjmp) - movq (%rsp),%rax /* return address */ - movq %r15,VMXCTX_HOST_R15(%rdi) - movq %r14,VMXCTX_HOST_R14(%rdi) - movq %r13,VMXCTX_HOST_R13(%rdi) - movq %r12,VMXCTX_HOST_R12(%rdi) - movq %rbp,VMXCTX_HOST_RBP(%rdi) - movq %rsp,VMXCTX_HOST_RSP(%rdi) - movq %rbx,VMXCTX_HOST_RBX(%rdi) - movq %rax,VMXCTX_HOST_RIP(%rdi) + /* Setup the invept descriptor on the host stack */ + mov %rsp, %r11 + movq VMXCTX_EPTP(%rdi), %rax + movq %rax, -16(%r11) + movq $0x0, -8(%r11) + mov $0x1, %eax /* Single context invalidate */ + invept -16(%r11), %rax + jbe invept_error /* Check invept instruction error */ +guest_restore: + cmpl $0, %esi + je do_launch + + VMX_GUEST_RESTORE + vmresume /* - * XXX save host debug registers + * In the common case 'vmresume' returns back to the host through + * 'vmx_exit_guest' with %rsp pointing to 'vmxctx'. + * + * If there is an error we return VMX_VMRESUME_ERROR to the caller. */ - movl $VMX_RETURN_DIRECT,%eax - ret -END(vmx_setjmp) + movq %rsp, %rdi /* point %rdi back to 'vmxctx' */ + movl $VMX_VMRESUME_ERROR, %eax + jmp decode_inst_error -/* - * void vmx_return(struct vmxctx *ctxp, int retval) - * %rdi = ctxp - * %rsi = retval - * Return to vmm context through vmx_setjmp() with a value of 'retval'. - */ -ENTRY(vmx_return) - /* The pmap is no longer active on the host cpu */ - VMX_CLEAR_PM_ACTIVE +do_launch: + VMX_GUEST_RESTORE + vmlaunch + /* + * In the common case 'vmlaunch' returns back to the host through + * 'vmx_exit_guest' with %rsp pointing to 'vmxctx'. + * + * If there is an error we return VMX_VMLAUNCH_ERROR to the caller. + */ + movq %rsp, %rdi /* point %rdi back to 'vmxctx' */ + movl $VMX_VMLAUNCH_ERROR, %eax + jmp decode_inst_error - /* Restore host context. */ - movq VMXCTX_HOST_R15(%rdi),%r15 - movq VMXCTX_HOST_R14(%rdi),%r14 - movq VMXCTX_HOST_R13(%rdi),%r13 - movq VMXCTX_HOST_R12(%rdi),%r12 - movq VMXCTX_HOST_RBP(%rdi),%rbp - movq VMXCTX_HOST_RSP(%rdi),%rsp - movq VMXCTX_HOST_RBX(%rdi),%rbx - movq VMXCTX_HOST_RIP(%rdi),%rax - movq %rax,(%rsp) /* return address */ +invept_error: + movl $VMX_INVEPT_ERROR, %eax + jmp decode_inst_error +decode_inst_error: + movl $VM_FAIL_VALID, %r11d + jz inst_error + movl $VM_FAIL_INVALID, %r11d +inst_error: + movl %r11d, VMXCTX_INST_FAIL_STATUS(%rdi) + /* - * XXX restore host debug registers + * The return value is already populated in %eax so we cannot use + * it as a scratch register beyond this point. */ - movl %esi,%eax + + /* + * Deactivate guest pmap from this cpu. + */ + movq VMXCTX_PMAP(%rdi), %r11 + movl PCPU(CPUID), %r10d + LK btrl %r10d, PM_ACTIVE(%r11) + + VMX_HOST_RESTORE(%r10) ret -END(vmx_return) +END(vmx_enter_guest) /* - * void vmx_longjmp(void) + * void vmx_exit_guest(void) * %rsp points to the struct vmxctx */ -ENTRY(vmx_longjmp) +ENTRY(vmx_exit_guest) /* * Save guest state that is not automatically saved in the vmcs. */ @@ -242,80 +217,20 @@ movq %rdi,VMXCTX_GUEST_CR2(%rsp) movq %rsp,%rdi - movq $VMX_RETURN_LONGJMP,%rsi - addq $VMXCTX_TMPSTKTOP,%rsp - callq vmx_return -END(vmx_longjmp) - -/* - * void vmx_resume(struct vmxctx *ctxp) - * %rdi = ctxp - * - * Although the return type is a 'void' this function may return indirectly - * through vmx_setjmp() with a return value of 2. - */ -ENTRY(vmx_resume) - VMX_DISABLE_INTERRUPTS - - VMX_CHECK_AST - - VMX_SET_PM_ACTIVE /* This vcpu is now active on the host cpu */ - - VMX_CHECK_EPTGEN /* Check if we have to invalidate TLB */ - /* - * Restore guest state that is not automatically loaded from the vmcs. + * Deactivate guest pmap from this cpu. */ - VMX_GUEST_RESTORE + movq VMXCTX_PMAP(%rdi), %r11 + movl PCPU(CPUID), %r10d + LK btrl %r10d, PM_ACTIVE(%r11) - vmresume + VMX_HOST_RESTORE(%r10) /* - * Capture the reason why vmresume failed. + * This will return to the caller of 'vmx_enter_guest()' with a return + * value of VMX_GUEST_VMEXIT. */ - VM_INSTRUCTION_ERROR(%eax, %rsp) - - /* Return via vmx_setjmp with return value of VMX_RETURN_VMRESUME */ - movq %rsp,%rdi - movq $VMX_RETURN_VMRESUME,%rsi - - addq $VMXCTX_TMPSTKTOP,%rsp - callq vmx_return -END(vmx_resume) - -/* - * void vmx_launch(struct vmxctx *ctxp) - * %rdi = ctxp - * - * Although the return type is a 'void' this function may return indirectly - * through vmx_setjmp() with a return value of 3. - */ -ENTRY(vmx_launch) - VMX_DISABLE_INTERRUPTS - - VMX_CHECK_AST - - VMX_SET_PM_ACTIVE /* This vcpu is now active on the host cpu */ - - VMX_CHECK_EPTGEN /* Check if we have to invalidate TLB */ - - /* - * Restore guest state that is not automatically loaded from the vmcs. - */ - VMX_GUEST_RESTORE - - vmlaunch - - /* - * Capture the reason why vmlaunch failed. - */ - VM_INSTRUCTION_ERROR(%eax, %rsp) - - /* Return via vmx_setjmp with return value of VMX_RETURN_VMLAUNCH */ - movq %rsp,%rdi - movq $VMX_RETURN_VMLAUNCH,%rsi - - addq $VMXCTX_TMPSTKTOP,%rsp - callq vmx_return -END(vmx_launch) + movl $VMX_GUEST_VMEXIT, %eax + ret +END(vmx_exit_guest) Index: usr.sbin/bhyve/bhyverun.c =================================================================== --- usr.sbin/bhyve/bhyverun.c (revision 264519) +++ usr.sbin/bhyve/bhyverun.c (working copy) @@ -388,10 +388,12 @@ fprintf(stderr, "\treason\t\tVMX\n"); fprintf(stderr, "\trip\t\t0x%016lx\n", vmexit->rip); fprintf(stderr, "\tinst_length\t%d\n", vmexit->inst_length); - fprintf(stderr, "\terror\t\t%d\n", vmexit->u.vmx.error); + fprintf(stderr, "\tstatus\t\t%d\n", vmexit->u.vmx.status); fprintf(stderr, "\texit_reason\t%u\n", vmexit->u.vmx.exit_reason); fprintf(stderr, "\tqualification\t0x%016lx\n", vmexit->u.vmx.exit_qualification); + fprintf(stderr, "\tinst_type\t\t%d\n", vmexit->u.vmx.inst_type); + fprintf(stderr, "\tinst_error\t\t%d\n", vmexit->u.vmx.inst_error); return (VMEXIT_ABORT); } Index: usr.sbin/bhyvectl/bhyvectl.c =================================================================== --- usr.sbin/bhyvectl/bhyvectl.c (revision 264519) +++ usr.sbin/bhyvectl/bhyvectl.c (working copy) @@ -273,11 +273,13 @@ break; case VM_EXITCODE_VMX: printf("\treason\t\tVMX\n"); - printf("\terror\t\t%d\n", vmexit->u.vmx.error); + printf("\tstatus\t\t%d\n", vmexit->u.vmx.status); printf("\texit_reason\t0x%08x (%u)\n", vmexit->u.vmx.exit_reason, vmexit->u.vmx.exit_reason); printf("\tqualification\t0x%016lx\n", vmexit->u.vmx.exit_qualification); + printf("\tinst_type\t\t%d\n", vmexit->u.vmx.inst_type); + printf("\tinst_error\t\t%d\n", vmexit->u.vmx.inst_error); break; default: printf("*** unknown vm run exitcode %d\n", vmexit->exitcode); Index: . =================================================================== --- . (revision 264519) +++ . (working copy) Property changes on: . ___________________________________________________________________ Modified: svn:mergeinfo Merged /head:r258860,260167,260238,260397