--- //depot/vendor/freebsd/src/lib/libvmmapi/vmmapi.c +++ //depot/user/jhb/bhyve/lib/libvmmapi/vmmapi.c @@ -397,6 +397,18 @@ } int +vm_lapic_local_irq(struct vmctx *ctx, int vcpu, int vector) +{ + struct vm_lapic_irq vmirq; + + bzero(&vmirq, sizeof(vmirq)); + vmirq.cpuid = vcpu; + vmirq.vector = vector; + + return (ioctl(ctx->fd, VM_LAPIC_LOCAL_IRQ, &vmirq)); +} + +int vm_ioapic_assert_irq(struct vmctx *ctx, int irq) { struct vm_ioapic_irq ioapic_irq; --- //depot/vendor/freebsd/src/lib/libvmmapi/vmmapi.h +++ //depot/user/jhb/bhyve/lib/libvmmapi/vmmapi.h @@ -67,6 +67,7 @@ int vm_inject_event2(struct vmctx *ctx, int vcpu, enum vm_event_type type, int vector, int error_code); int vm_lapic_irq(struct vmctx *ctx, int vcpu, int vector); +int vm_lapic_local_irq(struct vmctx *ctx, int vcpu, int vector); int vm_ioapic_assert_irq(struct vmctx *ctx, int irq); int vm_ioapic_deassert_irq(struct vmctx *ctx, int irq); int vm_ioapic_pulse_irq(struct vmctx *ctx, int irq); --- //depot/vendor/freebsd/src/sys/amd64/include/vmm_dev.h +++ //depot/user/jhb/bhyve/sys/amd64/include/vmm_dev.h @@ -175,6 +175,7 @@ IOCNUM_IOAPIC_ASSERT_IRQ = 33, IOCNUM_IOAPIC_DEASSERT_IRQ = 34, IOCNUM_IOAPIC_PULSE_IRQ = 35, + IOCNUM_LAPIC_LOCAL_IRQ = 36, /* PCI pass-thru */ IOCNUM_BIND_PPTDEV = 40, @@ -217,6 +218,8 @@ _IOW('v', IOCNUM_IOAPIC_DEASSERT_IRQ, struct vm_ioapic_irq) #define VM_IOAPIC_PULSE_IRQ \ _IOW('v', IOCNUM_IOAPIC_PULSE_IRQ, struct vm_ioapic_irq) +#define VM_LAPIC_LOCAL_IRQ \ + _IOW('v', IOCNUM_LAPIC_LOCAL_IRQ, struct vm_lapic_irq) #define VM_SET_CAPABILITY \ _IOW('v', IOCNUM_SET_CAPABILITY, struct vm_capability) #define VM_GET_CAPABILITY \ --- //depot/vendor/freebsd/src/sys/amd64/vmm/io/vlapic.c +++ //depot/user/jhb/bhyve/sys/amd64/vmm/io/vlapic.c @@ -91,7 +91,7 @@ #define PRIO(x) ((x) >> 4) #define VLAPIC_VERSION (16) -#define VLAPIC_MAXLVT_ENTRIES (5) +#define VLAPIC_MAXLVT_ENTRIES (APIC_LVT_CMCI) #define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0) @@ -107,7 +107,7 @@ struct LAPIC apic; - int esr_update; + uint32_t esr_pending; struct callout callout; /* vlapic timer */ struct bintime timer_fire_bt; /* callout expiry time */ @@ -252,7 +252,8 @@ vlapic_update_errors(struct vlapic *vlapic) { struct LAPIC *lapic = &vlapic->apic; - lapic->esr = 0; // XXX + lapic->esr = vlapic->esr_pending; + vlapic->esr_pending = 0; } static void @@ -263,7 +264,8 @@ lapic->version |= (VLAPIC_MAXLVT_ENTRIES < MAXLVTSHIFT); lapic->dfr = 0xffffffff; lapic->svr = APIC_SVR_VECTOR; - vlapic_mask_lvts(&lapic->lvt_timer, VLAPIC_MAXLVT_ENTRIES+1); + vlapic_mask_lvts(&lapic->lvt_timer, 6); + vlapic_mask_lvts(&lapic->lvt_cmci, 1); } static int @@ -301,6 +303,11 @@ return; } + if (vector < 16) { + vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR); + return; + } + idx = (vector / 32) * 4; mask = 1 << (vector % 32); @@ -327,11 +334,15 @@ struct LAPIC *lapic = &vlapic->apic; int i; - if (offset < APIC_OFFSET_TIMER_LVT || offset > APIC_OFFSET_ERROR_LVT) { + switch (offset) { + case APIC_OFFSET_CMCI_LVT: + return (&lapic->lvt_cmci); + case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: + i = (offset - APIC_OFFSET_TIMER_LVT) >> 2; + return ((&lapic->lvt_timer) + i);; + default: panic("vlapic_get_lvt: invalid LVT\n"); } - i = (offset - APIC_OFFSET_TIMER_LVT) >> 2; - return ((&lapic->lvt_timer) + i);; } static __inline uint32_t @@ -344,7 +355,7 @@ static void vlapic_set_lvt(struct vlapic *vlapic, uint32_t offset, uint32_t val) { - uint32_t *lvtptr; + uint32_t *lvtptr, mask; struct LAPIC *lapic; lapic = &vlapic->apic; @@ -355,12 +366,56 @@ if (!(lapic->svr & APIC_SVR_ENABLE)) val |= APIC_LVT_M; - *lvtptr = val; + mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR; + switch (offset) { + case APIC_OFFSET_TIMER_LVT: + mask |= APIC_LVTT_TM; + break; + case APIC_OFFSET_ERROR_LVT: + break; + case APIC_OFFSET_LINT0_LVT: + case APIC_OFFSET_LINT1_LVT: + mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP; + default: + mask |= APIC_LVT_DM; + break; + } + *lvtptr = val & mask; if (offset == APIC_OFFSET_TIMER_LVT) VLAPIC_TIMER_UNLOCK(vlapic); } +static int +vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt) +{ + uint32_t vec, mode; + + if (lvt & APIC_LVT_M) + return (0); + + vec = lvt & APIC_LVT_VECTOR; + mode = lvt & APIC_LVT_DM; + + switch (mode) { + case APIC_LVT_DM_FIXED: + if (vec < 16) { + vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR); + return (0); + } + vlapic_set_intr_ready(vlapic, vec, false); + vcpu_notify_event(vlapic->vm, vlapic->vcpuid); + break; + case APIC_LVT_DM_NMI: + vm_inject_nmi(vlapic->vm, vlapic->vcpuid); + break; + default: + // Other modes ignored + return (0); + } + return (1); +} + #if 1 static void dump_isrvec_stk(struct vlapic *vlapic) @@ -499,24 +554,92 @@ return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC)); } +static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic"); + +void +vlapic_set_error(struct vlapic *vlapic, uint32_t mask) +{ + uint32_t lvt; + + vlapic->esr_pending |= mask; + + // The error LVT always uses the fixed delivery mode. + lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT); + if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) { + vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1); + } +} + static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic"); static void vlapic_fire_timer(struct vlapic *vlapic) { - int vector; uint32_t lvt; KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked")); + // The timer LVT always uses the fixed delivery mode. lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT); + if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) { + vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1); + } +} + +static VMM_STAT(VLAPIC_INTR_CMC, + "corrected machine check interrupts generated by vlapic"); + +void +vlapic_fire_cmci(struct vlapic *vlapic) +{ + uint32_t lvt; + + lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT); + if (vlapic_fire_lvt(vlapic, lvt)) { + vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1); + } +} + +static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_ENTRIES, + "lvts triggered"); + +int +vlapic_trigger_lvt(struct vlapic *vlapic, int vector) +{ + uint32_t lvt; - if (!vlapic_get_lvt_field(lvt, APIC_LVTT_M)) { - vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1); - vector = vlapic_get_lvt_field(lvt, APIC_LVTT_VECTOR); - vlapic_set_intr_ready(vlapic, vector, false); - vcpu_notify_event(vlapic->vm, vlapic->vcpuid); + switch (vector) { + case APIC_LVT_LINT0: + lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT); + break; + case APIC_LVT_LINT1: + lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT); + break; + case APIC_LVT_TIMER: + lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT); + lvt |= APIC_LVT_DM_FIXED; + break; + case APIC_LVT_ERROR: + lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT); + lvt |= APIC_LVT_DM_FIXED; + break; + case APIC_LVT_PMC: + lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT); + break; + case APIC_LVT_THERMAL: + lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT); + break; + case APIC_LVT_CMCI: + lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT); + break; + default: + return (EINVAL); + } + if (vlapic_fire_lvt(vlapic, lvt)) { + vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, + LVTS_TRIGGERRED, vector, 1); } + return (0); } static void @@ -628,6 +751,11 @@ vec = icrval & APIC_VECTOR_MASK; mode = icrval & APIC_DELMODE_MASK; + if (mode == APIC_DELMODE_FIXED && vec < 16) { + vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR); + return (0); + } + if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) { switch (icrval & APIC_DEST_MASK) { case APIC_DEST_DESTFLD: @@ -872,6 +1000,7 @@ case APIC_OFFSET_ICR_HI: *data = lapic->icr_hi; break; + case APIC_OFFSET_CMCI_LVT: case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: *data = vlapic_get_lvt(vlapic, offset); break; @@ -939,6 +1068,7 @@ lapic->icr_hi = data; } break; + case APIC_OFFSET_CMCI_LVT: case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: vlapic_set_lvt(vlapic, offset, data); break; --- //depot/vendor/freebsd/src/sys/amd64/vmm/io/vlapic.h +++ //depot/user/jhb/bhyve/sys/amd64/vmm/io/vlapic.h @@ -69,6 +69,7 @@ #define APIC_OFFSET_IRR6 0x260 // IRR 192-223 R #define APIC_OFFSET_IRR7 0x270 // IRR 224-255 R #define APIC_OFFSET_ESR 0x280 // Error Status Register R +#define APIC_OFFSET_CMCI_LVT 0x2F0 // Local Vector Table (CMCI) R/W #define APIC_OFFSET_ICR_LOW 0x300 // Interrupt Command Reg. (0-31) R/W #define APIC_OFFSET_ICR_HI 0x310 // Interrupt Command Reg. (32-63) R/W #define APIC_OFFSET_TIMER_LVT 0x320 // Local Vector Table (Timer) R/W @@ -95,6 +96,9 @@ int vlapic_pending_intr(struct vlapic *vlapic); void vlapic_intr_accepted(struct vlapic *vlapic, int vector); void vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level); +void vlapic_set_error(struct vlapic *vlapic, uint32_t mask); +void vlapic_fire_cmci(struct vlapic *vlapic); +int vlapic_trigger_lvt(struct vlapic *vlapic, int vector); uint64_t vlapic_get_apicbase(struct vlapic *vlapic); void vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val); --- //depot/vendor/freebsd/src/sys/amd64/vmm/vmm_dev.c +++ //depot/user/jhb/bhyve/sys/amd64/vmm/vmm_dev.c @@ -308,6 +308,11 @@ ioapic_irq = (struct vm_ioapic_irq *)data; error = vioapic_pulse_irq(sc->vm, ioapic_irq->irq); break; + case VM_LAPIC_LOCAL_IRQ: + vmirq = (struct vm_lapic_irq *)data; + error = lapic_set_local_intr(sc->vm, vmirq->cpuid, + vmirq->vector); + break; case VM_MAP_MEMORY: seg = (struct vm_memory_segment *)data; error = vm_malloc(sc->vm, seg->gpa, seg->len); --- //depot/vendor/freebsd/src/sys/amd64/vmm/vmm_lapic.c +++ //depot/user/jhb/bhyve/sys/amd64/vmm/vmm_lapic.c @@ -80,6 +80,33 @@ return (0); } +int +lapic_set_local_intr(struct vm *vm, int cpu, int vector) +{ + struct vlapic *vlapic; + cpuset_t dmask; + int error; + + if (cpu < -1 || cpu >= VM_MAXCPU) + return (EINVAL); + + if (cpu == -1) + dmask = vm_active_cpus(vm); + else + CPU_SETOF(cpu, &dmask); + error = 0; + while ((cpu = CPU_FFS(&dmask)) != 0) { + cpu--; + CPU_CLR(cpu, &dmask); + vlapic = vm_lapic(vm, cpu); + error = vlapic_trigger_lvt(vlapic, vector); + if (error) + break; + } + + return (error); +} + static boolean_t x2apic_msr(u_int msr) { --- //depot/vendor/freebsd/src/sys/amd64/vmm/vmm_lapic.h +++ //depot/user/jhb/bhyve/sys/amd64/vmm/vmm_lapic.h @@ -82,4 +82,10 @@ return (lapic_set_intr(vm, cpu, vector, LAPIC_TRIG_EDGE)); } +/* + * Triggers the LAPIC local interrupt (LVT) 'vector' on 'cpu'. 'cpu' can + * be set to -1 to trigger the interrupt on all CPUs. + */ +int lapic_set_local_intr(struct vm *vm, int cpu, int vector); + #endif --- //depot/vendor/freebsd/src/usr.sbin/bhyve/acpi.c +++ //depot/user/jhb/bhyve/usr.sbin/bhyve/acpi.c @@ -290,6 +290,16 @@ EFPRINTF(fp, "\t\t\tTrigger Mode : 0\n"); EFPRINTF(fp, "\n"); + /* Local APIC NMI is connected to LINT 1 on all CPUs */ + EFPRINTF(fp, "[0001]\t\tSubtable Type : 04\n"); + EFPRINTF(fp, "[0001]\t\tLength : 06\n"); + EFPRINTF(fp, "[0001]\t\tProcessorId : FF\n"); + EFPRINTF(fp, "[0002]\t\tFlags (decoded below) : 0005\n"); + EFPRINTF(fp, "\t\t\tPolarity : 1\n"); + EFPRINTF(fp, "\t\t\tTrigger Mode : 1\n"); + EFPRINTF(fp, "[0001]\t\tInterrupt : 01\n"); + EFPRINTF(fp, "\n"); + EFFLUSH(fp); return (0); --- //depot/vendor/freebsd/src/usr.sbin/bhyve/mptbl.c +++ //depot/user/jhb/bhyve/usr.sbin/bhyve/mptbl.c @@ -71,6 +71,9 @@ #define MPEP_FEATURES (0xBFEBFBFF) /* XXX Intel i7 */ +/* Number of local intr entries */ +#define MPEII_NUM_LOCAL_IRQ 2 + /* Number of i/o intr entries */ #define MPEII_MAX_IRQ 24 @@ -140,6 +143,30 @@ } static void +mpt_build_localint_entries(int_entry_ptr mpie) +{ + + /* Hardcode LINT0 as ExtINT on all CPUs. */ + memset(mpie, 0, sizeof(*mpie)); + mpie->type = MPCT_ENTRY_LOCAL_INT; + mpie->int_type = INTENTRY_TYPE_EXTINT; + mpie->int_flags = INTENTRY_FLAGS_POLARITY_CONFORM | + INTENTRY_FLAGS_TRIGGER_CONFORM; + mpie->dst_apic_id = 0xff; + mpie->dst_apic_int = 0; + mpie++; + + /* Hardcode LINT1 as NMI on all CPUs. */ + memset(mpie, 0, sizeof(*mpie)); + mpie->type = MPCT_ENTRY_LOCAL_INT; + mpie->int_type = INTENTRY_TYPE_NMI; + mpie->int_flags = INTENTRY_FLAGS_POLARITY_CONFORM | + INTENTRY_FLAGS_TRIGGER_CONFORM; + mpie->dst_apic_id = 0xff; + mpie->dst_apic_int = 1; +} + +static void mpt_build_bus_entries(bus_entry_ptr mpeb) { @@ -275,6 +302,11 @@ curraddr += sizeof(*mpie) * MPEII_MAX_IRQ; mpch->entry_count += MPEII_MAX_IRQ; + mpie = (int_entry_ptr)curraddr; + mpt_build_localint_entries(mpie); + curraddr += sizeof(*mpie) * MPEII_NUM_LOCAL_IRQ; + mpch->entry_count += MPEII_NUM_LOCAL_IRQ; + if (oem_tbl_start) { mpch->oem_table_pointer = curraddr - startaddr + MPTABLE_BASE; mpch->oem_table_size = oem_tbl_size; --- //depot/vendor/freebsd/src/usr.sbin/bhyvectl/bhyvectl.c +++ //depot/user/jhb/bhyve/usr.sbin/bhyvectl/bhyvectl.c @@ -189,12 +189,15 @@ " [--set-mem=]\n" " [--get-lowmem]\n" " [--get-highmem]\n" - " [--get-gpa-pmap]\n", + " [--get-gpa-pmap]\n" + " [--assert-lapic-lvt=]\n" + " [--inject-nmi]\n", progname); exit(1); } static int get_stats, getcap, setcap, capval, get_gpa_pmap; +static int inject_nmi, assert_lapic_lvt; static const char *capname; static int create, destroy, get_lowmem, get_highmem; static uint64_t memsize; @@ -379,6 +382,7 @@ CAPNAME, UNASSIGN_PPTDEV, GET_GPA_PMAP, + ASSERT_LAPIC_LVT, }; int @@ -431,6 +435,7 @@ { "unassign-pptdev", REQ_ARG, 0, UNASSIGN_PPTDEV }, { "setcap", REQ_ARG, 0, SET_CAP }, { "get-gpa-pmap", REQ_ARG, 0, GET_GPA_PMAP }, + { "assert-lapic-lvt", REQ_ARG, 0, ASSERT_LAPIC_LVT }, { "getcap", NO_ARG, &getcap, 1 }, { "get-stats", NO_ARG, &get_stats, 1 }, { "get-desc-ds",NO_ARG, &get_desc_ds, 1 }, @@ -557,10 +562,12 @@ { "run", NO_ARG, &run, 1 }, { "create", NO_ARG, &create, 1 }, { "destroy", NO_ARG, &destroy, 1 }, + { "inject-nmi", NO_ARG, &inject_nmi, 1 }, { NULL, 0, NULL, 0 } }; vcpu = 0; + assert_lapic_lvt = -1; progname = basename(argv[0]); while ((ch = getopt_long(argc, argv, "", opts, NULL)) != -1) { @@ -682,6 +689,9 @@ if (sscanf(optarg, "%d/%d/%d", &bus, &slot, &func) != 3) usage(); break; + case ASSERT_LAPIC_LVT: + assert_lapic_lvt = atoi(optarg); + break; default: usage(); } @@ -825,6 +835,14 @@ vmcs_entry_interruption_info); } + if (!error && inject_nmi) { + error = vm_inject_nmi(ctx, vcpu); + } + + if (!error && assert_lapic_lvt != -1) { + error = vm_lapic_local_irq(ctx, vcpu, assert_lapic_lvt); + } + if (!error && (get_lowmem || get_all)) { gpa = 0; error = vm_get_memory_seg(ctx, gpa, &len, &wired);