Index: sys/amd64/include/vmm.h
===================================================================
--- sys/amd64/include/vmm.h	(revision 260830)
+++ sys/amd64/include/vmm.h	(working copy)
@@ -298,6 +298,7 @@
 	VM_EXITCODE_SPINUP_AP,
 	VM_EXITCODE_SPINDOWN_CPU,
 	VM_EXITCODE_RENDEZVOUS,
+	VM_EXITCODE_IOAPIC_EOI,
 	VM_EXITCODE_MAX
 };
 
@@ -354,6 +355,9 @@
 		struct {
 			uint64_t	rflags;
 		} hlt;
+		struct {
+			int		vector;
+		} ioapic_eoi;
 	} u;
 };
 
Index: sys/amd64/vmm/intel/vmcs.h
===================================================================
--- sys/amd64/vmm/intel/vmcs.h	(revision 260830)
+++ sys/amd64/vmm/intel/vmcs.h	(working copy)
@@ -136,6 +136,7 @@
 #define	VMCS_EOI_EXIT1			0x0000201E
 #define	VMCS_EOI_EXIT2			0x00002020
 #define	VMCS_EOI_EXIT3			0x00002022
+#define	VMCS_EOI_EXIT(vector)		(VMCS_EOI_EXIT0 + ((vector) / 64) * 2)
 
 /* 64-bit read-only fields */
 #define	VMCS_GUEST_PHYSICAL_ADDRESS	0x00002400
@@ -318,6 +319,7 @@
 #define EXIT_REASON_MCE			41
 #define EXIT_REASON_TPR			43
 #define EXIT_REASON_APIC_ACCESS		44
+#define	EXIT_REASON_VIRTUALIZED_EOI	45
 #define EXIT_REASON_GDTR_IDTR		46
 #define EXIT_REASON_LDTR_TR		47
 #define EXIT_REASON_EPT_FAULT		48
Index: sys/amd64/vmm/intel/vmx.c
===================================================================
--- sys/amd64/vmm/intel/vmx.c	(revision 260830)
+++ sys/amd64/vmm/intel/vmx.c	(working copy)
@@ -1620,6 +1620,11 @@
 			vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
 		}
 		break;
+	case EXIT_REASON_VIRTUALIZED_EOI:
+		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
+		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
+		vmexit->inst_length = 0;	/* trap-like */
+		break;
 	case EXIT_REASON_APIC_ACCESS:
 		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
 		break;
@@ -2212,6 +2217,7 @@
 struct vlapic_vtx {
 	struct vlapic	vlapic;
 	struct pir_desc	*pir_desc;
+	struct vmx	*vmx;
 };
 
 #define	VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)	\
@@ -2237,9 +2243,6 @@
 	uint64_t mask;
 	int idx, notify;
 
-	/*
-	 * XXX need to deal with level triggered interrupts
-	 */
 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
 	pir_desc = vlapic_vtx->pir_desc;
 
@@ -2314,6 +2317,33 @@
 }
 
 static void
+vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
+{
+	struct vlapic_vtx *vlapic_vtx;
+	struct vmx *vmx;
+	struct vmcs *vmcs;
+	uint64_t mask, val;
+
+	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
+	KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
+	    ("vmx_set_tmr: vcpu cannot be running"));
+
+	vlapic_vtx = (struct vlapic_vtx *)vlapic;
+	vmx = vlapic_vtx->vmx;
+	vmcs = &vmx->vmcs[vlapic->vcpuid];
+	mask = 1UL << (vector % 64);
+
+	VMPTRLD(vmcs);
+	val = vmcs_read(VMCS_EOI_EXIT(vector));
+	if (level)
+		val |= mask;
+	else
+		val &= ~mask;
+	vmcs_write(VMCS_EOI_EXIT(vector), val);
+	VMCLEAR(vmcs);
+}
+
+static void
 vmx_post_intr(struct vlapic *vlapic, int hostcpu)
 {
 
@@ -2411,11 +2441,13 @@
 
 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
 	vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
+	vlapic_vtx->vmx = vmx;
 
 	if (virtual_interrupt_delivery) {
 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
 		vlapic->ops.pending_intr = vmx_pending_intr;
 		vlapic->ops.intr_accepted = vmx_intr_accepted;
+		vlapic->ops.set_tmr = vmx_set_tmr;
 	}
 
 	if (posted_interrupts)
Index: sys/amd64/vmm/io/vlapic.c
===================================================================
--- sys/amd64/vmm/io/vlapic.c	(revision 260830)
+++ sys/amd64/vmm/io/vlapic.c	(working copy)
@@ -1300,6 +1300,7 @@
 	lapic->dfr = 0xffffffff;
 	lapic->svr = APIC_SVR_VECTOR;
 	vlapic_mask_lvts(vlapic);
+	vlapic_reset_tmr(vlapic);
 
 	lapic->dcr_timer = 0;
 	vlapic_dcr_write_handler(vlapic);
@@ -1457,32 +1458,42 @@
 		return (false);
 }
 
+static void
+vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
+{
+	struct LAPIC *lapic;
+	uint32_t *tmrptr, mask;
+	int idx;
+
+	lapic = vlapic->apic_page;
+	tmrptr = &lapic->tmr0;
+	idx = (vector / 32) * 4;
+	mask = 1 << (vector % 32);
+	if (level)
+		tmrptr[idx] |= mask;
+	else
+		tmrptr[idx] &= ~mask;
+
+	if (vlapic->ops.set_tmr != NULL)
+		(*vlapic->ops.set_tmr)(vlapic, vector, level);
+}
+
 void
 vlapic_reset_tmr(struct vlapic *vlapic)
 {
-	struct LAPIC *lapic;
+	int vector;
 
 	VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
 
-	lapic = vlapic->apic_page;
-	lapic->tmr0 = 0;
-	lapic->tmr1 = 0;
-	lapic->tmr2 = 0;
-	lapic->tmr3 = 0;
-	lapic->tmr4 = 0;
-	lapic->tmr5 = 0;
-	lapic->tmr6 = 0;
-	lapic->tmr7 = 0;
+	for (vector = 0; vector <= 255; vector++)
+		vlapic_set_tmr(vlapic, vector, false);
 }
 
 void
 vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
     int delmode, int vector)
 {
-	struct LAPIC *lapic;
-	uint32_t *tmrptr, mask;
 	cpuset_t dmask;
-	int idx;
 	bool lowprio;
 
 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
@@ -1502,11 +1513,6 @@
 	if (!CPU_ISSET(vlapic->vcpuid, &dmask))
 		return;
 
-	lapic = vlapic->apic_page;
-	tmrptr = &lapic->tmr0;
-	idx = (vector / 32) * 4;
-	mask = 1 << (vector % 32);
-	tmrptr[idx] |= mask;
-
 	VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
+	vlapic_set_tmr(vlapic, vector, true);
 }
Index: sys/amd64/vmm/io/vlapic_priv.h
===================================================================
--- sys/amd64/vmm/io/vlapic_priv.h	(revision 260830)
+++ sys/amd64/vmm/io/vlapic_priv.h	(working copy)
@@ -139,6 +139,7 @@
 	int (*pending_intr)(struct vlapic *vlapic, int *vecptr);
 	void (*intr_accepted)(struct vlapic *vlapic, int vector);
 	void (*post_intr)(struct vlapic *vlapic, int hostcpu);
+	void (*set_tmr)(struct vlapic *vlapic, int vector, bool level);
 };
 
 struct vlapic {
Index: sys/amd64/vmm/vmm.c
===================================================================
--- sys/amd64/vmm/vmm.c	(revision 260830)
+++ sys/amd64/vmm/vmm.c	(working copy)
@@ -1149,6 +1149,10 @@
 	if (error == 0) {
 		retu = false;
 		switch (vme->exitcode) {
+		case VM_EXITCODE_IOAPIC_EOI:
+			vioapic_process_eoi(vm, vcpuid,
+			    vme->u.ioapic_eoi.vector);
+			break;
 		case VM_EXITCODE_RENDEZVOUS:
 			vm_handle_rendezvous(vm, vcpuid);
 			error = 0;
Index: sys/kern/subr_witness.c
===================================================================
--- sys/kern/subr_witness.c	(revision 260830)
+++ sys/kern/subr_witness.c	(working copy)
@@ -132,7 +132,7 @@
 /* Define this to check for blessed mutexes */
 #undef BLESSING
 
-#define	WITNESS_COUNT 		1024
+#define	WITNESS_COUNT 		2048
 #define	WITNESS_CHILDCOUNT 	(WITNESS_COUNT * 4)
 #define	WITNESS_HASH_SIZE	251	/* Prime, gives load factor < 2 */
 #define	WITNESS_PENDLIST	1024
Index: usr.sbin/bhyve/legacy_irq.c
===================================================================
--- usr.sbin/bhyve/legacy_irq.c	(revision 260830)
+++ usr.sbin/bhyve/legacy_irq.c	(working copy)
@@ -58,7 +58,8 @@
 {
 	int i;
 
-	assert(irq < NLIRQ);
+	if (irq >= NLIRQ)
+		return (irq);
 
 	if (irq < 0) {
 		for (i = 0; i < NLIRQ; i++) {
Index: usr.sbin/bhyve/pci_virtio_block.c
===================================================================
--- usr.sbin/bhyve/pci_virtio_block.c	(revision 260830)
+++ usr.sbin/bhyve/pci_virtio_block.c	(working copy)
@@ -117,6 +117,7 @@
  */
 struct pci_vtblk_softc {
 	struct virtio_softc vbsc_vs;
+	pthread_mutex_t vsc_mtx;
 	struct vqueue_info vbsc_vq;
 	int		vbsc_fd;
 	struct vtblk_config vbsc_cfg;	
@@ -304,8 +305,12 @@
 	/* record fd of storage device/file */
 	sc->vbsc_fd = fd;
 
+	pthread_mutex_init(&sc->vsc_mtx, NULL);
+
 	/* init virtio softc and virtqueues */
 	vi_softc_linkup(&sc->vbsc_vs, &vtblk_vi_consts, sc, pi, &sc->vbsc_vq);
+	sc->vbsc_vs.vs_mtx = &sc->vsc_mtx;
+
 	sc->vbsc_vq.vq_qsize = VTBLK_RINGSZ;
 	/* sc->vbsc_vq.vq_notify = we have no per-queue notify */
 
@@ -339,6 +344,8 @@
 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
 	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_BLOCK);
 
+	pci_lintr_request(pi, 20);
+
 	if (vi_intr_init(&sc->vbsc_vs, 1, fbsdrun_virtio_msix()))
 		return (1);
 	vi_set_io_bar(&sc->vbsc_vs, 0);
Index: usr.sbin/bhyve/pci_virtio_net.c
===================================================================
--- usr.sbin/bhyve/pci_virtio_net.c	(revision 260830)
+++ usr.sbin/bhyve/pci_virtio_net.c	(working copy)
@@ -519,6 +519,8 @@
 	pthread_mutex_init(&sc->vsc_mtx, NULL);
 
 	vi_softc_linkup(&sc->vsc_vs, &vtnet_vi_consts, sc, pi, sc->vsc_queues);
+	sc->vsc_vs.vs_mtx = &sc->vsc_mtx;
+
 	sc->vsc_queues[VTNET_RXQ].vq_qsize = VTNET_RINGSZ;
 	sc->vsc_queues[VTNET_RXQ].vq_notify = pci_vtnet_ping_rxq;
 	sc->vsc_queues[VTNET_TXQ].vq_qsize = VTNET_RINGSZ;
@@ -608,6 +610,8 @@
 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_NETWORK);
 	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_NET);
 
+	pci_lintr_request(pi, 20);
+
 	/* link always up */
 	sc->vsc_config.status = 1;
 	
Index: usr.sbin/bhyve/virtio.c
===================================================================
--- usr.sbin/bhyve/virtio.c	(revision 260830)
+++ usr.sbin/bhyve/virtio.c	(working copy)
@@ -99,7 +99,11 @@
 	vs->vs_negotiated_caps = 0;
 	vs->vs_curq = 0;
 	/* vs->vs_status = 0; -- redundant */
+	VS_LOCK(vs);
+	if (vs->vs_isr)
+		pci_lintr_deassert(vs->vs_pi);
 	vs->vs_isr = 0;
+	VS_UNLOCK(vs);
 	vs->vs_msix_cfg_idx = VIRTIO_MSI_NO_VECTOR;
 }
 
@@ -139,8 +143,10 @@
 			return (1);
 	} else {
 		vs->vs_flags &= ~VIRTIO_USE_MSIX;
+#if 0
 		/* Only 1 MSI vector for bhyve */
 		pci_emul_add_msicap(vs->vs_pi, 1);
+#endif
 	}
 	return (0);
 }
@@ -591,6 +597,8 @@
 	case VTCFG_R_ISR:
 		value = vs->vs_isr;
 		vs->vs_isr = 0;		/* a read clears this flag */
+		if (value)
+			pci_lintr_deassert(pi);
 		break;
 	case VTCFG_R_CFGVEC:
 		value = vs->vs_msix_cfg_idx;
Index: usr.sbin/bhyve/virtio.h
===================================================================
--- usr.sbin/bhyve/virtio.h	(revision 260830)
+++ usr.sbin/bhyve/virtio.h	(working copy)
@@ -328,6 +328,18 @@
 	uint16_t vs_msix_cfg_idx;	/* MSI-X vector for config event */
 };
 
+#define	VS_LOCK(vs)							\
+do {									\
+	if (vs->vs_mtx)							\
+		pthread_mutex_lock(vs->vs_mtx);				\
+} while (0)
+
+#define	VS_UNLOCK(vs)							\
+do {									\
+	if (vs->vs_mtx)							\
+		pthread_mutex_unlock(vs->vs_mtx);			\
+} while (0)
+
 struct virtio_consts {
 	const char *vc_name;		/* name of driver (for diagnostics) */
 	int	vc_nvq;			/* number of virtual queues */
@@ -434,8 +446,10 @@
 	if (vs->vs_flags & VIRTIO_USE_MSIX)
 		pci_generate_msix(vs->vs_pi, vq->vq_msix_idx);
 	else {
+		VS_LOCK(vs);
 		vs->vs_isr |= VTCFG_ISR_QUEUES;
-		pci_generate_msi(vs->vs_pi, 0);
+		pci_lintr_assert(vs->vs_pi);
+		VS_UNLOCK(vs);
 	}
 }