Index: lib/libvmmapi/vmmapi.c
===================================================================
--- lib/libvmmapi/vmmapi.c	(revision 264996)
+++ lib/libvmmapi/vmmapi.c	(working copy)
@@ -349,6 +349,13 @@
 	return (ioctl(ctx->fd, VM_SUSPEND, 0));
 }
 
+int
+vm_reset(struct vmctx *ctx)
+{
+
+	return (ioctl(ctx->fd, VM_RESET, 0));
+}
+
 static int
 vm_inject_exception_real(struct vmctx *ctx, int vcpu, int vector,
     int error_code, int error_code_valid)
Index: lib/libvmmapi/vmmapi.h
===================================================================
--- lib/libvmmapi/vmmapi.h	(revision 264996)
+++ lib/libvmmapi/vmmapi.h	(working copy)
@@ -62,6 +62,7 @@
 int	vm_run(struct vmctx *ctx, int vcpu, uint64_t rip,
 	       struct vm_exit *ret_vmexit);
 int	vm_suspend(struct vmctx *ctx);
+int	vm_reset(struct vmctx *ctx);
 int	vm_apicid2vcpu(struct vmctx *ctx, int apicid);
 int	vm_inject_exception(struct vmctx *ctx, int vcpu, int vec);
 int	vm_inject_exception2(struct vmctx *ctx, int vcpu, int vec, int errcode);
Index: sys/amd64/include/vmm.h
===================================================================
--- sys/amd64/include/vmm.h	(revision 264996)
+++ sys/amd64/include/vmm.h	(working copy)
@@ -96,6 +96,7 @@
 
 int vm_create(const char *name, struct vm **retvm);
 void vm_destroy(struct vm *vm);
+int vm_reset(struct vm *vm);
 const char *vm_name(struct vm *vm);
 int vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len);
 int vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
@@ -134,6 +135,7 @@
 void vm_activate_cpu(struct vm *vm, int vcpu);
 cpuset_t vm_active_cpus(struct vm *vm);
 struct vm_exit *vm_exitinfo(struct vm *vm, int vcpuid);
+void vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip);
 
 /*
  * Rendezvous all vcpus specified in 'dest' and execute 'func(arg)'.
Index: sys/amd64/include/vmm_dev.h
===================================================================
--- sys/amd64/include/vmm_dev.h	(revision 264996)
+++ sys/amd64/include/vmm_dev.h	(working copy)
@@ -166,6 +166,7 @@
 	IOCNUM_SET_CAPABILITY = 2,
 	IOCNUM_GET_CAPABILITY = 3,
 	IOCNUM_SUSPEND = 4,
+	IOCNUM_RESET = 5,
 
 	/* memory apis */
 	IOCNUM_MAP_MEMORY = 10,
@@ -215,6 +216,8 @@
 	_IOWR('v', IOCNUM_RUN, struct vm_run)
 #define	VM_SUSPEND	\
 	_IO('v', IOCNUM_SUSPEND)
+#define	VM_RESET	\
+	_IO('v', IOCNUM_RESET)
 #define	VM_MAP_MEMORY	\
 	_IOWR('v', IOCNUM_MAP_MEMORY, struct vm_memory_segment)
 #define	VM_GET_MEMORY_SEG \
Index: sys/amd64/vmm/vmm.c
===================================================================
--- sys/amd64/vmm/vmm.c	(revision 264996)
+++ sys/amd64/vmm/vmm.c	(working copy)
@@ -83,28 +83,33 @@
 struct vlapic;
 
 struct vcpu {
-	int		flags;
+	struct mtx	mtx;
 	enum vcpu_state	state;
-	struct mtx	mtx;
-	int		hostcpu;	/* host cpuid this vcpu last ran on */
-	uint64_t	guest_msrs[VMM_MSR_NUM];
-	struct vlapic	*vlapic;
-	int		 vcpuid;
-	struct savefpu	*guestfpu;	/* guest fpu state */
-	uint64_t	guest_xcr0;
-	void		*stats;
-	struct vm_exit	exitinfo;
+	int		hostcpu;	/* host cpu this vcpu is running on */
+
+/* Cleared during vm_reset() */
+#define	vcpu_startzero	vlapic
+	struct vlapic	*vlapic;	/* APIC state */
 	enum x2apic_state x2apic_state;
-	int		nmi_pending;
+	struct vm_exit	exitinfo;	/* exit reason and collateral */
+	int		nmi_pending;	/* pending interrupts and exceptions */ 
 	int		extint_pending;
 	struct vm_exception exception;
 	int		exception_pending;
+	struct savefpu	*guestfpu;	/* guest fpu state */
+	uint64_t	guest_xcr0;
+	void		*stats;		/* statistics */
+	uint64_t	guest_msrs[VMM_MSR_NUM];
+/* Cleared during vm_reset() */
+#define	vcpu_endzero	endzero
+	int	endzero[0];
 };
 
 #define	vcpu_lock_init(v)	mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
 #define	vcpu_lock(v)		mtx_lock_spin(&((v)->mtx))
 #define	vcpu_unlock(v)		mtx_unlock_spin(&((v)->mtx))
 #define	vcpu_assert_locked(v)	mtx_assert(&((v)->mtx), MA_OWNED)
+#define	vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
 
 struct mem_seg {
 	vm_paddr_t	gpa;
@@ -115,17 +120,17 @@
 #define	VM_MAX_MEMORY_SEGMENTS	2
 
 struct vm {
+/*
+ * struct members from 'vm_startzero' to 'vm_endzero' are cleared and
+ * initialized in vm_init().
+ */
+#define	vm_startzero	cookie
 	void		*cookie;	/* processor-specific data */
 	void		*iommu;		/* iommu-specific data */
 	struct vhpet	*vhpet;		/* virtual HPET */
 	struct vioapic	*vioapic;	/* virtual ioapic */
 	struct vatpic	*vatpic;	/* virtual atpic */
 	struct vatpit	*vatpit;	/* virtual atpit */
-	struct vmspace	*vmspace;	/* guest's address space */
-	struct vcpu	vcpu[VM_MAXCPU];
-	int		num_mem_segs;
-	struct mem_seg	mem_segs[VM_MAX_MEMORY_SEGMENTS];
-	char		name[VM_MAX_NAMELEN];
 
 	/*
 	 * Set of active vcpus.
@@ -134,14 +139,20 @@
 	 */
 	volatile cpuset_t active_cpus;
 
-	struct mtx	rendezvous_mtx;
+	int		suspend;
+	volatile cpuset_t suspended_cpus;
+
 	cpuset_t	rendezvous_req_cpus;
 	cpuset_t	rendezvous_done_cpus;
 	void		*rendezvous_arg;
 	vm_rendezvous_func_t rendezvous_func;
-
-	int		suspend;
-	volatile cpuset_t suspended_cpus;
+#define	vm_endzero	rendezvous_mtx
+	struct mtx	rendezvous_mtx;
+	int		num_mem_segs;
+	struct mem_seg	mem_segs[VM_MAX_MEMORY_SEGMENTS];
+	struct vmspace	*vmspace;	/* guest's address space */
+	struct vcpu	vcpu[VM_MAXCPU];
+	char		name[VM_MAX_NAMELEN];
 };
 
 static int vmm_initialized;
@@ -207,12 +218,18 @@
 vcpu_init(struct vm *vm, uint32_t vcpu_id)
 {
 	struct vcpu *vcpu;
-	
+
 	vcpu = &vm->vcpu[vcpu_id];
 
-	vcpu_lock_init(vcpu);
-	vcpu->hostcpu = NOCPU;
-	vcpu->vcpuid = vcpu_id;
+	if (!vcpu_lock_initialized(vcpu)) {
+		vcpu_lock_init(vcpu);
+		vcpu->state = VCPU_IDLE;
+		vcpu->hostcpu = NOCPU;
+	}
+
+	bzero(&vcpu->vcpu_startzero,
+	    __rangeof(struct vcpu, vcpu_startzero, vcpu_endzero));
+
 	vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id);
 	vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
 	vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
@@ -323,15 +340,35 @@
 DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY);
 MODULE_VERSION(vmm, 1);
 
+static void
+vm_init(struct vm *vm)
+{
+	int i;
+	const int BSP = 0;
+
+	bzero(&vm->vm_startzero,
+	    __rangeof(struct vm, vm_startzero, vm_endzero));
+
+	vm->cookie = VMINIT(vm, vmspace_pmap(vm->vmspace));
+	vm->vioapic = vioapic_init(vm);
+	vm->vhpet = vhpet_init(vm);
+	vm->vatpic = vatpic_init(vm);
+	vm->vatpit = vatpit_init(vm);
+
+	for (i = 0; i < VM_MAXCPU; i++) {
+		vcpu_init(vm, i);
+		guest_msrs_init(vm, i);
+	}
+
+	vm_activate_cpu(vm, BSP);
+}
+
 int
 vm_create(const char *name, struct vm **retvm)
 {
-	int i;
 	struct vm *vm;
 	struct vmspace *vmspace;
 
-	const int BSP = 0;
-
 	/*
 	 * If vmm.ko could not be successfully initialized then don't attempt
 	 * to create the virtual machine.
@@ -350,19 +387,9 @@
 	strcpy(vm->name, name);
 	vm->vmspace = vmspace;
 	mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF);
-	vm->cookie = VMINIT(vm, vmspace_pmap(vmspace));
-	vm->vioapic = vioapic_init(vm);
-	vm->vhpet = vhpet_init(vm);
-	vm->vatpic = vatpic_init(vm);
-	vm->vatpit = vatpit_init(vm);
 
-	for (i = 0; i < VM_MAXCPU; i++) {
-		vcpu_init(vm, i);
-		guest_msrs_init(vm, i);
-	}
+	vm_init(vm);
 
-	vm_activate_cpu(vm, BSP);
-
 	*retvm = vm;
 	return (0);
 }
@@ -377,36 +404,65 @@
 	bzero(seg, sizeof(*seg));
 }
 
-void
-vm_destroy(struct vm *vm)
+static void
+vm_cleanup(struct vm *vm, bool freemem)
 {
 	int i;
 
 	ppt_unassign_all(vm);
 
-	if (vm->iommu != NULL)
+	if (vm->iommu != NULL) {
 		iommu_destroy_domain(vm->iommu);
+		vm->iommu = NULL;
+	}
 
 	vatpit_cleanup(vm->vatpit);
 	vhpet_cleanup(vm->vhpet);
 	vatpic_cleanup(vm->vatpic);
 	vioapic_cleanup(vm->vioapic);
 
-	for (i = 0; i < vm->num_mem_segs; i++)
-		vm_free_mem_seg(vm, &vm->mem_segs[i]);
-
-	vm->num_mem_segs = 0;
-
 	for (i = 0; i < VM_MAXCPU; i++)
 		vcpu_cleanup(vm, i);
 
-	VMSPACE_FREE(vm->vmspace);
-
 	VMCLEANUP(vm->cookie);
 
+	if (freemem) {
+		for (i = 0; i < vm->num_mem_segs; i++)
+			vm_free_mem_seg(vm, &vm->mem_segs[i]);
+
+		vm->num_mem_segs = 0;
+
+		VMSPACE_FREE(vm->vmspace);
+		vm->vmspace = NULL;
+	}
+}
+
+void
+vm_destroy(struct vm *vm)
+{
+	vm_cleanup(vm, true);
 	free(vm, M_VM);
 }
 
+int
+vm_reset(struct vm *vm)
+{
+	int error;
+
+	/*
+	 * A virtual machine can be reset only if all vcpus are suspended.
+	 */
+	if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
+		vm_cleanup(vm, false);
+		vm_init(vm);
+		error = 0;
+	} else {
+		error = EBUSY;
+	}
+
+	return (error);
+}
+
 const char *
 vm_name(struct vm *vm)
 {
Index: sys/amd64/vmm/vmm_dev.c
===================================================================
--- sys/amd64/vmm/vmm_dev.c	(revision 264996)
+++ sys/amd64/vmm/vmm_dev.c	(working copy)
@@ -211,6 +211,7 @@
 	case VM_BIND_PPTDEV:
 	case VM_UNBIND_PPTDEV:
 	case VM_MAP_MEMORY:
+	case VM_RESET:
 		/*
 		 * ioctls that operate on the entire virtual machine must
 		 * prevent all vcpus from running.
@@ -243,6 +244,9 @@
 	case VM_SUSPEND:
 		error = vm_suspend(sc->vm);
 		break;
+	case VM_RESET:
+		error = vm_reset(sc->vm);
+		break;
 	case VM_STAT_DESC: {
 		statdesc = (struct vm_stat_desc *)data;
 		error = vmm_stat_desc_copy(statdesc->index,
Index: usr.sbin/bhyveload/bhyveload.c
===================================================================
--- usr.sbin/bhyveload/bhyveload.c	(revision 264996)
+++ usr.sbin/bhyveload/bhyveload.c	(working copy)
@@ -642,7 +642,7 @@
 	void *h;
 	void (*func)(struct loader_callbacks *, void *, int, int);
 	uint64_t mem_size;
-	int opt, error;
+	int opt, error, need_reset;
 
 	progname = basename(argv[0]);
 
@@ -691,11 +691,14 @@
 
 	vmname = argv[0];
 
+	need_reset = 0;
 	error = vm_create(vmname);
-	if (error != 0 && errno != EEXIST) {
-		perror("vm_create");
-		exit(1);
-
+	if (error) {
+		if (errno != EEXIST) {
+			perror("vm_create");
+			exit(1);
+		}
+		need_reset = 1;
 	}
 
 	ctx = vm_open(vmname);
@@ -704,6 +707,14 @@
 		exit(1);
 	}
 
+	if (need_reset) {
+		error = vm_reset(ctx);
+		if (error) {
+			perror("vm_reset");
+			exit(1);
+		}
+	}
+
 	error = vm_setup_memory(ctx, mem_size, VM_MMAP_ALL);
 	if (error) {
 		perror("vm_setup_memory");