From 4a33a283049b0ef76aebb7fbe37e53e40942dccc Mon Sep 17 00:00:00 2001
From: Ryan Stone <rstone@FreeBSD.org>
Date: Wed, 21 May 2014 18:23:57 -0400
Subject: [PATCH 10/21] Add interface to destroy SR-IOV VFs

---
 sys/dev/pci/pci_iov.c         | 113 +++++++++++++++++++++++++++++++++++++++++-
 sys/dev/pci/pci_iov_private.h |   1 +
 sys/sys/iov.h                 |   1 +
 3 files changed, 113 insertions(+), 2 deletions(-)

diff --git a/sys/dev/pci/pci_iov.c b/sys/dev/pci/pci_iov.c
index 397ff65..5a5b3ac 100755
--- a/sys/dev/pci/pci_iov.c
+++ b/sys/dev/pci/pci_iov.c
@@ -143,7 +143,7 @@ pci_iov_detach_method(device_t bus, device_t dev)
 		return (0);
 	}
 
-	if (iov->iov_num_vfs != 0) {
+	if (iov->iov_num_vfs != 0 || iov->iov_flags & IOV_BUSY) {
 		mtx_unlock(&Giant);
 		return (EBUSY);
 	}
@@ -398,10 +398,11 @@ pci_iov_config(struct cdev *cdev, struct pci_iov_arg *arg)
 	bus = device_get_parent(dev);
 	iov_inited = 0;
 
-	if (iov->iov_num_vfs != 0) {
+	if ((iov->iov_flags & IOV_BUSY) || iov->iov_num_vfs != 0) {
 		mtx_unlock(&Giant);
 		return (EBUSY);
 	}
+	iov->iov_flags |= IOV_BUSY;
 
 	total_vfs = IOV_READ(dinfo, PCIR_SRIOV_TOTAL_VFS, 2);
 
@@ -491,10 +492,116 @@ out:
 		iov->iov_flags &= ~IOV_RMAN_INITED;
 	}
 	iov->iov_num_vfs = 0;
+	iov->iov_flags &= ~IOV_BUSY;
 	mtx_unlock(&Giant);
 	return (error);
 }
 
+/* Return true if child is a VF of the given PF. */
+static int
+pci_iov_is_child_vf(struct pcicfg_iov *pf, device_t child)
+{
+	struct pci_devinfo *vfinfo;
+
+	vfinfo = device_get_ivars(child);
+
+	if (!(vfinfo->cfg.flags & PCICFG_VF))
+		return (0);
+
+	return (pf == vfinfo->cfg.iov);
+}
+
+static int
+pci_iov_delete(struct cdev *cdev)
+{
+	device_t bus, dev, vf, *devlist;
+	struct pci_devinfo *dinfo;
+	struct pcicfg_iov *iov;
+	int i, error, devcount;
+	uint32_t iov_ctl;
+
+	mtx_lock(&Giant);
+	dinfo = cdev->si_drv1;
+	iov = dinfo->cfg.iov;
+	dev = dinfo->cfg.dev;
+	bus = device_get_parent(dev);
+	devlist = NULL;
+
+	if (iov->iov_flags & IOV_BUSY) {
+		mtx_unlock(&Giant);
+		return (EBUSY);
+	}
+
+	if (iov->iov_num_vfs == 0) {
+		mtx_unlock(&Giant);
+		return (ECHILD);
+	}
+
+	iov->iov_flags |= IOV_BUSY;
+
+	error = device_get_children(bus, &devlist, &devcount);
+
+	if (error != 0)
+		goto out;
+
+	for (i = 0; i < devcount; i++) {
+		vf = devlist[i];
+
+		if (!pci_iov_is_child_vf(iov, vf))
+			continue;
+
+		error = device_detach(vf);
+		if (error != 0) {
+			/*
+			 * If any device fails to detach, then re-attach all
+			 * VFs to ensure that we leave things in the same state
+			 * that we started in.
+			 */
+			bus_generic_attach(bus);
+			goto out;
+		}
+	}
+
+	for (i = 0; i < devcount; i++) {
+		vf = devlist[i];
+
+		if (pci_iov_is_child_vf(iov, vf))
+			pci_delete_child(bus, vf);
+	}
+	PCI_UNINIT_IOV(dev);
+
+	iov_ctl = IOV_READ(dinfo, PCIR_SRIOV_CTL, 2);
+	iov_ctl &= ~(PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE);
+	IOV_WRITE(dinfo, PCIR_SRIOV_CTL, iov_ctl, 2);
+	IOV_WRITE(dinfo, PCIR_SRIOV_NUM_VFS, 0, 2);
+
+	iov->iov_num_vfs = 0;
+
+	for (i = 0; i <= PCIR_MAX_BAR_0; i++) {
+		if (iov->iov_bar[i].res != NULL) {
+			pci_release_resource(bus, dev, SYS_RES_MEMORY,
+			    iov->iov_pos + PCIR_SRIOV_BAR(i),
+			    iov->iov_bar[i].res);
+			pci_delete_resource(bus, dev, SYS_RES_MEMORY,
+			    iov->iov_pos + PCIR_SRIOV_BAR(i));
+			iov->iov_bar[i].res = NULL;
+		}
+	}
+
+	if (iov->iov_flags & IOV_RMAN_INITED) {
+		rman_fini(&iov->rman);
+		iov->iov_flags &= ~IOV_RMAN_INITED;
+	}
+
+	error = 0;
+out:
+	free(devlist, M_TEMP);
+	iov->iov_flags &= ~IOV_BUSY;
+	mtx_unlock(&Giant);
+	return (error);
+}
+
+
 static int
 pci_iov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
     struct thread *td)
@@ -503,6 +610,8 @@ pci_iov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
 	switch (cmd) {
 	case IOV_CONFIG:
 		return (pci_iov_config(dev, (struct pci_iov_arg *)data));
+	case IOV_DELETE:
+		return (pci_iov_delete(dev));
 	default:
 		return (EINVAL);
 	}
diff --git a/sys/dev/pci/pci_iov_private.h b/sys/dev/pci/pci_iov_private.h
index d4b81e4..e5f594c 100755
--- a/sys/dev/pci/pci_iov_private.h
+++ b/sys/dev/pci/pci_iov_private.h
@@ -48,6 +48,7 @@ struct pcicfg_iov {
 };
 
 #define	IOV_RMAN_INITED	(1 << 0)
+#define	IOV_BUSY	(1 << 1)
 
 #endif
 
diff --git a/sys/sys/iov.h b/sys/sys/iov.h
index a77dcea..c0449ec 100755
--- a/sys/sys/iov.h
+++ b/sys/sys/iov.h
@@ -38,6 +38,7 @@ struct pci_iov_arg
 };
 
 #define	IOV_CONFIG	_IOWR('p', 10, struct pci_iov_arg)
+#define	IOV_DELETE	_IO('p', 11)
 
 #endif
 
-- 
1.9.2