commit 1ab8c15e498b887704e4f380c4430d0fd42fc49b Author: Bryan Venteicher Date: Sun Aug 4 18:45:32 2013 -0500 Add VMWare vmxnet3 driver diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index c898ef8..51adcf3 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -304,6 +304,7 @@ dev/tpm/tpm_acpi.c optional tpm acpi dev/tpm/tpm_isa.c optional tpm isa dev/uart/uart_cpu_x86.c optional uart dev/viawd/viawd.c optional viawd +dev/vmware/vmxnet3/if_vmx.c optional vmx pci dev/wbwd/wbwd.c optional wbwd dev/wpi/if_wpi.c optional wpi dev/isci/isci.c optional isci diff --git a/sys/conf/files.i386 b/sys/conf/files.i386 index 9a27bde..b6ae5ed 100644 --- a/sys/conf/files.i386 +++ b/sys/conf/files.i386 @@ -267,6 +267,7 @@ dev/tpm/tpm_acpi.c optional tpm acpi dev/tpm/tpm_isa.c optional tpm isa dev/uart/uart_cpu_x86.c optional uart dev/viawd/viawd.c optional viawd +dev/vmware/vmxnet3/if_vmx.c optional vmx pci dev/acpica/acpi_if.m standard dev/acpi_support/acpi_wmi_if.m standard dev/wbwd/wbwd.c optional wbwd diff --git a/sys/dev/vmware/vmxnet3/if_vmx.c b/sys/dev/vmware/vmxnet3/if_vmx.c new file mode 100644 index 0000000..b231b98 --- /dev/null +++ b/sys/dev/vmware/vmxnet3/if_vmx.c @@ -0,0 +1,3265 @@ +/*- + * Copyright (c) 2013 Tsubai Masanari + * Copyright (c) 2013 Bryan Venteicher + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ + */ + +/* Driver for VMware vmxnet3 virtual ethernet devices. */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include "if_vmxreg.h" +#include "if_vmxvar.h" + +#include "opt_inet.h" +#include "opt_inet6.h" + +/* Always enable for now - useful for queue hangs. */ +#define VMXNET3_DEBUG_SYSCTL + +#ifdef VMXNET3_FAILPOINTS +#include +static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, + "vmxnet3 fail points"); +#define VMXNET3_FP _debug_fail_point_vmxnet3 +#endif + +static int vmxnet3_probe(device_t); +static int vmxnet3_attach(device_t); +static int vmxnet3_detach(device_t); +static int vmxnet3_shutdown(device_t); + +static int vmxnet3_alloc_resources(struct vmxnet3_softc *); +static void vmxnet3_free_resources(struct vmxnet3_softc *); +static int vmxnet3_check_version(struct vmxnet3_softc *); +static void vmxnet3_initial_config(struct vmxnet3_softc *); + +static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); +static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); +static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); +static int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, + struct vmxnet3_interrupt *); +static int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); +static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); +static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); +static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); +static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); + +static void vmxnet3_free_interrupt(struct vmxnet3_softc *, + struct vmxnet3_interrupt *); +static void vmxnet3_free_interrupts(struct vmxnet3_softc *); + +static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); +static int vmxnet3_init_txq(struct vmxnet3_softc *, int); +static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); +static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); +static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); +static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); + +static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); +static void vmxnet3_free_shared_data(struct vmxnet3_softc *); +static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); +static void vmxnet3_free_txq_data(struct vmxnet3_softc *); +static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); +static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); +static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); +static void vmxnet3_free_queue_data(struct vmxnet3_softc *); +static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); +static void vmxnet3_init_shared_data(struct vmxnet3_softc *); +static void vmxnet3_reinit_interface(struct vmxnet3_softc *); +static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); +static int vmxnet3_alloc_data(struct vmxnet3_softc *); +static void vmxnet3_free_data(struct vmxnet3_softc *); +static int vmxnet3_setup_interface(struct vmxnet3_softc *); + +static void vmxnet3_evintr(struct vmxnet3_softc *); +static void vmxnet3_txq_eof(struct vmxnet3_txqueue *); +static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); +static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); +static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, + struct vmxnet3_rxring *, int); +static void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); +static void vmxnet3_legacy_intr(void *); +static void vmxnet3_txq_intr(void *); +static void vmxnet3_rxq_intr(void *); +static void vmxnet3_event_intr(void *); + +static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); +static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); +static void vmxnet3_stop(struct vmxnet3_softc *); + +static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); +static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); +static int vmxnet3_reinit_queues(struct vmxnet3_softc *); +static int vmxnet3_enable_device(struct vmxnet3_softc *); +static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); +static int vmxnet3_reinit(struct vmxnet3_softc *); +static void vmxnet3_init_locked(struct vmxnet3_softc *); +static void vmxnet3_init(void *); + +static int vmxnet3_txq_offload_ctx(struct mbuf *, int *, int *, int *); +static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, + bus_dmamap_t, bus_dma_segment_t [], int *); +static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); +static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); +static void vmxnet3_start_locked(struct ifnet *); +static void vmxnet3_start(struct ifnet *); + +static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, + uint16_t); +static void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); +static void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); +static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); +static int vmxnet3_change_mtu(struct vmxnet3_softc *, int); +static int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); + +static int vmxnet3_watchdog(struct vmxnet3_txqueue *); +static void vmxnet3_tick(void *); +static void vmxnet3_link_status(struct vmxnet3_softc *); +static void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); +static int vmxnet3_media_change(struct ifnet *); +static void vmxnet3_set_lladdr(struct vmxnet3_softc *); +static void vmxnet3_get_lladdr(struct vmxnet3_softc *); + +static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, + struct sysctl_ctx_list *, struct sysctl_oid_list *); +static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, + struct sysctl_ctx_list *, struct sysctl_oid_list *); +static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, + struct sysctl_ctx_list *, struct sysctl_oid_list *); +static void vmxnet3_setup_sysctl(struct vmxnet3_softc *); + +static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, + uint32_t); +static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); +static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, + uint32_t); +static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); +static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); + +static void vmxnet3_enable_intr(struct vmxnet3_softc *, int); +static void vmxnet3_disable_intr(struct vmxnet3_softc *, int); +static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); +static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); + +static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, + bus_size_t, struct vmxnet3_dma_alloc *); +static void vmxnet3_dma_free(struct vmxnet3_softc *, + struct vmxnet3_dma_alloc *); + +typedef enum { + VMXNET3_BARRIER_RD, + VMXNET3_BARRIER_WR, + VMXNET3_BARRIER_RDWR, +} vmxnet3_barrier_t; + +static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); + +static device_method_t vmxnet3_methods[] = { + /* Device interface. */ + DEVMETHOD(device_probe, vmxnet3_probe), + DEVMETHOD(device_attach, vmxnet3_attach), + DEVMETHOD(device_detach, vmxnet3_detach), + DEVMETHOD(device_shutdown, vmxnet3_shutdown), + + DEVMETHOD_END +}; + +static driver_t vmxnet3_driver = { + "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) +}; + +static devclass_t vmxnet3_devclass; +DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); + +MODULE_DEPEND(vmx, pci, 1, 1, 1); +MODULE_DEPEND(vmx, ether, 1, 1, 1); + +#define VMXNET3_VMWARE_VENDOR_ID 0x15AD +#define VMXNET3_VMWARE_DEVICE_ID 0x07B0 + +static int +vmxnet3_probe(device_t dev) +{ + + if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && + pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { + device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); + return (BUS_PROBE_DEFAULT); + } + + return (ENXIO); +} + +static int +vmxnet3_attach(device_t dev) +{ + struct vmxnet3_softc *sc; + int error; + + sc = device_get_softc(dev); + sc->vmx_dev = dev; + + pci_enable_busmaster(dev); + + VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); + callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0); + + vmxnet3_initial_config(sc); + + error = vmxnet3_alloc_resources(sc); + if (error) + goto fail; + + error = vmxnet3_check_version(sc); + if (error) + goto fail; + + error = vmxnet3_alloc_rxtx_queues(sc); + if (error) + goto fail; + + error = vmxnet3_alloc_interrupts(sc); + if (error) + goto fail; + + error = vmxnet3_alloc_data(sc); + if (error) + goto fail; + + error = vmxnet3_setup_interface(sc); + if (error) + goto fail; + + error = vmxnet3_setup_interrupts(sc); + if (error) { + ether_ifdetach(sc->vmx_ifp); + device_printf(dev, "could not set up interrupt\n"); + goto fail; + } + + vmxnet3_setup_sysctl(sc); + vmxnet3_link_status(sc); + +fail: + if (error) + vmxnet3_detach(dev); + + return (error); +} + +static int +vmxnet3_detach(device_t dev) +{ + struct vmxnet3_softc *sc; + struct ifnet *ifp; + + sc = device_get_softc(dev); + ifp = sc->vmx_ifp; + + if (device_is_attached(dev)) { + ether_ifdetach(ifp); + VMXNET3_CORE_LOCK(sc); + vmxnet3_stop(sc); + VMXNET3_CORE_UNLOCK(sc); + callout_drain(&sc->vmx_tick); + } + + if (sc->vmx_vlan_attach != NULL) { + EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); + sc->vmx_vlan_attach = NULL; + } + if (sc->vmx_vlan_detach != NULL) { + EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); + sc->vmx_vlan_detach = NULL; + } + + vmxnet3_free_interrupts(sc); + + if (ifp != NULL) { + if_free(ifp); + sc->vmx_ifp = NULL; + } + + ifmedia_removeall(&sc->vmx_media); + + vmxnet3_free_data(sc); + vmxnet3_free_resources(sc); + vmxnet3_free_rxtx_queues(sc); + + VMXNET3_CORE_LOCK_DESTROY(sc); + + return (0); +} + +static int +vmxnet3_shutdown(device_t dev) +{ + + return (0); +} + +static int +vmxnet3_alloc_resources(struct vmxnet3_softc *sc) +{ + device_t dev; + int rid; + + dev = sc->vmx_dev; + + rid = PCIR_BAR(0); + sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (sc->vmx_res0 == NULL) { + device_printf(dev, + "could not map BAR0 memory\n"); + return (ENXIO); + } + + sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); + sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); + + rid = PCIR_BAR(1); + sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (sc->vmx_res1 == NULL) { + device_printf(dev, + "could not map BAR1 memory\n"); + return (ENXIO); + } + + sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); + sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); + + if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { + rid = PCIR_BAR(2); + sc->vmx_msix_res = bus_alloc_resource_any(dev, + SYS_RES_MEMORY, &rid, RF_ACTIVE); + } + + if (sc->vmx_msix_res == NULL) + sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; + + return (0); +} + +static void +vmxnet3_free_resources(struct vmxnet3_softc *sc) +{ + device_t dev; + int rid; + + dev = sc->vmx_dev; + + if (sc->vmx_res0 != NULL) { + rid = PCIR_BAR(0); + bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); + sc->vmx_res0 = NULL; + } + + if (sc->vmx_res1 != NULL) { + rid = PCIR_BAR(1); + bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); + sc->vmx_res1 = NULL; + } + + if (sc->vmx_msix_res != NULL) { + rid = PCIR_BAR(2); + bus_release_resource(dev, SYS_RES_MEMORY, rid, + sc->vmx_msix_res); + sc->vmx_msix_res = NULL; + } +} + +static int +vmxnet3_check_version(struct vmxnet3_softc *sc) +{ + device_t dev; + uint32_t version; + + dev = sc->vmx_dev; + + version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); + if ((version & 0x01) == 0) { + device_printf(dev, "unsupported hardware version %#x\n", + version); + return (ENOTSUP); + } else + vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); + + version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); + if ((version & 0x01) == 0) { + device_printf(dev, "unsupported UPT version %#x\n", version); + return (ENOTSUP); + } else + vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); + + return (0); +} + +static void +vmxnet3_initial_config(struct vmxnet3_softc *sc) +{ + + sc->vmx_ntxqueues = 1; + sc->vmx_nrxqueues = 1; + sc->vmx_ntxdescs = VMXNET3_MAX_TX_NDESC; + sc->vmx_nrxdescs = VMXNET3_MAX_RX_NDESC; + sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; +} + +static int +vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) +{ + device_t dev; + int nmsix, cnt, required; + + dev = sc->vmx_dev; + + if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) + return (1); + + /* Allocate an additional vector for the events interrupt. */ + required = sc->vmx_nrxqueues + sc->vmx_ntxqueues + 1; + + nmsix = pci_msix_count(dev); + if (nmsix < required) + return (1); + + cnt = required; + if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { + sc->vmx_nintrs = required; + return (0); + } else + pci_release_msi(dev); + + return (1); +} + +static int +vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) +{ + device_t dev; + int nmsi, cnt, required; + + dev = sc->vmx_dev; + required = 1; + + nmsi = pci_msi_count(dev); + if (nmsi < required) + return (1); + + cnt = required; + if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { + sc->vmx_nintrs = 1; + return (0); + } else + pci_release_msi(dev); + + return (1); +} + +static int +vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) +{ + + sc->vmx_nintrs = 1; + return (0); +} + +static int +vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, + struct vmxnet3_interrupt *intr) +{ + struct resource *irq; + + irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags); + if (irq == NULL) + return (ENXIO); + + intr->vmxi_irq = irq; + intr->vmxi_rid = rid; + + return (0); +} + +static int +vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) +{ + int i, rid, flags, error; + + rid = 0; + flags = RF_ACTIVE; + + if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) + flags |= RF_SHAREABLE; + else + rid = 1; + + for (i = 0; i < sc->vmx_nintrs; i++, rid++) { + error = vmxnet3_alloc_interrupt(sc, rid, flags, + &sc->vmx_intrs[i]); + if (error) + return (error); + } + + return (0); +} + +/* + * NOTE: We only support the simple case of each Rx and Tx queue on its + * own MSIX vector. This is good enough until we support mulitqueue. + */ +static int +vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) +{ + device_t dev; + struct vmxnet3_txqueue *txq; + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_interrupt *intr; + enum intr_type type; + int i, error; + + dev = sc->vmx_dev; + intr = &sc->vmx_intrs[0]; + type = INTR_TYPE_NET | INTR_MPSAFE; + + for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { + txq = &sc->vmx_txq[i]; + error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, + vmxnet3_txq_intr, txq, &intr->vmxi_handler); + if (error) + return (error); + txq->vxtxq_intr_idx = intr->vmxi_rid - 1; + } + + for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { + rxq = &sc->vmx_rxq[i]; + error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, + vmxnet3_rxq_intr, rxq, &intr->vmxi_handler); + if (error) + return (error); + rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; + } + + error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, + vmxnet3_event_intr, sc, &intr->vmxi_handler); + if (error) + return (error); + sc->vmx_event_intr_idx = intr->vmxi_rid - 1; + + return (0); +} + +static int +vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) +{ + struct vmxnet3_interrupt *intr; + int i, error; + + intr = &sc->vmx_intrs[0]; + error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, + INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc, + &intr->vmxi_handler); + + for (i = 0; i < sc->vmx_ntxqueues; i++) + sc->vmx_txq[i].vxtxq_intr_idx = 0; + for (i = 0; i < sc->vmx_nrxqueues; i++) + sc->vmx_rxq[i].vxrxq_intr_idx = 0; + sc->vmx_event_intr_idx = 0; + + return (error); +} + +/* + * XXX BMV Should probably reorganize the attach and just do + * this in vmxnet3_init_shared_data(). + */ +static void +vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) +{ + struct vmxnet3_txqueue *txq; + struct vmxnet3_txq_shared *txs; + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_rxq_shared *rxs; + int i; + + sc->vmx_ds->evintr = sc->vmx_event_intr_idx; + + for (i = 0; i < sc->vmx_ntxqueues; i++) { + txq = &sc->vmx_txq[i]; + txs = txq->vxtxq_ts; + txs->intr_idx = txq->vxtxq_intr_idx; + } + + for (i = 0; i < sc->vmx_nrxqueues; i++) { + rxq = &sc->vmx_rxq[i]; + rxs = rxq->vxrxq_rs; + rxs->intr_idx = rxq->vxrxq_intr_idx; + } +} + +static int +vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) +{ + int error; + + error = vmxnet3_alloc_intr_resources(sc); + if (error) + return (error); + + switch (sc->vmx_intr_type) { + case VMXNET3_IT_MSIX: + error = vmxnet3_setup_msix_interrupts(sc); + break; + case VMXNET3_IT_MSI: + case VMXNET3_IT_LEGACY: + error = vmxnet3_setup_legacy_interrupt(sc); + break; + default: + panic("%s: invalid interrupt type %d", __func__, + sc->vmx_intr_type); + } + + if (error == 0) + vmxnet3_set_interrupt_idx(sc); + + return (error); +} + +static int +vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) +{ + device_t dev; + uint32_t config; + int error; + + dev = sc->vmx_dev; + config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); + + sc->vmx_intr_type = config & 0x03; + sc->vmx_intr_mask_mode = (config >> 2) & 0x03; + + switch (sc->vmx_intr_type) { + case VMXNET3_IT_AUTO: + sc->vmx_intr_type = VMXNET3_IT_MSIX; + /* FALLTHROUGH */ + case VMXNET3_IT_MSIX: + error = vmxnet3_alloc_msix_interrupts(sc); + if (error == 0) + break; + sc->vmx_intr_type = VMXNET3_IT_MSI; + /* FALLTHROUGH */ + case VMXNET3_IT_MSI: + error = vmxnet3_alloc_msi_interrupts(sc); + if (error == 0) + break; + sc->vmx_intr_type = VMXNET3_IT_LEGACY; + /* FALLTHROUGH */ + case VMXNET3_IT_LEGACY: + error = vmxnet3_alloc_legacy_interrupts(sc); + if (error == 0) + break; + /* FALLTHROUGH */ + default: + sc->vmx_intr_type = -1; + device_printf(dev, "cannot allocate any interrupt resources\n"); + return (ENXIO); + } + + return (error); +} + +static void +vmxnet3_free_interrupt(struct vmxnet3_softc *sc, + struct vmxnet3_interrupt *intr) +{ + device_t dev; + + dev = sc->vmx_dev; + + if (intr->vmxi_handler != NULL) { + bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); + intr->vmxi_handler = NULL; + } + + if (intr->vmxi_irq != NULL) { + bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, + intr->vmxi_irq); + intr->vmxi_irq = NULL; + intr->vmxi_rid = -1; + } +} + +static void +vmxnet3_free_interrupts(struct vmxnet3_softc *sc) +{ + int i; + + for (i = 0; i < sc->vmx_nintrs; i++) + vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); + + if (sc->vmx_intr_type == VMXNET3_IT_MSI || + sc->vmx_intr_type == VMXNET3_IT_MSIX) + pci_release_msi(sc->vmx_dev); +} + +static int +vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) +{ + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_rxring *rxr; + int i; + + rxq = &sc->vmx_rxq[q]; + + snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", + device_get_nameunit(sc->vmx_dev), q); + mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF); + + rxq->vxrxq_sc = sc; + rxq->vxrxq_id = q; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + rxr->vxrxr_rid = i; + rxr->vxrxr_ndesc = sc->vmx_nrxdescs; + rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc * + sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO); + if (rxr->vxrxr_rxbuf == NULL) + return (ENOMEM); + } + + rxq->vxrxq_comp_ring.vxcr_ndesc = + sc->vmx_nrxdescs * VMXNET3_RXRINGS_PERQ; + + return (0); +} + +static int +vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) +{ + struct vmxnet3_txqueue *txq; + struct vmxnet3_txring *txr; + + txq = &sc->vmx_txq[q]; + txr = &txq->vxtxq_cmd_ring; + + snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", + device_get_nameunit(sc->vmx_dev), q); + mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF); + + txq->vxtxq_sc = sc; + txq->vxtxq_id = q; + + txr->vxtxr_ndesc = sc->vmx_ntxdescs; + txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc * + sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO); + if (txr->vxtxr_txbuf == NULL) + return (ENOMEM); + + txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; + + return (0); +} + +static int +vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) +{ + int i, error; + + sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * + sc->vmx_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); + sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * + sc->vmx_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); + if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) + return (ENOMEM); + + for (i = 0; i < sc->vmx_nrxqueues; i++) { + error = vmxnet3_init_rxq(sc, i); + if (error) + return (error); + } + + for (i = 0; i < sc->vmx_ntxqueues; i++) { + error = vmxnet3_init_txq(sc, i); + if (error) + return (error); + } + + return (0); +} + +static void +vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) +{ + struct vmxnet3_rxring *rxr; + int i; + + rxq->vxrxq_sc = NULL; + rxq->vxrxq_id = -1; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + if (rxr->vxrxr_rxbuf != NULL) { + free(rxr->vxrxr_rxbuf, M_DEVBUF); + rxr->vxrxr_rxbuf = NULL; + } + } + + if (mtx_initialized(&rxq->vxrxq_mtx) != 0) + mtx_destroy(&rxq->vxrxq_mtx); +} + +static void +vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) +{ + struct vmxnet3_txring *txr; + + txr = &txq->vxtxq_cmd_ring; + + txq->vxtxq_sc = NULL; + txq->vxtxq_id = -1; + + if (txr->vxtxr_txbuf != NULL) { + free(txr->vxtxr_txbuf, M_DEVBUF); + txr->vxtxr_txbuf = NULL; + } + + if (mtx_initialized(&txq->vxtxq_mtx) != 0) + mtx_destroy(&txq->vxtxq_mtx); +} + +static void +vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) +{ + int i; + + if (sc->vmx_rxq != NULL) { + for (i = 0; i < sc->vmx_nrxqueues; i++) + vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); + free(sc->vmx_rxq, M_DEVBUF); + sc->vmx_rxq = NULL; + } + + if (sc->vmx_txq != NULL) { + for (i = 0; i < sc->vmx_ntxqueues; i++) + vmxnet3_destroy_txq(&sc->vmx_txq[i]); + free(sc->vmx_txq, M_DEVBUF); + sc->vmx_txq = NULL; + } +} + +static int +vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) +{ + device_t dev; + uint8_t *kva; + size_t size; + int i, error; + + dev = sc->vmx_dev; + + size = sizeof(struct vmxnet3_driver_shared); + error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); + if (error) { + device_printf(dev, "cannot alloc shared memory\n"); + return (error); + } + sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; + + size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + + sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); + error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); + if (error) { + device_printf(dev, "cannot alloc queue shared memory\n"); + return (error); + } + sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; + kva = sc->vmx_qs; + + for (i = 0; i < sc->vmx_ntxqueues; i++) { + sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; + kva += sizeof(struct vmxnet3_txq_shared); + } + for (i = 0; i < sc->vmx_nrxqueues; i++) { + sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; + kva += sizeof(struct vmxnet3_rxq_shared); + } + + return (0); +} + +static void +vmxnet3_free_shared_data(struct vmxnet3_softc *sc) +{ + + if (sc->vmx_qs != NULL) { + vmxnet3_dma_free(sc, &sc->vmx_qs_dma); + sc->vmx_qs = NULL; + } + + if (sc->vmx_ds != NULL) { + vmxnet3_dma_free(sc, &sc->vmx_ds_dma); + sc->vmx_ds = NULL; + } +} + +static int +vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) +{ + device_t dev; + struct vmxnet3_txqueue *txq; + struct vmxnet3_txring *txr; + struct vmxnet3_comp_ring *txc; + size_t descsz, compsz; + int i, q, error; + + dev = sc->vmx_dev; + + for (q = 0; q < sc->vmx_ntxqueues; q++) { + txq = &sc->vmx_txq[q]; + txr = &txq->vxtxq_cmd_ring; + txc = &txq->vxtxq_comp_ring; + + descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); + compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); + + error = bus_dma_tag_create(bus_get_dma_tag(dev), + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + VMXNET3_TSO_MAXSIZE, /* maxsize */ + VMXNET3_TX_MAXSEGS, /* nsegments */ + VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &txr->vxtxr_txtag); + if (error) { + device_printf(dev, + "unable to create Tx buffer tag for queue %d\n", q); + return (error); + } + + error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); + if (error) { + device_printf(dev, "cannot alloc Tx descriptors for " + "queue %d error %d\n", q, error); + return (error); + } + txr->vxtxr_txd = + (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; + + error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); + if (error) { + device_printf(dev, "cannot alloc Tx comp descriptors " + "for queue %d error %d\n", q, error); + return (error); + } + txc->vxcr_u.txcd = + (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; + + for (i = 0; i < txr->vxtxr_ndesc; i++) { + error = bus_dmamap_create(txr->vxtxr_txtag, 0, + &txr->vxtxr_txbuf[i].vtxb_dmamap); + if (error) { + device_printf(dev, "unable to create Tx buf " + "dmamap for queue %d idx %d\n", q, i); + return (error); + } + } + } + + return (0); +} + +static void +vmxnet3_free_txq_data(struct vmxnet3_softc *sc) +{ + device_t dev; + struct vmxnet3_txqueue *txq; + struct vmxnet3_txring *txr; + struct vmxnet3_comp_ring *txc; + struct vmxnet3_txbuf *txb; + int i, q; + + dev = sc->vmx_dev; + + for (q = 0; q < sc->vmx_ntxqueues; q++) { + txq = &sc->vmx_txq[q]; + txr = &txq->vxtxq_cmd_ring; + txc = &txq->vxtxq_comp_ring; + + for (i = 0; i < txr->vxtxr_ndesc; i++) { + txb = &txr->vxtxr_txbuf[i]; + if (txb->vtxb_dmamap != NULL) { + bus_dmamap_destroy(txr->vxtxr_txtag, + txb->vtxb_dmamap); + txb->vtxb_dmamap = NULL; + } + } + + if (txc->vxcr_u.txcd != NULL) { + vmxnet3_dma_free(sc, &txc->vxcr_dma); + txc->vxcr_u.txcd = NULL; + } + + if (txr->vxtxr_txd != NULL) { + vmxnet3_dma_free(sc, &txr->vxtxr_dma); + txr->vxtxr_txd = NULL; + } + + if (txr->vxtxr_txtag != NULL) { + bus_dma_tag_destroy(txr->vxtxr_txtag); + txr->vxtxr_txtag = NULL; + } + } +} + +static int +vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) +{ + device_t dev; + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_rxring *rxr; + struct vmxnet3_comp_ring *rxc; + int descsz, compsz; + int i, j, q, error; + + dev = sc->vmx_dev; + compsz = 0; + + for (q = 0; q < sc->vmx_nrxqueues; q++) { + rxq = &sc->vmx_rxq[q]; + rxc = &rxq->vxrxq_comp_ring; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + descsz = rxr->vxrxr_ndesc * + sizeof(struct vmxnet3_rxdesc); + compsz += rxr->vxrxr_ndesc * + sizeof(struct vmxnet3_rxcompdesc); + + error = bus_dma_tag_create(bus_get_dma_tag(dev), + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MJUMPAGESIZE, /* maxsize */ + 1, /* nsegments */ + MJUMPAGESIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &rxr->vxrxr_rxtag); + if (error) { + device_printf(dev, + "unable to create Rx buffer tag for " + "queue %d\n", q); + return (error); + } + + error = vmxnet3_dma_malloc(sc, descsz, 512, + &rxr->vxrxr_dma); + if (error) { + device_printf(dev, "cannot allocate Rx " + "descriptors for queue %d/%d error %d\n", + i, q, error); + return (error); + } + rxr->vxrxr_rxd = + (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; + } + + error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); + if (error) { + device_printf(dev, "cannot alloc Rx comp descriptors " + "for queue %d error %d\n", q, error); + return (error); + } + rxc->vxcr_u.rxcd = + (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, + &rxr->vxrxr_spare_dmap); + if (error) { + device_printf(dev, "unable to create spare " + "dmamap for queue %d/%d error %d\n", + q, i, error); + return (error); + } + + for (j = 0; j < rxr->vxrxr_ndesc; j++) { + error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, + &rxr->vxrxr_rxbuf[j].vrxb_dmamap); + if (error) { + device_printf(dev, "unable to create " + "dmamap for queue %d/%d slot %d " + "error %d\n", + q, i, j, error); + return (error); + } + } + } + } + + return (0); +} + +static void +vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) +{ + device_t dev; + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_rxring *rxr; + struct vmxnet3_comp_ring *rxc; + struct vmxnet3_rxbuf *rxb; + int i, j, q; + + dev = sc->vmx_dev; + + for (q = 0; q < sc->vmx_nrxqueues; q++) { + rxq = &sc->vmx_rxq[q]; + rxc = &rxq->vxrxq_comp_ring; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + if (rxr->vxrxr_spare_dmap != NULL) { + bus_dmamap_destroy(rxr->vxrxr_rxtag, + rxr->vxrxr_spare_dmap); + rxr->vxrxr_spare_dmap = NULL; + } + + for (j = 0; j < rxr->vxrxr_ndesc; j++) { + rxb = &rxr->vxrxr_rxbuf[j]; + if (rxb->vrxb_dmamap != NULL) { + bus_dmamap_destroy(rxr->vxrxr_rxtag, + rxb->vrxb_dmamap); + rxb->vrxb_dmamap = NULL; + } + } + } + + if (rxc->vxcr_u.rxcd != NULL) { + vmxnet3_dma_free(sc, &rxc->vxcr_dma); + rxc->vxcr_u.rxcd = NULL; + } + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + if (rxr->vxrxr_rxd != NULL) { + vmxnet3_dma_free(sc, &rxr->vxrxr_dma); + rxr->vxrxr_rxd = NULL; + } + + if (rxr->vxrxr_rxtag != NULL) { + bus_dma_tag_destroy(rxr->vxrxr_rxtag); + rxr->vxrxr_rxtag = NULL; + } + } + } +} + +static int +vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) +{ + int error; + + error = vmxnet3_alloc_txq_data(sc); + if (error) + return (error); + + error = vmxnet3_alloc_rxq_data(sc); + if (error) + return (error); + + return (0); +} + +static void +vmxnet3_free_queue_data(struct vmxnet3_softc *sc) +{ + + vmxnet3_free_rxq_data(sc); + vmxnet3_free_txq_data(sc); +} + +static int +vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) +{ + int error; + + error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, + 32, &sc->vmx_mcast_dma); + if (error) + device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); + else + sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; + + return (error); +} + +static void +vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) +{ + + if (sc->vmx_mcast != NULL) { + vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); + sc->vmx_mcast = NULL; + } +} + +static void +vmxnet3_init_shared_data(struct vmxnet3_softc *sc) +{ + struct vmxnet3_driver_shared *ds; + struct vmxnet3_txqueue *txq; + struct vmxnet3_txq_shared *txs; + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_rxq_shared *rxs; + int i; + + ds = sc->vmx_ds; + + /* + * Initialize fields of the shared data that remains the same across + * reinits. Note the shared data is zero'd when allocated. + */ + + ds->magic = VMXNET3_REV1_MAGIC; + + /* DriverInfo */ + ds->version = VMXNET3_DRIVER_VERSION; + ds->guest = VMXNET3_GOS_FREEBSD | VMXNET3_GUEST_OS_VERSION | +#ifdef __LP64__ + VMXNET3_GOS_64BIT; +#else + VMXNET3_GOS_32BIT; +#endif + ds->vmxnet3_revision = 1; + ds->upt_version = 1; + + /* Misc. conf */ + ds->driver_data = vtophys(sc); + ds->driver_data_len = sizeof(struct vmxnet3_softc); + ds->queue_shared = sc->vmx_qs_dma.dma_paddr; + ds->queue_shared_len = sc->vmx_qs_dma.dma_size; + ds->nrxsg_max = sc->vmx_max_rxsegs; + + /* Interrupt control. */ + ds->automask = 1; /* VMXNET3_IMM_AUTO */ + ds->nintr = sc->vmx_nintrs; + ds->evintr = sc->vmx_event_intr_idx; + ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; + + for (i = 0; i < sc->vmx_nintrs; i++) + ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; + + /* Receive filter. */ + ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; + ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; + + /* Tx queues */ + for (i = 0; i < sc->vmx_ntxqueues; i++) { + txq = &sc->vmx_txq[i]; + txs = txq->vxtxq_ts; + + txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; + txs->cmd_ring_len = sc->vmx_ntxdescs; + txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; + txs->comp_ring_len = sc->vmx_ntxdescs; + txs->driver_data = vtophys(txq); + txs->driver_data_len = sizeof(struct vmxnet3_txqueue); + } + + /* Rx queues */ + for (i = 0; i < sc->vmx_nrxqueues; i++) { + rxq = &sc->vmx_rxq[i]; + rxs = rxq->vxrxq_rs; + + rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; + rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; + rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; + rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; + rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; + rxs->comp_ring_len = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc + + rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; + rxs->driver_data = vtophys(rxq); + rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); + } +} + +static void +vmxnet3_reinit_interface(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp; + + ifp = sc->vmx_ifp; + + /* Use the current MAC address. */ + bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); + vmxnet3_set_lladdr(sc); + + ifp->if_hwassist = 0; + if (ifp->if_capenable & IFCAP_TXCSUM) + ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD; + if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) + ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; + if (ifp->if_capenable & IFCAP_TSO4) + ifp->if_hwassist |= CSUM_TSO; + if (ifp->if_capenable & IFCAP_TSO6) + ifp->if_hwassist |= CSUM_TSO; /* No CSUM_TSO_IPV6. */ +} + +static void +vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp; + struct vmxnet3_driver_shared *ds; + + ifp = sc->vmx_ifp; + ds = sc->vmx_ds; + + ds->upt_features = 0; + if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) + ds->upt_features |= UPT1_F_VLAN; + if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) + ds->upt_features |= UPT1_F_CSUM; + if (ifp->if_capenable & IFCAP_LRO) + ds->upt_features |= UPT1_F_LRO; + + ds->mtu = ifp->if_mtu; + ds->ntxqueue = sc->vmx_ntxqueues; + ds->nrxqueue = sc->vmx_nrxqueues; + + vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); + vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, + (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); +} + +static int +vmxnet3_alloc_data(struct vmxnet3_softc *sc) +{ + int error; + + error = vmxnet3_alloc_shared_data(sc); + if (error) + return (error); + + error = vmxnet3_alloc_queue_data(sc); + if (error) + return (error); + + error = vmxnet3_alloc_mcast_table(sc); + if (error) + return (error); + + vmxnet3_init_shared_data(sc); + + return (0); +} + +static void +vmxnet3_free_data(struct vmxnet3_softc *sc) +{ + + vmxnet3_free_mcast_table(sc); + vmxnet3_free_queue_data(sc); + vmxnet3_free_shared_data(sc); +} + +static int +vmxnet3_setup_interface(struct vmxnet3_softc *sc) +{ + device_t dev; + struct ifnet *ifp; + + dev = sc->vmx_dev; + + ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); + if (ifp == NULL) { + device_printf(dev, "cannot allocate ifnet structure\n"); + return (ENOSPC); + } + + if_initname(ifp, device_get_name(dev), device_get_unit(dev)); +#if __FreeBSD_version < 1000025 + ifp->if_baudrate = 1000000000; +#else + if_initbaudrate(ifp, IF_Gbps(10)); +#endif + ifp->if_softc = sc; + ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + ifp->if_init = vmxnet3_init; + ifp->if_ioctl = vmxnet3_ioctl; + ifp->if_start = vmxnet3_start; + ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1; + IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1); + IFQ_SET_READY(&ifp->if_snd); + + vmxnet3_get_lladdr(sc); + ether_ifattach(ifp, sc->vmx_lladdr); + + ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; + ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; + ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; + ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING; + ifp->if_hwassist |= VMXNET3_CSUM_ALL_OFFLOAD; + + ifp->if_capenable = ifp->if_capabilities; + + /* + * Capabilities after here are not enabled by default. + */ + + ifp->if_capabilities |= IFCAP_LRO; + + ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; + sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, + vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); + sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, + vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); + + ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, + vmxnet3_media_status); + ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); + + return (0); +} + +static void +vmxnet3_evintr(struct vmxnet3_softc *sc) +{ + device_t dev; + struct ifnet *ifp; + struct vmxnet3_txq_shared *ts; + struct vmxnet3_rxq_shared *rs; + uint32_t event; + int reset; + + dev = sc->vmx_dev; + ifp = sc->vmx_ifp; + reset = 0; + + VMXNET3_CORE_LOCK(sc); + + /* Clear events. */ + event = sc->vmx_ds->event; + vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); + + if (event & VMXNET3_EVENT_LINK) + vmxnet3_link_status(sc); + + if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { + reset = 1; + vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); + ts = sc->vmx_txq[0].vxtxq_ts; + if (ts->stopped != 0) + device_printf(dev, "Tx queue error %#x\n", ts->error); + rs = sc->vmx_rxq[0].vxrxq_rs; + if (rs->stopped != 0) + device_printf(dev, "Rx queue error %#x\n", rs->error); + device_printf(dev, "Rx/Tx queue error event ... resetting\n"); + } + + if (event & VMXNET3_EVENT_DIC) + device_printf(dev, "device implementation change event\n"); + if (event & VMXNET3_EVENT_DEBUG) + device_printf(dev, "debug event\n"); + + if (reset != 0) { + ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + vmxnet3_init_locked(sc); + } + + VMXNET3_CORE_UNLOCK(sc); +} + +static void +vmxnet3_txq_eof(struct vmxnet3_txqueue *txq) +{ + struct vmxnet3_softc *sc; + struct ifnet *ifp; + struct vmxnet3_txring *txr; + struct vmxnet3_comp_ring *txc; + struct vmxnet3_txcompdesc *txcd; + struct vmxnet3_txbuf *txb; + u_int sop; + + sc = txq->vxtxq_sc; + ifp = sc->vmx_ifp; + txr = &txq->vxtxq_cmd_ring; + txc = &txq->vxtxq_comp_ring; + + VMXNET3_TXQ_LOCK_ASSERT(txq); + + for (;;) { + txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; + if (txcd->gen != txc->vxcr_gen) + break; + + if (++txc->vxcr_next == txc->vxcr_ndesc) { + txc->vxcr_next = 0; + txc->vxcr_gen ^= 1; + } + + sop = txr->vxtxr_next; + txb = &txr->vxtxr_txbuf[sop]; + + if (txb->vtxb_m != NULL) { + bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); + + m_freem(txb->vtxb_m); + txb->vtxb_m = NULL; + + ifp->if_opackets++; + } + + txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; + } + + if (txr->vxtxr_head == txr->vxtxr_next) + txq->vxtxq_watchdog = 0; +} + +static int +vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) +{ + struct ifnet *ifp; + struct mbuf *m; + struct vmxnet3_rxdesc *rxd; + struct vmxnet3_rxbuf *rxb; + bus_dma_tag_t tag; + bus_dmamap_t dmap; + bus_dma_segment_t segs[1]; + int idx, clsize, btype, flags, nsegs, error; + + ifp = sc->vmx_ifp; + tag = rxr->vxrxr_rxtag; + dmap = rxr->vxrxr_spare_dmap; + idx = rxr->vxrxr_fill; + rxd = &rxr->vxrxr_rxd[idx]; + rxb = &rxr->vxrxr_rxbuf[idx]; + +#ifdef VMXNET3_FAILPOINTS + KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); + if (rxr->vxrxr_rid != 0) + KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); +#endif + + if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { + flags = M_PKTHDR; + clsize = MCLBYTES; + btype = VMXNET3_BTYPE_HEAD; + } else { +#if __FreeBSD_version < 902001 + /* + * These mbufs will never be used for the start of a + * frame. However, roughly prior to branching releng/9.2, + * bus_dmamap_load_mbuf_sg() required the mbuf to always be + * a packet header. Avoid unnecessary mbuf initialization + * in newer versions where that is not the case. + */ + flags = M_PKTHDR; +#else + flags = 0; +#endif + clsize = MJUMPAGESIZE; + btype = VMXNET3_BTYPE_BODY; + } + + m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); + if (m == NULL) + return (ENOBUFS); + + if (btype == VMXNET3_BTYPE_HEAD) { + m->m_len = m->m_pkthdr.len = clsize; + m_adj(m, ETHER_ALIGN); + } else + m->m_len = clsize; + + error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs, + BUS_DMA_NOWAIT); + if (error) { + m_freem(m); + return (error); + } + KASSERT(nsegs == 1, + ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); + + if (rxb->vrxb_m != NULL) { + bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(tag, rxb->vrxb_dmamap); + } + + rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; + rxb->vrxb_dmamap = dmap; + rxb->vrxb_m = m; + + rxd->addr = segs[0].ds_addr; + rxd->len = segs[0].ds_len; + rxd->btype = btype; + rxd->gen = rxr->vxrxr_gen; + + vmxnet3_rxr_increment_fill(rxr); + return (0); +} + +static void +vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, + struct vmxnet3_rxring *rxr, int idx) +{ + struct vmxnet3_rxdesc *rxd; + + rxd = &rxr->vxrxr_rxd[idx]; + rxd->gen = rxr->vxrxr_gen; + vmxnet3_rxr_increment_fill(rxr); +} + +static void +vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_rxring *rxr; + struct vmxnet3_comp_ring *rxc; + struct vmxnet3_rxcompdesc *rxcd; + int idx, eof; + + sc = rxq->vxrxq_sc; + rxc = &rxq->vxrxq_comp_ring; + + do { + rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; + if (rxcd->gen != rxc->vxcr_gen) + break; /* Not expected. */ + vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); + + if (++rxc->vxcr_next == rxc->vxcr_ndesc) { + rxc->vxcr_next = 0; + rxc->vxcr_gen ^= 1; + } + + idx = rxcd->rxd_idx; + eof = rxcd->eop; + if (rxcd->qid < sc->vmx_nrxqueues) + rxr = &rxq->vxrxq_cmd_ring[0]; + else + rxr = &rxq->vxrxq_cmd_ring[1]; + vmxnet3_rxq_eof_discard(rxq, rxr, idx); + } while (!eof); +} + +static void +vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) +{ + + if (rxcd->ipv4) { + m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; + if (rxcd->ipcsum_ok) + m->m_pkthdr.csum_flags |= CSUM_IP_VALID; + } + + if (!rxcd->fragment) { + if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { + m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | + CSUM_PSEUDO_HDR; + m->m_pkthdr.csum_data = 0xFFFF; + } + } +} + +static void +vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, + struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) +{ + struct vmxnet3_softc *sc; + struct ifnet *ifp; + + sc = rxq->vxrxq_sc; + ifp = sc->vmx_ifp; + + if (rxcd->error) { + ifp->if_ierrors++; + m_freem(m); + return; + } + + if (!rxcd->no_csum) + vmxnet3_rx_csum(rxcd, m); + if (rxcd->vlan) { + m->m_flags |= M_VLANTAG; + m->m_pkthdr.ether_vtag = rxcd->vtag; + } + + ifp->if_ipackets++; + VMXNET3_RXQ_UNLOCK(rxq); + (*ifp->if_input)(ifp, m); + VMXNET3_RXQ_LOCK(rxq); +} + +static void +vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) +{ + struct vmxnet3_softc *sc; + struct ifnet *ifp; + struct vmxnet3_rxring *rxr; + struct vmxnet3_comp_ring *rxc; + struct vmxnet3_rxdesc *rxd; + struct vmxnet3_rxcompdesc *rxcd; + struct mbuf *m, *m_head, *m_tail; + int idx, length; + + sc = rxq->vxrxq_sc; + ifp = sc->vmx_ifp; + rxc = &rxq->vxrxq_comp_ring; + m_head = m_tail = NULL; + + VMXNET3_RXQ_LOCK_ASSERT(rxq); + + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + return; + + for (;;) { + rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; + if (rxcd->gen != rxc->vxcr_gen) + break; + vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); + + if (++rxc->vxcr_next == rxc->vxcr_ndesc) { + rxc->vxcr_next = 0; + rxc->vxcr_gen ^= 1; + } + + idx = rxcd->rxd_idx; + length = rxcd->len; + if (rxcd->qid < sc->vmx_nrxqueues) + rxr = &rxq->vxrxq_cmd_ring[0]; + else + rxr = &rxq->vxrxq_cmd_ring[1]; + rxd = &rxr->vxrxr_rxd[idx]; + + m = rxr->vxrxr_rxbuf[idx].vrxb_m; + KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", + __func__, rxcd->qid, idx)); + + /* + * The host may skip descriptors. We detect this when this + * descriptor does not match the previous fill index. Catch + * up with the host now. + */ + if (__predict_false(rxr->vxrxr_fill != idx)) { + while (rxr->vxrxr_fill != idx) { + rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = + rxr->vxrxr_gen; + vmxnet3_rxr_increment_fill(rxr); + } + } + + if (rxcd->sop) { + KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, + ("%s: start of frame w/o head buffer", __func__)); + KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], + ("%s: start of frame not in ring 0", __func__)); + KASSERT((idx % sc->vmx_rx_max_chain) == 0, + ("%s: start of frame at unexcepted index %d (%d)", + __func__, idx, sc->vmx_rx_max_chain)); + KASSERT(m_head == NULL, + ("%s: duplicate start of frame?", __func__)); + + if (length == 0) { + /* Just ignore this descriptor. */ + vmxnet3_rxq_eof_discard(rxq, rxr, idx); + goto nextp; + } + + if (vmxnet3_newbuf(sc, rxr) != 0) { + ifp->if_iqdrops++; + vmxnet3_rxq_eof_discard(rxq, rxr, idx); + if (!rxcd->eop) + vmxnet3_rxq_discard_chain(rxq); + goto nextp; + } + + m->m_pkthdr.rcvif = ifp; + m->m_pkthdr.len = m->m_len = length; + m->m_pkthdr.csum_flags = 0; + m_head = m_tail = m; + + } else { + KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, + ("%s: non start of frame w/o body buffer", __func__)); + KASSERT(m_head != NULL, + ("%s: frame not started?", __func__)); + + if (vmxnet3_newbuf(sc, rxr) != 0) { + ifp->if_iqdrops++; + vmxnet3_rxq_eof_discard(rxq, rxr, idx); + if (!rxcd->eop) + vmxnet3_rxq_discard_chain(rxq); + m_freem(m_head); + m_head = m_tail = NULL; + goto nextp; + } + + m->m_len = length; + m_head->m_pkthdr.len += length; + m_tail->m_next = m; + m_tail = m; + } + + if (rxcd->eop) { + vmxnet3_rxq_input(rxq, rxcd, m_head); + m_head = m_tail = NULL; + + /* Must recheck after dropping the Rx lock. */ + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + break; + } + +nextp: + if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { + int qid = rxcd->qid; + bus_size_t r; + + idx = (idx + 1) % rxr->vxrxr_ndesc; + if (qid >= sc->vmx_nrxqueues) { + qid -= sc->vmx_nrxqueues; + r = VMXNET3_BAR0_RXH2(qid); + } else + r = VMXNET3_BAR0_RXH1(qid); + vmxnet3_write_bar0(sc, r, idx); + } + } +} + +static void +vmxnet3_legacy_intr(void *xsc) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_txqueue *txq; + struct ifnet *ifp; + + sc = xsc; + rxq = &sc->vmx_rxq[0]; + txq = &sc->vmx_txq[0]; + ifp = sc->vmx_ifp; + + if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { + if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) + return; + } + + if (sc->vmx_ds->event != 0) + vmxnet3_evintr(sc); + + VMXNET3_RXQ_LOCK(rxq); + vmxnet3_rxq_eof(rxq); + VMXNET3_RXQ_UNLOCK(rxq); + + VMXNET3_TXQ_LOCK(txq); + vmxnet3_txq_eof(txq); + if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + vmxnet3_start_locked(ifp); + VMXNET3_TXQ_UNLOCK(txq); + + vmxnet3_enable_intr(sc, 0); +} + +static void +vmxnet3_txq_intr(void *xtxq) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_txqueue *txq; + struct ifnet *ifp; + + txq = xtxq; + sc = txq->vxtxq_sc; + ifp = sc->vmx_ifp; + + VMXNET3_TXQ_LOCK(txq); + vmxnet3_txq_eof(txq); + if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + vmxnet3_start_locked(ifp); + VMXNET3_TXQ_UNLOCK(txq); + + vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); +} + +static void +vmxnet3_rxq_intr(void *xrxq) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_rxqueue *rxq; + + rxq = xrxq; + sc = rxq->vxrxq_sc; + + VMXNET3_RXQ_LOCK(rxq); + vmxnet3_rxq_eof(rxq); + VMXNET3_RXQ_UNLOCK(rxq); + + vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); +} + +static void +vmxnet3_event_intr(void *xsc) +{ + struct vmxnet3_softc *sc; + + sc = xsc; + + if (sc->vmx_ds->event != 0) + vmxnet3_evintr(sc); + + vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); +} + +static void +vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) +{ + struct vmxnet3_txring *txr; + struct vmxnet3_txbuf *txb; + int i; + + txr = &txq->vxtxq_cmd_ring; + + for (i = 0; i < txr->vxtxr_ndesc; i++) { + txb = &txr->vxtxr_txbuf[i]; + + if (txb->vtxb_m == NULL) + continue; + + bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); + m_freem(txb->vtxb_m); + txb->vtxb_m = NULL; + } +} + +static void +vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) +{ + struct vmxnet3_rxring *rxr; + struct vmxnet3_rxbuf *rxb; + int i, j; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + for (j = 0; j < rxr->vxrxr_ndesc; j++) { + rxb = &rxr->vxrxr_rxbuf[j]; + + if (rxb->vrxb_m == NULL) + continue; + bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); + m_freem(rxb->vrxb_m); + rxb->vrxb_m = NULL; + } + } +} + +static void +vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) +{ + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_txqueue *txq; + int i; + + for (i = 0; i < sc->vmx_nrxqueues; i++) { + rxq = &sc->vmx_rxq[i]; + VMXNET3_RXQ_LOCK(rxq); + VMXNET3_RXQ_UNLOCK(rxq); + } + + for (i = 0; i < sc->vmx_ntxqueues; i++) { + txq = &sc->vmx_txq[i]; + VMXNET3_TXQ_LOCK(txq); + VMXNET3_TXQ_UNLOCK(txq); + } +} + +static void +vmxnet3_stop(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp; + int q; + + ifp = sc->vmx_ifp; + VMXNET3_CORE_LOCK_ASSERT(sc); + + ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + sc->vmx_link_active = 0; + callout_stop(&sc->vmx_tick); + + /* Disable interrupts. */ + vmxnet3_disable_all_intrs(sc); + vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); + + vmxnet3_stop_rendezvous(sc); + + for (q = 0; q < sc->vmx_ntxqueues; q++) + vmxnet3_txstop(sc, &sc->vmx_txq[q]); + for (q = 0; q < sc->vmx_nrxqueues; q++) + vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); + + vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); +} + +static void +vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) +{ + struct vmxnet3_txring *txr; + struct vmxnet3_comp_ring *txc; + + txr = &txq->vxtxq_cmd_ring; + txr->vxtxr_head = 0; + txr->vxtxr_next = 0; + txr->vxtxr_gen = VMXNET3_INIT_GEN; + bzero(txr->vxtxr_txd, + txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); + + txc = &txq->vxtxq_comp_ring; + txc->vxcr_next = 0; + txc->vxcr_gen = VMXNET3_INIT_GEN; + bzero(txc->vxcr_u.txcd, + txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); +} + +static int +vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) +{ + struct ifnet *ifp; + struct vmxnet3_rxring *rxr; + struct vmxnet3_comp_ring *rxc; + int i, idx, frame_size, error; + + ifp = sc->vmx_ifp; + frame_size = ifp->if_mtu + sizeof(struct ether_vlan_header); + + /* + * If the MTU causes us to exceed what a regular sized cluster + * can handle, we allocate a second MJUMPAGESIZE cluster after + * it in ring 0. Ring 1 always contains MJUMPAGESIZE clusters. + * + * Keep rx_max_chain a divisor of the maximum Rx ring size to + * to make our life easier. We do not support changing the ring + * size after the attach. + * + * TODO If LRO is not enabled, there is little point of even + * populating the second ring. + * + */ + if (frame_size <= MCLBYTES) + sc->vmx_rx_max_chain = 1; + else + sc->vmx_rx_max_chain = 2; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + rxr->vxrxr_fill = 0; + rxr->vxrxr_gen = VMXNET3_INIT_GEN; + bzero(rxr->vxrxr_rxd, + rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); + + for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { + error = vmxnet3_newbuf(sc, rxr); + if (error) + return (error); + } + } + + rxc = &rxq->vxrxq_comp_ring; + rxc->vxcr_next = 0; + rxc->vxcr_gen = VMXNET3_INIT_GEN; + bzero(rxc->vxcr_u.rxcd, + rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); + + return (0); +} + +static int +vmxnet3_reinit_queues(struct vmxnet3_softc *sc) +{ + device_t dev; + int q, error; + + dev = sc->vmx_dev; + + for (q = 0; q < sc->vmx_ntxqueues; q++) + vmxnet3_txinit(sc, &sc->vmx_txq[q]); + + for (q = 0; q < sc->vmx_nrxqueues; q++) { + error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); + if (error) { + device_printf(dev, "cannot populate Rx queue %d\n", q); + return (error); + } + } + + return (0); +} + +static int +vmxnet3_enable_device(struct vmxnet3_softc *sc) +{ + int q; + + if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { + device_printf(sc->vmx_dev, "device enable command failed!\n"); + return (1); + } + + /* Reset the Rx queue heads. */ + for (q = 0; q < sc->vmx_nrxqueues; q++) { + vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); + vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); + } + + return (0); +} + +static void +vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp; + + ifp = sc->vmx_ifp; + + vmxnet3_set_rxfilter(sc); + + if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) + bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, + sizeof(sc->vmx_ds->vlan_filter)); + else + bzero(sc->vmx_ds->vlan_filter, + sizeof(sc->vmx_ds->vlan_filter)); + vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); +} + +static int +vmxnet3_reinit(struct vmxnet3_softc *sc) +{ + + vmxnet3_reinit_interface(sc); + vmxnet3_reinit_shared_data(sc); + + if (vmxnet3_reinit_queues(sc) != 0) + return (ENXIO); + + if (vmxnet3_enable_device(sc) != 0) + return (ENXIO); + + vmxnet3_reinit_rxfilters(sc); + + return (0); +} + +static void +vmxnet3_init_locked(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp; + + ifp = sc->vmx_ifp; + + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + return; + + vmxnet3_stop(sc); + + if (vmxnet3_reinit(sc) != 0) { + vmxnet3_stop(sc); + return; + } + + ifp->if_drv_flags |= IFF_DRV_RUNNING; + vmxnet3_link_status(sc); + + vmxnet3_enable_all_intrs(sc); + callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); +} + +static void +vmxnet3_init(void *xsc) +{ + struct vmxnet3_softc *sc; + + sc = xsc; + + VMXNET3_CORE_LOCK(sc); + vmxnet3_init_locked(sc); + VMXNET3_CORE_UNLOCK(sc); +} + +/* + * BMV: Much of this can go away once we finally have offsets in + * the mbuf packet header. Bug andre@. + */ +static int +vmxnet3_txq_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start) +{ + struct ether_vlan_header *evh; + int offset; + + evh = mtod(m, struct ether_vlan_header *); + if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + /* BMV: We should handle nested VLAN tags too. */ + *etype = ntohs(evh->evl_proto); + offset = sizeof(struct ether_vlan_header); + } else { + *etype = ntohs(evh->evl_encap_proto); + offset = sizeof(struct ether_header); + } + + switch (*etype) { +#if defined(INET) + case ETHERTYPE_IP: { + struct ip *ip, iphdr; + if (__predict_false(m->m_len < offset + sizeof(struct ip))) { + m_copydata(m, offset, sizeof(struct ip), + (caddr_t) &iphdr); + ip = &iphdr; + } else + ip = (struct ip *)(m->m_data + offset); + *proto = ip->ip_p; + *start = offset + (ip->ip_hl << 2); + break; + } +#endif +#if defined(INET6) + case ETHERTYPE_IPV6: + *proto = -1; + *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); + /* Assert the network stack sent us a valid packet. */ + KASSERT(*start > offset, + ("%s: mbuf %p start %d offset %d proto %d", __func__, m, + *start, offset, *proto)); + break; +#endif + default: + return (EINVAL); + } + + if (m->m_pkthdr.csum_flags & CSUM_TSO) { + struct tcphdr *tcp, tcphdr; + + if (__predict_false(*proto != IPPROTO_TCP)) { + /* Likely failed to correctly parse the mbuf. */ + return (EINVAL); + } + + if (m->m_len < *start + sizeof(struct tcphdr)) { + m_copydata(m, offset, sizeof(struct tcphdr), + (caddr_t) &tcphdr); + tcp = &tcphdr; + } else + tcp = (struct tcphdr *)(m->m_data + *start); + + /* + * For TSO, the size of the protocol header is also + * included in the descriptor header size. + */ + *start += (tcp->th_off << 2); + } + + return (0); +} + +static int +vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, + bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) +{ + struct vmxnet3_txring *txr; + struct mbuf *m; + bus_dma_tag_t tag; + int maxsegs, error; + + txr = &txq->vxtxq_cmd_ring; + m = *m0; + tag = txr->vxtxr_txtag; + maxsegs = VMXNET3_TX_MAXSEGS; + + error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); + if (error == 0 || error != EFBIG) + return (error); + + m = m_collapse(m, M_NOWAIT, maxsegs); + if (m != NULL) { + *m0 = m; + error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); + } else + error = ENOBUFS; + + if (error) { + m_freem(*m0); + *m0 = NULL; + } else + txq->vxtxq_sc->vmx_stats.vmst_collapsed++; + + return (error); +} + +static void +vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) +{ + struct vmxnet3_txring *txr; + + txr = &txq->vxtxq_cmd_ring; + bus_dmamap_unload(txr->vxtxr_txtag, dmap); +} + +static int +vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) +{ + struct vmxnet3_softc *sc; + struct ifnet *ifp; + struct vmxnet3_txring *txr; + struct vmxnet3_txdesc *txd, *sop; + struct mbuf *m; + bus_dmamap_t dmap; + bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; + int i, gen, nsegs, etype, proto, start, error; + + sc = txq->vxtxq_sc; + ifp = sc->vmx_ifp; + start = 0; + txd = NULL; + txr = &txq->vxtxq_cmd_ring; + dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; + + error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); + if (error) + return (error); + + m = *m0; + M_ASSERTPKTHDR(m); + KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, + ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); + + if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { + txq->vxtxq_stats.vtxrs_full++; + vmxnet3_txq_unload_mbuf(txq, dmap); + return (ENOSPC); + } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { + error = vmxnet3_txq_offload_ctx(m, &etype, &proto, &start); + if (error) { + vmxnet3_txq_unload_mbuf(txq, dmap); + m_freem(m); + *m0 = NULL; + return (error); + } + } + + txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m = *m0; + sop = &txr->vxtxr_txd[txr->vxtxr_head]; + gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ + + for (i = 0; i < nsegs; i++) { + txd = &txr->vxtxr_txd[txr->vxtxr_head]; + + txd->addr = segs[i].ds_addr; + txd->len = segs[i].ds_len; + txd->gen = gen; + txd->dtype = 0; + txd->offload_mode = VMXNET3_OM_NONE; + txd->offload_pos = 0; + txd->hlen = 0; + txd->eop = 0; + txd->compreq = 0; + txd->vtag_mode = 0; + txd->vtag = 0; + + if (++txr->vxtxr_head == txr->vxtxr_ndesc) { + txr->vxtxr_head = 0; + txr->vxtxr_gen ^= 1; + } + gen = txr->vxtxr_gen; + } + txd->eop = 1; + txd->compreq = 1; + + if (m->m_flags & M_VLANTAG) { + sop->vtag_mode = 1; + sop->vtag = m->m_pkthdr.ether_vtag; + } + + if (m->m_pkthdr.csum_flags & CSUM_TSO) { + sop->offload_mode = VMXNET3_OM_TSO; + sop->hlen = start; + sop->offload_pos = m->m_pkthdr.tso_segsz; + } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | + VMXNET3_CSUM_OFFLOAD_IPV6)) { + sop->offload_mode = VMXNET3_OM_CSUM; + sop->hlen = start; + sop->offload_pos = start + m->m_pkthdr.csum_data; + } + + /* Finally, change the ownership. */ + vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); + sop->gen ^= 1; + + if (++txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { + txq->vxtxq_ts->npending = 0; + vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), + txr->vxtxr_head); + } + + return (0); +} + +static void +vmxnet3_start_locked(struct ifnet *ifp) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_txqueue *txq; + struct vmxnet3_txring *txr; + struct mbuf *m_head; + int tx; + + sc = ifp->if_softc; + txq = &sc->vmx_txq[0]; + txr = &txq->vxtxq_cmd_ring; + tx = 0; + + VMXNET3_TXQ_LOCK_ASSERT(txq); + + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || + sc->vmx_link_active == 0) + return; + + while (VMXNET3_TXRING_AVAIL(txr) > 0) { + IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); + if (m_head == NULL) + break; + + if (vmxnet3_txq_encap(txq, &m_head) != 0) { + if (m_head != NULL) + IFQ_DRV_PREPEND(&ifp->if_snd, m_head); + break; + } + + tx++; + ETHER_BPF_MTAP(ifp, m_head); + } + + if (tx > 0) { + if (txq->vxtxq_ts->npending > 0) { + txq->vxtxq_ts->npending = 0; + vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), + txr->vxtxr_head); + } + txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; + } +} + +static void +vmxnet3_start(struct ifnet *ifp) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_txqueue *txq; + + sc = ifp->if_softc; + txq = &sc->vmx_txq[0]; + + VMXNET3_TXQ_LOCK(txq); + vmxnet3_start_locked(ifp); + VMXNET3_TXQ_UNLOCK(txq); +} + +static void +vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) +{ + struct ifnet *ifp; + int idx, bit; + + ifp = sc->vmx_ifp; + idx = (tag >> 5) & 0x7F; + bit = tag & 0x1F; + + if (tag == 0 || tag > 4095) + return; + + VMXNET3_CORE_LOCK(sc); + + /* Update our private VLAN bitvector. */ + if (add) + sc->vmx_vlan_filter[idx] |= (1 << bit); + else + sc->vmx_vlan_filter[idx] &= ~(1 << bit); + + if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { + if (add) + sc->vmx_ds->vlan_filter[idx] |= (1 << bit); + else + sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); + vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); + } + + VMXNET3_CORE_UNLOCK(sc); +} + +static void +vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) +{ + + if (ifp->if_softc == arg) + vmxnet3_update_vlan_filter(arg, 1, tag); +} + +static void +vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) +{ + + if (ifp->if_softc == arg) + vmxnet3_update_vlan_filter(arg, 0, tag); +} + +static void +vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp; + struct vmxnet3_driver_shared *ds; + struct ifmultiaddr *ifma; + u_int mode; + + ifp = sc->vmx_ifp; + ds = sc->vmx_ds; + + mode = VMXNET3_RXMODE_UCAST; + if (ifp->if_flags & IFF_BROADCAST) + mode |= VMXNET3_RXMODE_BCAST; + if (ifp->if_flags & IFF_PROMISC) + mode |= VMXNET3_RXMODE_PROMISC; + if (ifp->if_flags & IFF_ALLMULTI) + mode |= VMXNET3_RXMODE_ALLMULTI; + else { + int cnt = 0, overflow = 0; + + if_maddr_rlock(ifp); + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + else if (cnt == VMXNET3_MULTICAST_MAX) { + overflow = 1; + break; + } + + bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), + &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); + cnt++; + } + if_maddr_runlock(ifp); + + if (overflow != 0) { + cnt = 0; + mode |= VMXNET3_RXMODE_ALLMULTI; + } else if (cnt > 0) + mode |= VMXNET3_RXMODE_MCAST; + ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; + } + + ds->rxmode = mode; + + vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); + vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); +} + +static int +vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) +{ + struct ifnet *ifp; + + ifp = sc->vmx_ifp; + + if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) + return (EINVAL); + + ifp->if_mtu = mtu; + + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + vmxnet3_init_locked(sc); + } + + return (0); +} + +static int +vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) +{ + struct vmxnet3_softc *sc; + struct ifreq *ifr; + int reinit, mask, error; + + sc = ifp->if_softc; + ifr = (struct ifreq *) data; + error = 0; + + switch (cmd) { + case SIOCSIFMTU: + if (ifp->if_mtu != ifr->ifr_mtu) { + VMXNET3_CORE_LOCK(sc); + error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); + VMXNET3_CORE_UNLOCK(sc); + } + break; + + case SIOCSIFFLAGS: + VMXNET3_CORE_LOCK(sc); + if (ifp->if_flags & IFF_UP) { + if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { + if ((ifp->if_flags ^ sc->vmx_if_flags) & + (IFF_PROMISC | IFF_ALLMULTI)) { + vmxnet3_set_rxfilter(sc); + } + } else + vmxnet3_init_locked(sc); + } else { + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + vmxnet3_stop(sc); + } + sc->vmx_if_flags = ifp->if_flags; + VMXNET3_CORE_UNLOCK(sc); + break; + + case SIOCADDMULTI: + case SIOCDELMULTI: + VMXNET3_CORE_LOCK(sc); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + vmxnet3_set_rxfilter(sc); + VMXNET3_CORE_UNLOCK(sc); + break; + + case SIOCSIFMEDIA: + case SIOCGIFMEDIA: + error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); + break; + + case SIOCSIFCAP: + VMXNET3_CORE_LOCK(sc); + mask = ifr->ifr_reqcap ^ ifp->if_capenable; + + if (mask & IFCAP_TXCSUM) + ifp->if_capenable ^= IFCAP_TXCSUM; + if (mask & IFCAP_TXCSUM_IPV6) + ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; + if (mask & IFCAP_TSO4) + ifp->if_capenable ^= IFCAP_TSO4; + if (mask & IFCAP_TSO6) + ifp->if_capenable ^= IFCAP_TSO6; + + if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | + IFCAP_VLAN_HWFILTER)) { + /* These Rx features require us to renegotiate. */ + reinit = 1; + + if (mask & IFCAP_RXCSUM) + ifp->if_capenable ^= IFCAP_RXCSUM; + if (mask & IFCAP_RXCSUM_IPV6) + ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; + if (mask & IFCAP_LRO) + ifp->if_capenable ^= IFCAP_LRO; + if (mask & IFCAP_VLAN_HWFILTER) + ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; + } else + reinit = 0; + + if (mask & IFCAP_VLAN_HWTSO) + ifp->if_capenable ^= IFCAP_VLAN_HWTSO; + if (mask & IFCAP_VLAN_HWTAGGING) + ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + + if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { + ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + vmxnet3_init_locked(sc); + } + + VMXNET3_CORE_UNLOCK(sc); + VLAN_CAPABILITIES(ifp); + break; + + default: + error = ether_ioctl(ifp, cmd, data); + break; + } + + VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); + + return (error); +} + +static int +vmxnet3_watchdog(struct vmxnet3_txqueue *txq) +{ + struct vmxnet3_softc *sc; + + sc = txq->vxtxq_sc; + + VMXNET3_TXQ_LOCK(txq); + if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { + VMXNET3_TXQ_UNLOCK(txq); + return (0); + } + VMXNET3_TXQ_UNLOCK(txq); + + if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", + txq->vxtxq_id); + return (1); +} + +static void +vmxnet3_refresh_stats(struct vmxnet3_softc *sc) +{ + + vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); +} + +static void +vmxnet3_tick(void *xsc) +{ + struct vmxnet3_softc *sc; + struct ifnet *ifp; + int i, timedout; + + sc = xsc; + ifp = sc->vmx_ifp; + timedout = 0; + + VMXNET3_CORE_LOCK_ASSERT(sc); + vmxnet3_refresh_stats(sc); + + for (i = 0; i < sc->vmx_ntxqueues; i++) + timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); + + if (timedout != 0) { + ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + vmxnet3_init_locked(sc); + } else + callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); +} + +static int +vmxnet3_link_is_up(struct vmxnet3_softc *sc) +{ + uint32_t status; + + /* Also update the link speed while here. */ + status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); + sc->vmx_link_speed = status >> 16; + return !!(status & 0x1); +} + +static void +vmxnet3_link_status(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp; + int link; + + ifp = sc->vmx_ifp; + link = vmxnet3_link_is_up(sc); + + if (link != 0 && sc->vmx_link_active == 0) { + sc->vmx_link_active = 1; + if_link_state_change(ifp, LINK_STATE_UP); + } else if (link == 0 && sc->vmx_link_active != 0) { + sc->vmx_link_active = 0; + if_link_state_change(ifp, LINK_STATE_DOWN); + } +} + +static void +vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) +{ + struct vmxnet3_softc *sc; + + sc = ifp->if_softc; + + ifmr->ifm_active = IFM_ETHER | IFM_AUTO; + ifmr->ifm_status = IFM_AVALID; + + VMXNET3_CORE_LOCK(sc); + if (vmxnet3_link_is_up(sc) != 0) + ifmr->ifm_status |= IFM_ACTIVE; + else + ifmr->ifm_status |= IFM_NONE; + VMXNET3_CORE_UNLOCK(sc); +} + +static int +vmxnet3_media_change(struct ifnet *ifp) +{ + + /* Ignore. */ + return (0); +} + +static void +vmxnet3_set_lladdr(struct vmxnet3_softc *sc) +{ + uint32_t ml, mh; + + ml = sc->vmx_lladdr[0]; + ml |= sc->vmx_lladdr[1] << 8; + ml |= sc->vmx_lladdr[2] << 16; + ml |= sc->vmx_lladdr[3] << 24; + vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); + + mh = sc->vmx_lladdr[4]; + mh |= sc->vmx_lladdr[5] << 8; + vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); +} + +static void +vmxnet3_get_lladdr(struct vmxnet3_softc *sc) +{ + uint32_t ml, mh; + + ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); + mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); + + sc->vmx_lladdr[0] = ml; + sc->vmx_lladdr[1] = ml >> 8; + sc->vmx_lladdr[2] = ml >> 16; + sc->vmx_lladdr[3] = ml >> 24; + sc->vmx_lladdr[4] = mh; + sc->vmx_lladdr[5] = mh >> 8; +} + +static void +vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, + struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) +{ + struct sysctl_oid *node, *txsnode; + struct sysctl_oid_list *list, *txslist; + struct vmxnet3_txq_stats *stats; + struct UPT1_TxStats *txstats; + char namebuf[16]; + + stats = &txq->vxtxq_stats; + txstats = &txq->vxtxq_ts->stats; + + snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); + node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, + NULL, "Transmit Queue"); + txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); + + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, + &stats->vtxrs_full, "Tx ring full"); + + /* + * Add statistics reported by the host. These are updated once + * per second. + */ + txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, + NULL, "Host Statistics"); + txslist = SYSCTL_CHILDREN(txsnode); + SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, + &txstats->TSO_packets, "TSO packets"); + SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, + &txstats->TSO_bytes, "TSO bytes"); + SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, + &txstats->ucast_packets, "Unicast packets"); + SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, + &txstats->ucast_bytes, "Unicast bytes"); + SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, + &txstats->mcast_packets, "Multicast packets"); + SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, + &txstats->mcast_bytes, "Multicast bytes"); + SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, + &txstats->error, "Errors"); + SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, + &txstats->discard, "Discards"); +} + +static void +vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, + struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) +{ + struct sysctl_oid *node, *rxsnode; + struct sysctl_oid_list *list, *rxslist; + struct vmxnet3_rxq_stats *stats; + struct UPT1_RxStats *rxstats; + char namebuf[16]; + + stats = &rxq->vxrxq_stats; + rxstats = &rxq->vxrxq_rs->stats; + + snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); + node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, + NULL, "Receive Queue"); + rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); + + /* + * Add statistics reported by the host. These are updated once + * per second. + */ + rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, + NULL, "Host Statistics"); + rxslist = SYSCTL_CHILDREN(rxsnode); + SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, + &rxstats->LRO_packets, "LRO packets"); + SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, + &rxstats->LRO_bytes, "LRO bytes"); + SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, + &rxstats->ucast_packets, "Unicast packets"); + SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, + &rxstats->ucast_bytes, "Unicast bytes"); + SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, + &rxstats->mcast_packets, "Multicast packets"); + SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, + &rxstats->mcast_bytes, "Multicast bytes"); + SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, + &rxstats->bcast_packets, "Broadcast packets"); + SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, + &rxstats->bcast_bytes, "Broadcast bytes"); + SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, + &rxstats->nobuffer, "No buffer"); + SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, + &rxstats->error, "Errors"); +} + +#ifdef VMXNET3_DEBUG_SYSCTL +static void +vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, + struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) +{ + struct sysctl_oid *node; + struct sysctl_oid_list *list; + int i; + + for (i = 0; i < sc->vmx_ntxqueues; i++) { + struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; + + node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, + "debug", CTLFLAG_RD, NULL, ""); + list = SYSCTL_CHILDREN(node); + + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, + &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, + &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, + &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); + SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, + &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, + &txq->vxtxq_comp_ring.vxcr_next, 0, ""); + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, + &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); + SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, + &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); + } + + for (i = 0; i < sc->vmx_nrxqueues; i++) { + struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; + + node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, + "debug", CTLFLAG_RD, NULL, ""); + list = SYSCTL_CHILDREN(node); + + /* Assumes VMXNET3_RXRINGS_PERQ == 2. */ + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, + &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, + &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); + SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, + &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, + &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, + &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); + SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, + &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, + &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, + &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); + SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, + &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); + } +} +#endif + +static void +vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, + struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) +{ + int i; + + for (i = 0; i < sc->vmx_ntxqueues; i++) + vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); + for (i = 0; i < sc->vmx_nrxqueues; i++) + vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); + +#ifdef VMXNET3_DEBUG_SYSCTL + vmxnet3_setup_debug_sysctl(sc, ctx, child); +#endif +} + +static void +vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) +{ + device_t dev; + struct vmxnet3_statistics *stats; + struct sysctl_ctx_list *ctx; + struct sysctl_oid *tree; + struct sysctl_oid_list *child; + + dev = sc->vmx_dev; + ctx = device_get_sysctl_ctx(dev); + tree = device_get_sysctl_tree(dev); + child = SYSCTL_CHILDREN(tree); + + SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, + &sc->vmx_ntxqueues, 0, "Number of Tx queues"); + SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, + &sc->vmx_nrxqueues, 0, "Number of Rx queues"); + + stats = &sc->vmx_stats; + SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "collapsed", CTLFLAG_RD, + &stats->vmst_collapsed, "Tx mbuf chains collapsed"); + + vmxnet3_setup_queue_sysctl(sc, ctx, child); +} + +static void +vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) +{ + + bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); +} + +static uint32_t +vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) +{ + + return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); +} + +static void +vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) +{ + + bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); +} + +static void +vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) +{ + + vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); +} + +static uint32_t +vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) +{ + + vmxnet3_write_cmd(sc, cmd); + bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, + BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); + return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); +} + +static void +vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) +{ + + vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); +} + +static void +vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) +{ + + vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); +} + +static void +vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) +{ + int i; + + sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; + for (i = 0; i < sc->vmx_nintrs; i++) + vmxnet3_enable_intr(sc, i); +} + +static void +vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) +{ + int i; + + sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; + for (i = 0; i < sc->vmx_nintrs; i++) + vmxnet3_disable_intr(sc, i); +} + +static void +vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + bus_addr_t *baddr = arg; + + if (error == 0) + *baddr = segs->ds_addr; +} + +static int +vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, + struct vmxnet3_dma_alloc *dma) +{ + device_t dev; + int error; + + dev = sc->vmx_dev; + bzero(dma, sizeof(struct vmxnet3_dma_alloc)); + + error = bus_dma_tag_create(bus_get_dma_tag(dev), + align, 0, /* alignment, bounds */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + size, /* maxsize */ + 1, /* nsegments */ + size, /* maxsegsize */ + BUS_DMA_ALLOCNOW, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockfuncarg */ + &dma->dma_tag); + if (error) { + device_printf(dev, "bus_dma_tag_create failed: %d\n", error); + goto fail; + } + + error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, + BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); + if (error) { + device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); + goto fail; + } + + error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, + size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); + if (error) { + device_printf(dev, "bus_dmamap_load failed: %d\n", error); + goto fail; + } + + dma->dma_size = size; + +fail: + if (error) + vmxnet3_dma_free(sc, dma); + + return (error); +} + +static void +vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) +{ + + if (dma->dma_tag != NULL) { + if (dma->dma_map != NULL) { + bus_dmamap_sync(dma->dma_tag, dma->dma_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(dma->dma_tag, dma->dma_map); + } + + if (dma->dma_vaddr != NULL) { + bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, + dma->dma_map); + } + + bus_dma_tag_destroy(dma->dma_tag); + } + bzero(dma, sizeof(struct vmxnet3_dma_alloc)); +} + +/* + * Since this is a purely paravirtualized device, we do not have + * to worry about DMA coherency. But at times, we must make sure + * both the compiler and CPU do not reorder memory operations. + */ +static inline void +vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) +{ + + switch (type) { + case VMXNET3_BARRIER_RD: + rmb(); + break; + case VMXNET3_BARRIER_WR: + wmb(); + break; + case VMXNET3_BARRIER_RDWR: + mb(); + break; + default: + panic("%s: bad barrier type %d", __func__, type); + } +} diff --git a/sys/dev/vmware/vmxnet3/if_vmxreg.h b/sys/dev/vmware/vmxnet3/if_vmxreg.h new file mode 100644 index 0000000..3c89979 --- /dev/null +++ b/sys/dev/vmware/vmxnet3/if_vmxreg.h @@ -0,0 +1,317 @@ +/*- + * Copyright (c) 2013 Tsubai Masanari + * Copyright (c) 2013 Bryan Venteicher + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * $OpenBSD: src/sys/dev/pci/if_vmxreg.h,v 1.2 2013/06/12 01:07:33 uebayasi Exp $ + * + * $FreeBSD$ + */ + +#ifndef _IF_VMXREG_H +#define _IF_VMXREG_H + +struct UPT1_TxStats { + uint64_t TSO_packets; + uint64_t TSO_bytes; + uint64_t ucast_packets; + uint64_t ucast_bytes; + uint64_t mcast_packets; + uint64_t mcast_bytes; + uint64_t bcast_packets; + uint64_t bcast_bytes; + uint64_t error; + uint64_t discard; +} __packed; + +struct UPT1_RxStats { + uint64_t LRO_packets; + uint64_t LRO_bytes; + uint64_t ucast_packets; + uint64_t ucast_bytes; + uint64_t mcast_packets; + uint64_t mcast_bytes; + uint64_t bcast_packets; + uint64_t bcast_bytes; + uint64_t nobuffer; + uint64_t error; +} __packed; + +/* Interrupt moderation levels */ +#define UPT1_IMOD_NONE 0 /* No moderation */ +#define UPT1_IMOD_HIGHEST 7 /* Least interrupts */ +#define UPT1_IMOD_ADAPTIVE 8 /* Adaptive interrupt moderation */ + +/* Hardware features */ +#define UPT1_F_CSUM 0x0001 /* Rx checksum verification */ +#define UPT1_F_RSS 0x0002 /* Receive side scaling */ +#define UPT1_F_VLAN 0x0004 /* VLAN tag stripping */ +#define UPT1_F_LRO 0x0008 /* Large receive offloading */ + +#define VMXNET3_BAR0_IMASK(irq) (0x000 + (irq) * 8) /* Interrupt mask */ +#define VMXNET3_BAR0_TXH(q) (0x600 + (q) * 8) /* Tx head */ +#define VMXNET3_BAR0_RXH1(q) (0x800 + (q) * 8) /* Ring1 Rx head */ +#define VMXNET3_BAR0_RXH2(q) (0xA00 + (q) * 8) /* Ring2 Rx head */ +#define VMXNET3_BAR1_VRRS 0x000 /* VMXNET3 revision report selection */ +#define VMXNET3_BAR1_UVRS 0x008 /* UPT version report selection */ +#define VMXNET3_BAR1_DSL 0x010 /* Driver shared address low */ +#define VMXNET3_BAR1_DSH 0x018 /* Driver shared address high */ +#define VMXNET3_BAR1_CMD 0x020 /* Command */ +#define VMXNET3_BAR1_MACL 0x028 /* MAC address low */ +#define VMXNET3_BAR1_MACH 0x030 /* MAC address high */ +#define VMXNET3_BAR1_INTR 0x038 /* Interrupt status */ +#define VMXNET3_BAR1_EVENT 0x040 /* Event status */ + +#define VMXNET3_CMD_ENABLE 0xCAFE0000 /* Enable VMXNET3 */ +#define VMXNET3_CMD_DISABLE 0xCAFE0001 /* Disable VMXNET3 */ +#define VMXNET3_CMD_RESET 0xCAFE0002 /* Reset device */ +#define VMXNET3_CMD_SET_RXMODE 0xCAFE0003 /* Set interface flags */ +#define VMXNET3_CMD_SET_FILTER 0xCAFE0004 /* Set address filter */ +#define VMXNET3_CMD_VLAN_FILTER 0xCAFE0005 /* Set VLAN filter */ +#define VMXNET3_CMD_GET_STATUS 0xF00D0000 /* Get queue errors */ +#define VMXNET3_CMD_GET_STATS 0xF00D0001 /* Get queue statistics */ +#define VMXNET3_CMD_GET_LINK 0xF00D0002 /* Get link status */ +#define VMXNET3_CMD_GET_MACL 0xF00D0003 /* Get MAC address low */ +#define VMXNET3_CMD_GET_MACH 0xF00D0004 /* Get MAC address high */ +#define VMXNET3_CMD_GET_INTRCFG 0xF00D0008 /* Get interrupt config */ + +#define VMXNET3_DMADESC_ALIGN 128 +#define VMXNET3_INIT_GEN 1 + +struct vmxnet3_txdesc { + uint64_t addr; + + uint32_t len:14; + uint32_t gen:1; /* Generation */ + uint32_t pad1:1; + uint32_t dtype:1; /* Descriptor type */ + uint32_t pad2:1; + uint32_t offload_pos:14; /* Offloading position */ + + uint32_t hlen:10; /* Header len */ + uint32_t offload_mode:2; /* Offloading mode */ + uint32_t eop:1; /* End of packet */ + uint32_t compreq:1; /* Completion request */ + uint32_t pad3:1; + uint32_t vtag_mode:1; /* VLAN tag insertion mode */ + uint32_t vtag:16; /* VLAN tag */ +} __packed; + +/* Offloading modes */ +#define VMXNET3_OM_NONE 0 +#define VMXNET3_OM_CSUM 2 +#define VMXNET3_OM_TSO 3 + +struct vmxnet3_txcompdesc { + uint32_t eop_idx:12; /* EOP index in Tx ring */ + uint32_t pad1:20; + + uint32_t pad2:32; + uint32_t pad3:32; + + uint32_t rsvd:24; + uint32_t type:7; + uint32_t gen:1; +} __packed; + +struct vmxnet3_rxdesc { + uint64_t addr; + + uint32_t len:14; + uint32_t btype:1; /* Buffer type */ + uint32_t dtype:1; /* Descriptor type */ + uint32_t rsvd:15; + uint32_t gen:1; + + uint32_t pad1:32; +} __packed; + +/* Buffer types */ +#define VMXNET3_BTYPE_HEAD 0 /* Head only */ +#define VMXNET3_BTYPE_BODY 1 /* Body only */ + +struct vmxnet3_rxcompdesc { + uint32_t rxd_idx:12; /* Rx descriptor index */ + uint32_t pad1:2; + uint32_t eop:1; /* End of packet */ + uint32_t sop:1; /* Start of packet */ + uint32_t qid:10; + uint32_t rss_type:4; + uint32_t no_csum:1; /* No checksum calculated */ + uint32_t pad2:1; + + uint32_t rss_hash:32; /* RSS hash value */ + + uint32_t len:14; + uint32_t error:1; + uint32_t vlan:1; /* 802.1Q VLAN frame */ + uint32_t vtag:16; /* VLAN tag */ + + uint32_t csum:16; + uint32_t csum_ok:1; /* TCP/UDP checksum ok */ + uint32_t udp:1; + uint32_t tcp:1; + uint32_t ipcsum_ok:1; /* IP checksum OK */ + uint32_t ipv6:1; + uint32_t ipv4:1; + uint32_t fragment:1; /* IP fragment */ + uint32_t fcs:1; /* Frame CRC correct */ + uint32_t type:7; + uint32_t gen:1; +} __packed; + +#define VMXNET3_REV1_MAGIC 0XBABEFEE1 + +#define VMXNET3_GOS_UNKNOWN 0x00 +#define VMXNET3_GOS_LINUX 0x04 +#define VMXNET3_GOS_WINDOWS 0x08 +#define VMXNET3_GOS_SOLARIS 0x0C +#define VMXNET3_GOS_FREEBSD 0x10 +#define VMXNET3_GOS_PXE 0x14 + +#define VMXNET3_GOS_32BIT 0x01 +#define VMXNET3_GOS_64BIT 0x02 + +#define VMXNET3_MAX_TX_QUEUES 8 +#define VMXNET3_MAX_RX_QUEUES 16 +#define VMXNET3_MAX_INTRS \ + (VMXNET3_MAX_TX_QUEUES + VMXNET3_MAX_RX_QUEUES + 1) + +#define VMXNET3_ICTRL_DISABLE_ALL 0x01 + +#define VMXNET3_RXMODE_UCAST 0x01 +#define VMXNET3_RXMODE_MCAST 0x02 +#define VMXNET3_RXMODE_BCAST 0x04 +#define VMXNET3_RXMODE_ALLMULTI 0x08 +#define VMXNET3_RXMODE_PROMISC 0x10 + +#define VMXNET3_EVENT_RQERROR 0x01 +#define VMXNET3_EVENT_TQERROR 0x02 +#define VMXNET3_EVENT_LINK 0x04 +#define VMXNET3_EVENT_DIC 0x08 +#define VMXNET3_EVENT_DEBUG 0x10 + +#define VMXNET3_MIN_MTU 60 +#define VMXNET3_MAX_MTU 9000 + +/* Interrupt mask mode. */ +#define VMXNET3_IMM_AUTO 0x00 +#define VMXNET3_IMM_ACTIVE 0x01 +#define VMXNET3_IMM_LAZY 0x02 + +/* Interrupt type. */ +#define VMXNET3_IT_AUTO 0x00 +#define VMXNET3_IT_LEGACY 0x01 +#define VMXNET3_IT_MSI 0x02 +#define VMXNET3_IT_MSIX 0x03 + +struct vmxnet3_driver_shared { + uint32_t magic; + uint32_t pad1; + + /* Misc. control */ + uint32_t version; /* Driver version */ + uint32_t guest; /* Guest OS */ + uint32_t vmxnet3_revision; /* Supported VMXNET3 revision */ + uint32_t upt_version; /* Supported UPT version */ + uint64_t upt_features; + uint64_t driver_data; + uint64_t queue_shared; + uint32_t driver_data_len; + uint32_t queue_shared_len; + uint32_t mtu; + uint16_t nrxsg_max; + uint8_t ntxqueue; + uint8_t nrxqueue; + uint32_t reserved1[4]; + + /* Interrupt control */ + uint8_t automask; + uint8_t nintr; + uint8_t evintr; + uint8_t modlevel[VMXNET3_MAX_INTRS]; + uint32_t ictrl; + uint32_t reserved2[2]; + + /* Receive filter parameters */ + uint32_t rxmode; + uint16_t mcast_tablelen; + uint16_t pad2; + uint64_t mcast_table; + uint32_t vlan_filter[4096 / 32]; + + struct { + uint32_t version; + uint32_t len; + uint64_t paddr; + } rss, pm, plugin; + + uint32_t event; + uint32_t reserved3[5]; +} __packed; + +struct vmxnet3_txq_shared { + /* Control */ + uint32_t npending; + uint32_t intr_threshold; + uint64_t reserved1; + + /* Config */ + uint64_t cmd_ring; + uint64_t data_ring; + uint64_t comp_ring; + uint64_t driver_data; + uint64_t reserved2; + uint32_t cmd_ring_len; + uint32_t data_ring_len; + uint32_t comp_ring_len; + uint32_t driver_data_len; + uint8_t intr_idx; + uint8_t pad1[7]; + + /* Queue status */ + uint8_t stopped; + uint8_t pad2[3]; + uint32_t error; + + struct UPT1_TxStats stats; + + uint8_t pad3[88]; +} __packed; + +struct vmxnet3_rxq_shared { + uint8_t update_rxhead; + uint8_t pad1[7]; + uint64_t reserved1; + + uint64_t cmd_ring[2]; + uint64_t comp_ring; + uint64_t driver_data; + uint64_t reserved2; + uint32_t cmd_ring_len[2]; + uint32_t comp_ring_len; + uint32_t driver_data_len; + uint8_t intr_idx; + uint8_t pad2[7]; + + uint8_t stopped; + uint8_t pad3[3]; + uint32_t error; + + struct UPT1_RxStats stats; + + uint8_t pad4[88]; +} __packed; + +#endif /* _IF_VMXREG_H */ diff --git a/sys/dev/vmware/vmxnet3/if_vmxvar.h b/sys/dev/vmware/vmxnet3/if_vmxvar.h new file mode 100644 index 0000000..44c1055 --- /dev/null +++ b/sys/dev/vmware/vmxnet3/if_vmxvar.h @@ -0,0 +1,289 @@ +/*- + * Copyright (c) 2013 Tsubai Masanari + * Copyright (c) 2013 Bryan Venteicher + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef _IF_VMXVAR_H +#define _IF_VMXVAR_H + +struct vmxnet3_softc; + +struct vmxnet3_dma_alloc { + bus_addr_t dma_paddr; + caddr_t dma_vaddr; + bus_dma_tag_t dma_tag; + bus_dmamap_t dma_map; + bus_size_t dma_size; +}; + +/* + * The number of Rx/Tx queues this driver supports. + */ +#define VMXNET3_RX_QUEUES 1 +#define VMXNET3_TX_QUEUES 1 + +/* + * The number of Rx rings in each Rx queue. + */ +#define VMXNET3_RXRINGS_PERQ 2 + +/* + * The maximum number of descriptors in each Rx/Tx ring. + */ +#define VMXNET3_MAX_TX_NDESC 512 +#define VMXNET3_MAX_RX_NDESC 256 +#define VMXNET3_MAX_TX_NCOMPDESC VMXNET3_MAX_TX_NDESC +#define VMXNET3_MAX_RX_NCOMPDESC \ + (VMXNET3_MAX_RX_NDESC * VMXNET3_RXRINGS_PERQ) + +/* + * The maximum number of Rx segments we accept. When LRO is enabled, + * this allows us to receive the maximum sized frame with one MCLBYTES + * cluster followed by 16 MJUMPAGESIZE clusters. + */ +#define VMXNET3_MAX_RX_SEGS 17 + +struct vmxnet3_txbuf { + bus_dmamap_t vtxb_dmamap; + struct mbuf *vtxb_m; +}; + +struct vmxnet3_txring { + struct vmxnet3_txbuf *vxtxr_txbuf; + u_int vxtxr_head; + u_int vxtxr_next; + u_int vxtxr_ndesc; + int vxtxr_gen; + bus_dma_tag_t vxtxr_txtag; + struct vmxnet3_txdesc *vxtxr_txd; + struct vmxnet3_dma_alloc vxtxr_dma; +}; + +#define VMXNET3_TXRING_AVAIL(_txr) \ + (((_txr)->vxtxr_head - (_txr)->vxtxr_head - 1) % VMXNET3_MAX_TX_NDESC) + +struct vmxnet3_rxbuf { + bus_dmamap_t vrxb_dmamap; + struct mbuf *vrxb_m; +}; + +struct vmxnet3_rxring { + struct vmxnet3_rxbuf *vxrxr_rxbuf; + struct vmxnet3_rxdesc *vxrxr_rxd; + u_int vxrxr_fill; + u_int vxrxr_ndesc; + int vxrxr_gen; + int vxrxr_rid; + bus_dma_tag_t vxrxr_rxtag; + struct vmxnet3_dma_alloc vxrxr_dma; + bus_dmamap_t vxrxr_spare_dmap; +}; + +static inline void +vmxnet3_rxr_increment_fill(struct vmxnet3_rxring *rxr) +{ + + if (++rxr->vxrxr_fill == rxr->vxrxr_ndesc) { + rxr->vxrxr_fill = 0; + rxr->vxrxr_gen ^= 1; + } +} + +struct vmxnet3_comp_ring { + union { + struct vmxnet3_txcompdesc *txcd; + struct vmxnet3_rxcompdesc *rxcd; + } vxcr_u; + u_int vxcr_next; + u_int vxcr_ndesc; + int vxcr_gen; + struct vmxnet3_dma_alloc vxcr_dma; +}; + +struct vmxnet3_txq_stats { + uint64_t vtxrs_full; + +}; + +struct vmxnet3_txqueue { + struct mtx vxtxq_mtx; + struct vmxnet3_softc *vxtxq_sc; + int vxtxq_id; + int vxtxq_intr_idx; + int vxtxq_watchdog; + struct vmxnet3_txring vxtxq_cmd_ring; + struct vmxnet3_comp_ring vxtxq_comp_ring; + struct vmxnet3_txq_stats vxtxq_stats; + struct vmxnet3_txq_shared *vxtxq_ts; + struct sysctl_oid_list *vxtxq_sysctl; + char vxtxq_name[16]; +}; + +#define VMXNET3_TXQ_LOCK(_txq) mtx_lock(&(_txq)->vxtxq_mtx) +#define VMXNET3_TXQ_TRYLOCK(_txq) mtx_trylock(&(_txq)->vxtxq_mtx) +#define VMXNET3_TXQ_UNLOCK(_txq) mtx_unlock(&(_txq)->vxtxq_mtx) +#define VMXNET3_TXQ_LOCK_ASSERT(_txq) \ + mtx_assert(&(_txq)->vxtxq_mtx, MA_OWNED) +#define VMXNET3_TXQ_LOCK_ASSERT_NOTOWNED(_txq) \ + mtx_assert(&(_txq)->vxtxq_mtx, MA_NOTOWNED) + +struct vmxnet3_rxq_stats { + +}; + +struct vmxnet3_rxqueue { + struct mtx vxrxq_mtx; + struct vmxnet3_softc *vxrxq_sc; + int vxrxq_id; + int vxrxq_intr_idx; + struct vmxnet3_rxring vxrxq_cmd_ring[VMXNET3_RXRINGS_PERQ]; + struct vmxnet3_comp_ring vxrxq_comp_ring; + struct vmxnet3_rxq_stats vxrxq_stats; + struct vmxnet3_rxq_shared *vxrxq_rs; + struct sysctl_oid_list *vxrxq_sysctl; + char vxrxq_name[16]; +}; + +#define VMXNET3_RXQ_LOCK(_rxq) mtx_lock(&(_rxq)->vxrxq_mtx) +#define VMXNET3_RXQ_UNLOCK(_rxq) mtx_unlock(&(_rxq)->vxrxq_mtx) +#define VMXNET3_RXQ_LOCK_ASSERT(_rxq) \ + mtx_assert(&(_rxq)->vxrxq_mtx, MA_OWNED) +#define VMXNET3_RXQ_LOCK_ASSERT_NOTOWNED(_rxq) \ + mtx_assert(&(_rxq)->vxrxq_mtx, MA_NOTOWNED) + +struct vmxnet3_statistics { + uint64_t vmst_collapsed; + +}; + +struct vmxnet3_interrupt { + struct resource *vmxi_irq; + int vmxi_rid; + void *vmxi_handler; +}; + +struct vmxnet3_softc { + device_t vmx_dev; + struct ifnet *vmx_ifp; + struct vmxnet3_driver_shared *vmx_ds; + uint32_t vmx_flags; +#define VMXNET3_FLAG_NO_MSIX 0x0001 + + struct vmxnet3_rxqueue *vmx_rxq; + struct vmxnet3_txqueue *vmx_txq; + + struct resource *vmx_res0; + bus_space_tag_t vmx_iot0; + bus_space_handle_t vmx_ioh0; + struct resource *vmx_res1; + bus_space_tag_t vmx_iot1; + bus_space_handle_t vmx_ioh1; + struct resource *vmx_msix_res; + + int vmx_link_active; + int vmx_link_speed; + int vmx_if_flags; + int vmx_ntxqueues; + int vmx_nrxqueues; + int vmx_ntxdescs; + int vmx_nrxdescs; + int vmx_max_rxsegs; + int vmx_rx_max_chain; + + struct vmxnet3_statistics vmx_stats; + + int vmx_intr_type; + int vmx_intr_mask_mode; + int vmx_event_intr_idx; + int vmx_nintrs; + struct vmxnet3_interrupt vmx_intrs[VMXNET3_MAX_INTRS]; + + struct mtx vmx_mtx; + uint8_t *vmx_mcast; + void *vmx_qs; + struct callout vmx_tick; + struct vmxnet3_dma_alloc vmx_ds_dma; + struct vmxnet3_dma_alloc vmx_qs_dma; + struct vmxnet3_dma_alloc vmx_mcast_dma; + struct ifmedia vmx_media; + eventhandler_tag vmx_vlan_attach; + eventhandler_tag vmx_vlan_detach; + uint8_t vmx_vlan_filter[4096/32]; + uint8_t vmx_lladdr[ETHER_ADDR_LEN]; +}; + +#define VMXNET3_CORE_LOCK_INIT(_sc, _name) \ + mtx_init(&(_sc)->vmx_mtx, _name, "VMXNET3 Lock", MTX_DEF) +#define VMXNET3_CORE_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->vmx_mtx) +#define VMXNET3_CORE_LOCK(_sc) mtx_lock(&(_sc)->vmx_mtx) +#define VMXNET3_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->vmx_mtx) +#define VMXNET3_CORE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->vmx_mtx, MA_OWNED) +#define VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(_sc) \ + mtx_assert(&(_sc)->vmx_mtx, MA_NOTOWNED) + +/* + * Our driver version we report to the hypervisor; we just keep + * this value constant. + */ +#define VMXNET3_DRIVER_VERSION 0x00010000 + +/* + * Convert the FreeBSD version in to something the hypervisor + * understands. This is apparently what VMware's driver reports + * so mimic it even though it probably is not required. + */ +#define VMXNET3_GUEST_OS_VERSION \ + (((__FreeBSD_version / 100000) << 14) | \ + (((__FreeBSD_version / 1000) % 100) << 6 ) | \ + (((__FreeBSD_version / 100) % 10) << 30) | \ + ((__FreeBSD_version % 100) << 22)) + +/* + * Max descriptors per Tx packet. We must limit the size of the + * any TSO packets based on the number of segments. + */ +#define VMXNET3_TX_MAXSEGS 32 +#define VMXNET3_TSO_MAXSIZE 65550 + +/* + * Maximum support Tx segments size. The length field in the + * Tx descriptor is 14 bits. + */ +#define VMXNET3_TX_MAXSEGSIZE (1 << 14) + +/* + * Predetermined size of the multicast MACs filter table. If the + * number of multicast addresses exceeds this size, then the + * ALL_MULTI mode is use instead. + */ +#define VMXNET3_MULTICAST_MAX 32 + +/* + * Our Tx watchdog timeout. + */ +#define VMXNET3_WATCHDOG_TIMEOUT 5 + +/* + * IP protocols that we can perform Tx checksum offloading of. + */ +#define VMXNET3_CSUM_OFFLOAD (CSUM_TCP | CSUM_UDP) +#define VMXNET3_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6) + +#define VMXNET3_CSUM_ALL_OFFLOAD \ + (VMXNET3_CSUM_OFFLOAD | VMXNET3_CSUM_OFFLOAD_IPV6 | CSUM_TSO) + +#endif /* _IF_VMXVAR_H */ diff --git a/sys/modules/Makefile b/sys/modules/Makefile index bd263e9..b051ac6 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -343,6 +343,7 @@ SUBDIR= \ vkbd \ ${_vmm} \ ${_vpo} \ + vmware \ vr \ vte \ vx \ diff --git a/sys/modules/vmware/Makefile b/sys/modules/vmware/Makefile new file mode 100644 index 0000000..695d344 --- /dev/null +++ b/sys/modules/vmware/Makefile @@ -0,0 +1,28 @@ +# +# $FreeBSD$ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +SUBDIR= vmxnet3 + +.include diff --git a/sys/modules/vmware/vmxnet3/Makefile b/sys/modules/vmware/vmxnet3/Makefile new file mode 100644 index 0000000..11cca58 --- /dev/null +++ b/sys/modules/vmware/vmxnet3/Makefile @@ -0,0 +1,45 @@ +# +# $FreeBSD$ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +.PATH: ${.CURDIR}/../../../dev/vmware/vmxnet3 + +KMOD= if_vmx +SRCS= if_vmx.c +SRCS+= bus_if.h device_if.h pci_if.h opt_inet.h opt_inet6.h +MFILES= kern/bus_if.m kern/device_if.m dev/pci/pci_if.m + +.if !defined(KERNBUILDDIR) +.if ${MK_INET_SUPPORT} != "no" +opt_inet.h: + @echo "#define INET 1" > ${.TARGET} +.endif + +.if ${MK_INET6_SUPPORT} != "no" +opt_inet6.h: + @echo "#define INET6 1" > ${.TARGET} +.endif +.endif + +.include