Index: dev/e1000/if_igb.c =================================================================== --- dev/e1000/if_igb.c (revision 229933) +++ dev/e1000/if_igb.c (working copy) @@ -172,10 +172,8 @@ static int igb_resume(device_t); #if __FreeBSD_version >= 800000 static int igb_mq_start(struct ifnet *, struct mbuf *); -static int igb_mq_start_locked(struct ifnet *, - struct tx_ring *, struct mbuf *); +static void igb_mq_start_locked(struct ifnet *, struct tx_ring *); static void igb_qflush(struct ifnet *); -static void igb_deferred_mq_start(void *, int); #else static void igb_start(struct ifnet *); static void igb_start_locked(struct tx_ring *, struct ifnet *ifp); @@ -802,7 +800,7 @@ for (int i = 0; i < adapter->num_queues; i++, txr++) { IGB_TX_LOCK(txr); if (!drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr, NULL); + igb_mq_start_locked(ifp, txr); IGB_TX_UNLOCK(txr); } #endif @@ -911,48 +909,38 @@ txr = &adapter->tx_rings[i]; que = &adapter->queues[i]; - if (IGB_TX_TRYLOCK(txr)) { - err = igb_mq_start_locked(ifp, txr, m); + err = drbr_enqueue(ifp, txr->br, m); + if (err == 0) { + IGB_TX_LOCK(txr); + igb_mq_start_locked(ifp, txr); IGB_TX_UNLOCK(txr); - } else { - err = drbr_enqueue(ifp, txr->br, m); - taskqueue_enqueue(que->tq, &txr->txq_task); - } + } else if (err == EINPROGRESS) + err = 0; return (err); } -static int -igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) +static void +igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct mbuf *next; - int err = 0, enq; + int enq; IGB_TX_LOCK_ASSERT(txr); if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING || adapter->link_active == 0) { - if (m != NULL) - err = drbr_enqueue(ifp, txr->br, m); - return (err); - } + IFF_DRV_RUNNING || adapter->link_active == 0) + return; enq = 0; - if (m == NULL) { - next = drbr_dequeue(ifp, txr->br); - } else if (drbr_needs_enqueue(ifp, txr->br)) { - if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) - return (err); - next = drbr_dequeue(ifp, txr->br); - } else - next = m; + next = drbr_dequeue(ifp, txr->br); /* Process the queue */ while (next != NULL) { - if ((err = igb_xmit(txr, &next)) != 0) { + if (igb_xmit(txr, &next) != 0) { if (next != NULL) - err = drbr_enqueue(ifp, txr->br, next); + drbr_enqueue(ifp, txr->br, next); break; } enq++; @@ -973,26 +961,9 @@ txr->queue_status = IGB_QUEUE_WORKING; txr->watchdog_time = ticks; } - return (err); } /* - * Called from a taskqueue to drain queued transmit packets. - */ -static void -igb_deferred_mq_start(void *arg, int pending) -{ - struct tx_ring *txr = arg; - struct adapter *adapter = txr->adapter; - struct ifnet *ifp = adapter->ifp; - - IGB_TX_LOCK(txr); - if (!drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr, NULL); - IGB_TX_UNLOCK(txr); -} - -/* ** Flush all ring buffers */ static void @@ -1339,7 +1310,7 @@ more = TRUE; #if __FreeBSD_version >= 800000 if (!drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr, NULL); + igb_mq_start_locked(ifp, txr); #else igb_start_locked(txr, ifp); #endif @@ -1463,7 +1434,7 @@ } while (loop-- && more); #if __FreeBSD_version >= 800000 if (!drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr, NULL); + igb_mq_start_locked(ifp, txr); #else igb_start_locked(txr, ifp); #endif @@ -2075,6 +2046,16 @@ device_printf(dev, "Link: thermal downshift\n"); /* This can sleep */ if_link_state_change(ifp, LINK_STATE_UP); +#if __FreeBSD_version < 800000 + igb_start(ifp); +#else + for (int i = 0; i < adapter->num_queues; i++, txr++) { + IGB_TX_LOCK(txr); + if (!drbr_empty(ifp, txr->br)) + igb_mq_start_locked(ifp, txr); + IGB_TX_UNLOCK(txr); + } +#endif } else if (!link_check && (adapter->link_active == 1)) { ifp->if_baudrate = adapter->link_speed = 0; adapter->link_duplex = 0; @@ -2212,7 +2193,6 @@ { device_t dev = adapter->dev; struct igb_queue *que = adapter->queues; - struct tx_ring *txr = adapter->tx_rings; int error, rid = 0; /* Turn off all interrupts */ @@ -2231,10 +2211,6 @@ return (ENXIO); } -#if __FreeBSD_version >= 800000 - TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr); -#endif - /* * Try allocating a fast interrupt and the associated deferred * processing contexts. @@ -2305,10 +2281,7 @@ */ if (adapter->num_queues > 1) bus_bind_intr(dev, que->res, i); -#if __FreeBSD_version >= 800000 - TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start, - que->txr); -#endif + /* Make tasklet for deferred handling */ TASK_INIT(&que->que_task, 0, igb_handle_que, que); que->tq = taskqueue_create("igb_que", M_NOWAIT, @@ -2529,9 +2502,6 @@ for (int i = 0; i < adapter->num_queues; i++, que++) { if (que->tq != NULL) { -#if __FreeBSD_version >= 800000 - taskqueue_drain(que->tq, &que->txr->txq_task); -#endif taskqueue_drain(que->tq, &que->que_task); taskqueue_free(que->tq); } Index: dev/e1000/if_igb.h =================================================================== --- dev/e1000/if_igb.h (revision 229933) +++ dev/e1000/if_igb.h (working copy) @@ -297,7 +297,6 @@ struct buf_ring *br; #endif bus_dma_tag_t txtag; - struct task txq_task; u32 bytes; u32 packets; Index: dev/e1000/if_em.c =================================================================== --- dev/e1000/if_em.c (revision 229933) +++ dev/e1000/if_em.c (working copy) @@ -192,13 +192,13 @@ static int em_shutdown(device_t); static int em_suspend(device_t); static int em_resume(device_t); -static void em_start(struct ifnet *); -static void em_start_locked(struct ifnet *, struct tx_ring *); #ifdef EM_MULTIQUEUE static int em_mq_start(struct ifnet *, struct mbuf *); -static int em_mq_start_locked(struct ifnet *, - struct tx_ring *, struct mbuf *); +static void em_mq_start_locked(struct ifnet *, struct tx_ring *); static void em_qflush(struct ifnet *); +#else +static void em_start(struct ifnet *); +static void em_start_locked(struct ifnet *, struct tx_ring *); #endif static int em_ioctl(struct ifnet *, u_long, caddr_t); static void em_init(void *); @@ -804,13 +804,25 @@ em_resume(device_t dev) { struct adapter *adapter = device_get_softc(dev); +#ifdef EM_MULTIQUEUE + struct tx_ring *txr = adapter->tx_rings; +#endif struct ifnet *ifp = adapter->ifp; EM_CORE_LOCK(adapter); em_init_locked(adapter); em_init_manageability(adapter); EM_CORE_UNLOCK(adapter); +#ifdef EM_MULTIQUEUE + for (int i = 0; i < adapter->num_queues; i++, txr++) { + EM_TX_LOCK(txr); + if (!drbr_empty(ifp, txr->br)) + em_mq_start_locked(ifp, txr); + EM_TX_UNLOCK(txr); + } +#else em_start(ifp); +#endif return bus_generic_resume(dev); } @@ -827,39 +839,29 @@ **********************************************************************/ #ifdef EM_MULTIQUEUE -static int -em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) +static void +em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct mbuf *next; - int err = 0, enq = 0; + int enq; if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING || adapter->link_active == 0) { - if (m != NULL) - err = drbr_enqueue(ifp, txr->br, m); - return (err); - } + IFF_DRV_RUNNING || adapter->link_active == 0) + return; /* Call cleanup if number of TX descriptors low */ if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) em_txeof(txr); enq = 0; - if (m == NULL) { - next = drbr_dequeue(ifp, txr->br); - } else if (drbr_needs_enqueue(ifp, txr->br)) { - if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) - return (err); - next = drbr_dequeue(ifp, txr->br); - } else - next = m; + next = drbr_dequeue(ifp, txr->br); /* Process the queue */ while (next != NULL) { - if ((err = em_xmit(txr, &next)) != 0) { + if (em_xmit(txr, &next) != 0) { if (next != NULL) - err = drbr_enqueue(ifp, txr->br, next); + drbr_enqueue(ifp, txr->br, next); break; } enq++; @@ -879,7 +881,6 @@ txr->queue_status = EM_QUEUE_WORKING; txr->watchdog_time = ticks; } - return (err); } /* @@ -892,11 +893,13 @@ struct tx_ring *txr = adapter->tx_rings; int error; - if (EM_TX_TRYLOCK(txr)) { - error = em_mq_start_locked(ifp, txr, m); + error = drbr_enqueue(ifp, txr->br, m); + if (error == 0) { + EM_TX_LOCK(txr); + em_mq_start_locked(ifp, txr); EM_TX_UNLOCK(txr); - } else - error = drbr_enqueue(ifp, txr->br, m); + } else if (error == EINPROGRESS) + error = 0; return (error); } @@ -920,7 +923,7 @@ if_qflush(ifp); } -#endif /* EM_MULTIQUEUE */ +#else /* !EM_MULTIQUEUE */ static void em_start_locked(struct ifnet *ifp, struct tx_ring *txr) @@ -985,6 +988,8 @@ return; } +#endif /* EM_MULTIQUEUE */ + /********************************************************************* * Ioctl entry point * @@ -1418,7 +1423,7 @@ em_txeof(txr); #ifdef EM_MULTIQUEUE if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr, NULL); + em_mq_start_locked(ifp, txr); #else em_start_locked(ifp, txr); #endif @@ -1491,7 +1496,7 @@ em_txeof(txr); #ifdef EM_MULTIQUEUE if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr, NULL); + em_mq_start_locked(ifp, txr); #else em_start_locked(ifp, txr); #endif @@ -1603,7 +1608,7 @@ em_txeof(txr); #ifdef EM_MULTIQUEUE if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr, NULL); + em_mq_start_locked(ifp, txr); #else em_start_locked(ifp, txr); #endif @@ -2285,6 +2290,16 @@ adapter->smartspeed = 0; ifp->if_baudrate = adapter->link_speed * 1000000; if_link_state_change(ifp, LINK_STATE_UP); +#ifdef EM_MULTIQUEUE + for (int i = 0; i < adapter->num_queues; i++, txr++) { + EM_TX_LOCK(txr); + if (!drbr_empty(ifp, txr->br)) + em_mq_start_locked(ifp, txr); + EM_TX_UNLOCK(txr); + } +#else + em_start(ifp); +#endif } else if (!link_check && (adapter->link_active == 1)) { ifp->if_baudrate = adapter->link_speed = 0; adapter->link_duplex = 0; @@ -2816,7 +2831,13 @@ ifp->if_softc = adapter; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = em_ioctl; +#ifdef EM_MULTIQUEUE + /* Multiqueue tx functions */ + ifp->if_transmit = em_mq_start; + ifp->if_qflush = em_qflush; +#else ifp->if_start = em_start; +#endif IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; IFQ_SET_READY(&ifp->if_snd); @@ -2825,11 +2846,6 @@ ifp->if_capabilities = ifp->if_capenable = 0; -#ifdef EM_MULTIQUEUE - /* Multiqueue tx functions */ - ifp->if_transmit = em_mq_start; - ifp->if_qflush = em_qflush; -#endif ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; ifp->if_capenable |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; Index: dev/cxgbe/adapter.h =================================================================== --- dev/cxgbe/adapter.h (revision 229933) +++ dev/cxgbe/adapter.h (working copy) @@ -621,7 +621,7 @@ void t4_intr_err(void *); void t4_intr_evt(void *); int t4_mgmt_tx(struct adapter *, struct mbuf *); -int t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *); +void t4_eth_tx(struct ifnet *, struct sge_txq *); void t4_update_fl_bufsize(struct ifnet *); #endif Index: dev/cxgbe/t4_sge.c =================================================================== --- dev/cxgbe/t4_sge.c (revision 229933) +++ dev/cxgbe/t4_sge.c (working copy) @@ -838,14 +838,14 @@ sizeof(struct cpl_tx_pkt_core) \ ) / 8 ) -int -t4_eth_tx(struct ifnet *ifp, struct sge_txq *txq, struct mbuf *m) +void +t4_eth_tx(struct ifnet *ifp, struct sge_txq *txq) { struct port_info *pi = (void *)ifp->if_softc; struct adapter *sc = pi->adapter; struct sge_eq *eq = &txq->eq; struct buf_ring *br = txq->br; - struct mbuf *next; + struct mbuf *m, *next; int rc, coalescing, can_reclaim; struct txpkts txpkts; struct sgl sgl; @@ -862,7 +862,8 @@ if (eq->avail < 8) reclaim_tx_descs(txq, 0, 8); - for (; m; m = next ? next : drbr_dequeue(ifp, br)) { + for (m = txq->m ? txq->m : drbr_dequeue(ifp, br); m; + m = next ? next : drbr_dequeue(ifp, br)) { if (eq->avail < 8) break; @@ -977,8 +978,6 @@ can_reclaim = reclaimable(eq); if (can_reclaim >= 32) reclaim_tx_descs(txq, can_reclaim, 128); - - return (0); } void Index: dev/cxgbe/t4_main.c =================================================================== --- dev/cxgbe/t4_main.c (revision 229933) +++ dev/cxgbe/t4_main.c (working copy) @@ -118,7 +118,6 @@ /* ifnet + media interface */ static void cxgbe_init(void *); static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); -static void cxgbe_start(struct ifnet *); static int cxgbe_transmit(struct ifnet *, struct mbuf *); static void cxgbe_qflush(struct ifnet *); static int cxgbe_media_change(struct ifnet *); @@ -301,7 +300,6 @@ #ifdef T4_DEVLOG static int sysctl_devlog(SYSCTL_HANDLER_ARGS); #endif -static inline void txq_start(struct ifnet *, struct sge_txq *); static uint32_t fconf_to_mode(uint32_t); static uint32_t mode_to_fconf(uint32_t); static uint32_t fspec_to_fconf(struct t4_filter_specification *); @@ -774,14 +772,9 @@ ifp->if_init = cxgbe_init; ifp->if_ioctl = cxgbe_ioctl; - ifp->if_start = cxgbe_start; ifp->if_transmit = cxgbe_transmit; ifp->if_qflush = cxgbe_qflush; - ifp->if_snd.ifq_drv_maxlen = 1024; - IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); - IFQ_SET_READY(&ifp->if_snd); - ifp->if_capabilities = T4_CAP; ifp->if_capenable = T4_CAP_ENABLE; ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO; @@ -1020,21 +1013,6 @@ return (rc); } -static void -cxgbe_start(struct ifnet *ifp) -{ - struct port_info *pi = ifp->if_softc; - struct sge_txq *txq; - int i; - - for_each_txq(pi, i, txq) { - if (TXQ_TRYLOCK(txq)) { - txq_start(ifp, txq); - TXQ_UNLOCK(txq); - } - } -} - static int cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) { @@ -1055,43 +1033,13 @@ txq += (m->m_pkthdr.flowid % pi->ntxq); br = txq->br; - if (TXQ_TRYLOCK(txq) == 0) { - /* - * XXX: make sure that this packet really is sent out. There is - * a small race where t4_eth_tx may stop draining the drbr and - * goes away, just before we enqueued this mbuf. - */ - - return (drbr_enqueue(ifp, br, m)); - } - - /* - * txq->m is the mbuf that is held up due to a temporary shortage of - * resources and it should be put on the wire first. Then what's in - * drbr and finally the mbuf that was just passed in to us. - * - * Return code should indicate the fate of the mbuf that was passed in - * this time. - */ - - TXQ_LOCK_ASSERT_OWNED(txq); - if (drbr_needs_enqueue(ifp, br) || txq->m) { - - /* Queued for transmission. */ - - rc = drbr_enqueue(ifp, br, m); - m = txq->m ? txq->m : drbr_dequeue(ifp, br); - (void) t4_eth_tx(ifp, txq, m); + rc = drbr_enqueue(ifp, br, m); + if (rc == 0) { + TXQ_LOCK(txq); + t4_eth_tx(ifp, txq); TXQ_UNLOCK(txq); - return (rc); - } - - /* Direct transmission. */ - rc = t4_eth_tx(ifp, txq, m); - if (rc != 0 && txq->m) - rc = 0; /* held, will be transmitted soon (hopefully) */ - - TXQ_UNLOCK(txq); + } else if (rc == EINPROGRESS) + rc = 0; return (rc); } @@ -2891,20 +2839,6 @@ } #endif -static inline void -txq_start(struct ifnet *ifp, struct sge_txq *txq) -{ - struct buf_ring *br; - struct mbuf *m; - - TXQ_LOCK_ASSERT_OWNED(txq); - - br = txq->br; - m = txq->m ? txq->m : drbr_dequeue(ifp, br); - if (m) - t4_eth_tx(ifp, txq, m); -} - void cxgbe_txq_start(void *arg, int count) { @@ -2913,7 +2847,7 @@ TXQ_LOCK(txq); if (txq->eq.flags & EQ_CRFLUSHED) { txq->eq.flags &= ~EQ_CRFLUSHED; - txq_start(txq->ifp, txq); + t4_eth_tx(txq->ifp, txq); } else wakeup_one(txq); /* txq is going away, wakeup free_txq */ TXQ_UNLOCK(txq); Index: dev/cxgb/cxgb_sge.c =================================================================== --- dev/cxgb/cxgb_sge.c (revision 229933) +++ dev/cxgb/cxgb_sge.c (working copy) @@ -1685,60 +1685,33 @@ if (txq->db_pending) check_ring_tx_db(pi->adapter, txq, 1); +#if 1 + /* + * XXX: Should be handled by the ISR noticing TX completions + * and thus more room in the queue? + */ if (!TXQ_RING_EMPTY(qs) && callout_pending(&txq->txq_timer) == 0 && pi->link_config.link_ok) callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout, qs, txq->txq_timer.c_cpu); +#endif if (m_head != NULL) m_freem(m_head); } -static int -cxgb_transmit_locked(struct ifnet *ifp, struct sge_qset *qs, struct mbuf *m) +static void +cxgb_transmit_locked(struct ifnet *ifp, struct sge_qset *qs) { struct port_info *pi = qs->port; struct sge_txq *txq = &qs->txq[TXQ_ETH]; struct buf_ring *br = txq->txq_mr; - int error, avail; - avail = txq->size - txq->in_use; TXQ_LOCK_ASSERT(qs); - /* - * We can only do a direct transmit if the following are true: - * - we aren't coalescing (ring < 3/4 full) - * - the link is up -- checked in caller - * - there are no packets enqueued already - * - there is space in hardware transmit queue - */ - if (check_pkt_coalesce(qs) == 0 && - !TXQ_RING_NEEDS_ENQUEUE(qs) && avail > TX_MAX_DESC) { - if (t3_encap(qs, &m)) { - if (m != NULL && - (error = drbr_enqueue(ifp, br, m)) != 0) - return (error); - } else { - if (txq->db_pending) - check_ring_tx_db(pi->adapter, txq, 1); - - /* - * We've bypassed the buf ring so we need to update - * the stats directly - */ - txq->txq_direct_packets++; - txq->txq_direct_bytes += m->m_pkthdr.len; - } - } else if ((error = drbr_enqueue(ifp, br, m)) != 0) - return (error); - reclaim_completed_tx(qs, cxgb_tx_reclaim_threshold, TXQ_ETH); if (!TXQ_RING_EMPTY(qs) && pi->link_config.link_ok && (!check_pkt_coalesce(qs) || (drbr_inuse(ifp, br) >= 7))) cxgb_start_locked(qs); - else if (!TXQ_RING_EMPTY(qs) && !callout_pending(&txq->txq_timer)) - callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout, - qs, txq->txq_timer.c_cpu); - return (0); } int @@ -1758,29 +1731,18 @@ qidx = (m->m_pkthdr.flowid % pi->nqsets) + pi->first_qset; qs = &pi->adapter->sge.qs[qidx]; - - if (TXQ_TRYLOCK(qs)) { - /* XXX running */ - error = cxgb_transmit_locked(ifp, qs, m); + + error = drbr_enqueue(ifp, qs->txq[TXQ_ETH].txq_mr, m); + if (error == 0) { + TXQ_LOCK(qs); + cxgb_transmit_locked(ifp, qs); TXQ_UNLOCK(qs); - } else - error = drbr_enqueue(ifp, qs->txq[TXQ_ETH].txq_mr, m); + } else if (error == EINPROGRESS) { + error = 0; + } return (error); } -void -cxgb_start(struct ifnet *ifp) -{ - struct port_info *pi = ifp->if_softc; - struct sge_qset *qs = &pi->adapter->sge.qs[pi->first_qset]; - - if (!pi->link_config.link_ok) - return; - TXQ_LOCK(qs); - cxgb_start_locked(qs); - TXQ_UNLOCK(qs); -} - void cxgb_qflush(struct ifnet *ifp) { Index: dev/cxgb/cxgb_main.c =================================================================== --- dev/cxgb/cxgb_main.c (revision 229933) +++ dev/cxgb/cxgb_main.c (working copy) @@ -1023,12 +1023,9 @@ ifp->if_softc = p; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = cxgb_ioctl; - ifp->if_start = cxgb_start; + ifp->if_transmit = cxgb_transmit; + ifp->if_qflush = cxgb_qflush; - ifp->if_snd.ifq_drv_maxlen = max(cxgb_snd_queue_len, ifqmaxlen); - IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); - IFQ_SET_READY(&ifp->if_snd); - ifp->if_capabilities = CXGB_CAP; ifp->if_capenable = CXGB_CAP_ENABLE; ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO; @@ -1043,8 +1040,6 @@ } ether_ifattach(ifp, p->hw_addr); - ifp->if_transmit = cxgb_transmit; - ifp->if_qflush = cxgb_qflush; #ifdef DEFAULT_JUMBO if (sc->params.nports <= 2) Index: dev/cxgb/cxgb_adapter.h =================================================================== --- dev/cxgb/cxgb_adapter.h (revision 229933) +++ dev/cxgb/cxgb_adapter.h (working copy) @@ -572,5 +572,4 @@ void cxgb_tx_watchdog(void *arg); int cxgb_transmit(struct ifnet *ifp, struct mbuf *m); void cxgb_qflush(struct ifnet *ifp); -void cxgb_start(struct ifnet *ifp); #endif Index: sys/buf_ring.h =================================================================== --- sys/buf_ring.h (revision 229933) +++ sys/buf_ring.h (working copy) @@ -49,10 +49,12 @@ uint64_t br_drops; uint64_t br_prod_bufs; uint64_t br_prod_bytes; + volatile int br_busy; /* * Pad out to next L2 cache line */ - uint64_t _pad0[11]; + uint32_t _pad0[1]; + uint64_t _pad1[10]; volatile uint32_t br_cons_head; volatile uint32_t br_cons_tail; @@ -62,7 +64,7 @@ /* * Pad out to next L2 cache line */ - uint64_t _pad1[14]; + uint64_t _pad2[14]; #ifdef DEBUG_BUFRING struct mtx *br_lock; #endif @@ -78,7 +80,7 @@ { uint32_t prod_head, prod_next; uint32_t cons_tail; - int success; + int busy, success; #ifdef DEBUG_BUFRING int i; for (i = br->br_cons_head; i != br->br_prod_head; @@ -98,16 +100,16 @@ critical_exit(); return (ENOBUFS); } - + success = atomic_cmpset_int(&br->br_prod_head, prod_head, prod_next); } while (success == 0); #ifdef DEBUG_BUFRING if (br->br_ring[prod_head] != NULL) panic("dangling value in enqueue"); -#endif - br->br_ring[prod_head] = buf; - wmb(); +#endif + atomic_store_rel_ptr((uintptr_t *)&br->br_ring[prod_head], + (uintptr_t)buf); /* * If there are other enqueues in progress @@ -119,8 +121,9 @@ br->br_prod_bufs++; br->br_prod_bytes += nbytes; br->br_prod_tail = prod_next; + busy = atomic_load_acq_int(&br->br_busy); critical_exit(); - return (0); + return (busy ? EINPROGRESS : 0); } static __inline int @@ -158,11 +161,10 @@ cons_next); } while (success == 0); - buf = br->br_ring[cons_head]; + buf = (void *)atomic_load_acq_ptr((uintptr_t *)&br->br_ring[cons_head]); #ifdef DEBUG_BUFRING br->br_ring[cons_head] = NULL; #endif - rmb(); /* * If there are other dequeues in progress @@ -196,8 +198,10 @@ cons_next = (cons_head + 1) & br->br_cons_mask; cons_next_next = (cons_head + 2) & br->br_cons_mask; - if (cons_head == prod_tail) + if (cons_head == prod_tail) { + KASSERT(br->br_busy == 0, ("empty buf_ring still busy")); return (NULL); + } #ifdef PREFETCH_DEFINED if (cons_next != prod_tail) { @@ -206,7 +210,8 @@ prefetch(br->br_ring[cons_next_next]); } #endif - br->br_cons_head = cons_next; + br->br_busy = (cons_next != prod_tail); + atomic_store_rel_int(&br->br_cons_head, cons_next); buf = br->br_ring[cons_head]; #ifdef DEBUG_BUFRING