diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h index 6222ae8b08cc..17a27427a211 100644 --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -219,9 +219,10 @@ struct vi_info { /* These need to be int as they are used in sysctl */ int ntxq; /* # of tx queues */ int first_txq; /* index of first tx queue */ - int rsrv_noflowq; /* Reserve queue 0 for non-flowid packets */ + int rsrv_noflowq; /* Reserve tx queue 0 for non-flowid packets */ int nrxq; /* # of rx queues */ int first_rxq; /* index of first rx queue */ + int rsrv_norssq; /* Reserve rx queue 0 for non-RSS packets */ int nofldtxq; /* # of offload tx queues */ int first_ofld_txq; /* index of first offload tx queue */ int nofldrxq; /* # of offload rx queues */ diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index 2fa49ddee10b..87225d91d723 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -314,6 +314,10 @@ static int t4_rsrv_noflowq = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, rsrv_noflowq, CTLFLAG_RDTUN, &t4_rsrv_noflowq, 0, "Reserve TX queue 0 of each VI for non-flowid packets"); +static int t4_rsrv_norssq = 0; +SYSCTL_INT(_hw_cxgbe, OID_AUTO, rsrv_norssq, CTLFLAG_RDTUN, &t4_rsrv_norssq, + 0, "Reserve RX queue 0 of each VI for non-RSS packets"); + #if defined(TCP_OFFLOAD) || defined(RATELIMIT) #define NOFLDTXQ 8 static int t4_nofldtxq = -NOFLDTXQ; @@ -1568,6 +1572,10 @@ t4_attach(device_t dev) vi->rsrv_noflowq = t4_rsrv_noflowq ? 1 : 0; else vi->rsrv_noflowq = 0; + if (j == 0 && vi->nrxq > 1) + vi->rsrv_norssq = t4_rsrv_norssq ? 1 : 0; + else + vi->rsrv_norssq = 0; #if defined(TCP_OFFLOAD) || defined(RATELIMIT) vi->first_ofld_txq = ofld_tqidx; @@ -6782,7 +6790,7 @@ vi_full_init(struct vi_info *vi) { struct adapter *sc = vi->adapter; struct sge_rxq *rxq; - int rc, i, j; + int rc, i, j, nrxq; #ifdef RSS int nbuckets = rss_getnumbuckets(); int hashconfig = rss_gethashconfig(); @@ -6801,19 +6809,20 @@ vi_full_init(struct vi_info *vi) /* * Setup RSS for this VI. Save a copy of the RSS table for later use. */ - if (vi->nrxq > vi->rss_size) { + nrxq = (vi->rsrv_norssq != 0 && vi->nrxq > 1) ? vi->nrxq - 1 : vi->nrxq; + if (nrxq > vi->rss_size) { CH_ALERT(vi, "nrxq (%d) > hw RSS table size (%d); " - "some queues will never receive traffic.\n", vi->nrxq, + "some queues will never receive traffic.\n", nrxq, vi->rss_size); - } else if (vi->rss_size % vi->nrxq) { + } else if (vi->rss_size % nrxq) { CH_ALERT(vi, "nrxq (%d), hw RSS table size (%d); " - "expect uneven traffic distribution.\n", vi->nrxq, + "expect uneven traffic distribution.\n", nrxq, vi->rss_size); } #ifdef RSS - if (vi->nrxq != nbuckets) { + if (nrxq != nbuckets) { CH_ALERT(vi, "nrxq (%d) != kernel RSS buckets (%d);" - "performance will be impacted.\n", vi->nrxq, nbuckets); + "performance will be impacted.\n", nrxq, nbuckets); } #endif if (vi->rss == NULL) @@ -6822,11 +6831,15 @@ vi_full_init(struct vi_info *vi) for (i = 0; i < vi->rss_size;) { #ifdef RSS j = rss_get_indirection_to_bucket(i); - j %= vi->nrxq; + j %= nrxq; + if (vi->rsrv_norssq != 0) + j++; rxq = &sc->sge.rxq[vi->first_rxq + j]; vi->rss[i++] = rxq->iq.abs_id; #else for_each_rxq(vi, j, rxq) { + if (j == 0 && vi->rsrv_norssq != 0) + continue; vi->rss[i++] = rxq->iq.abs_id; if (i == vi->rss_size) break; @@ -6883,8 +6896,9 @@ vi_full_init(struct vi_info *vi) F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; #endif - rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, vi->rss[0], - 0, 0); + rxq = &sc->sge.rxq[vi->first_rxq]; + rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, + rxq->iq.abs_id, 0, 0); if (rc != 0) { CH_ERR(vi, "rss hash/defaultq config failed: %d\n", rc); return (rc); diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c index 053cf724a9ba..69547211b718 100644 --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -214,6 +214,10 @@ static int t4_tx_coalesce = 1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce, CTLFLAG_RWTUN, &t4_tx_coalesce, 0, "tx coalescing allowed"); +static int t4_rx_budget = 0; +SYSCTL_INT(_hw_cxgbe, OID_AUTO, rx_budget, CTLFLAG_RWTUN, &t4_rx_budget, 0, + "rx descriptors to process in one run of the rx handler"); + /* * The driver will make aggressive attempts at tx coalescing if it sees these * many packets eligible for coalescing in quick succession, with no more than @@ -227,8 +231,8 @@ static int t4_tx_coalesce_gap = 5; SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce_gap, CTLFLAG_RWTUN, &t4_tx_coalesce_gap, 0, "tx gap (in microseconds)"); -static int service_iq(struct sge_iq *, int); -static int service_iq_fl(struct sge_iq *, int); +static int service_iq(struct sge_iq *); +static int service_iq_fl(struct sge_iq *, int, bool); static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t); static int eth_rx(struct adapter *, struct sge_rxq *, const struct iq_desc *, u_int); @@ -1328,7 +1332,7 @@ t4_intr_evt(void *arg) struct sge_iq *iq = arg; if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { - service_iq(iq, 0); + service_iq(iq); (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); } } @@ -1342,7 +1346,7 @@ t4_intr(void *arg) struct sge_iq *iq = arg; if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { - service_iq_fl(iq, 0); + service_iq_fl(iq, t4_rx_budget, false); (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); } } @@ -1382,12 +1386,12 @@ t4_vi_intr(void *arg) * Deals with interrupts on an iq-only (no freelist) queue. */ static int -service_iq(struct sge_iq *iq, int budget) +service_iq(struct sge_iq *iq) { struct sge_iq *q; struct adapter *sc = iq->adapter; struct iq_desc *d = &iq->desc[iq->cidx]; - int ndescs = 0, limit; + int ndescs = 0; int rsp_type; uint32_t lq; STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); @@ -1399,8 +1403,6 @@ service_iq(struct sge_iq *iq, int budget) MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); MPASS((iq->flags & IQ_LRO_ENABLED) == 0); - limit = budget ? budget : iq->qsize / 16; - /* * We always come back and check the descriptor ring for new indirect * interrupts and other responses after running a single handler. @@ -1443,7 +1445,7 @@ service_iq(struct sge_iq *iq, int budget) sc->sge.iq_base]; if (atomic_cmpset_int(&q->state, IQS_IDLE, IQS_BUSY)) { - if (service_iq_fl(q, q->qsize / 16) == 0) { + if (service_iq_fl(q, 32, true) == 0) { (void) atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE); } else { @@ -1469,16 +1471,12 @@ service_iq(struct sge_iq *iq, int budget) iq->gen ^= F_RSPD_GEN; d = &iq->desc[0]; } - if (__predict_false(++ndescs == limit)) { + if (__predict_false(++ndescs == iq->qsize >> 4)) { t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); ndescs = 0; - - if (budget) { - return (EINPROGRESS); - } } } @@ -1491,7 +1489,7 @@ service_iq(struct sge_iq *iq, int budget) */ q = STAILQ_FIRST(&iql); STAILQ_REMOVE_HEAD(&iql, link); - if (service_iq_fl(q, q->qsize / 8) == 0) + if (service_iq_fl(q, 32, true) == 0) (void) atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE); else STAILQ_INSERT_TAIL(&iql, q, link); @@ -1569,13 +1567,13 @@ move_to_next_rxbuf(struct sge_fl *fl) * Deals with interrupts on an iq+fl queue. */ static int -service_iq_fl(struct sge_iq *iq, int budget) +service_iq_fl(struct sge_iq *iq, int budget, bool indirect) { struct sge_rxq *rxq = iq_to_rxq(iq); struct sge_fl *fl; struct adapter *sc = iq->adapter; struct iq_desc *d = &iq->desc[iq->cidx]; - int ndescs, limit; + int ndescs, total_ndescs; int rsp_type, starved; uint32_t lq; uint16_t fl_hw_cidx; @@ -1606,7 +1604,7 @@ service_iq_fl(struct sge_iq *iq, int budget) MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); #endif - limit = budget ? budget : iq->qsize / 16; + total_ndescs = 0; fl = &rxq->fl; fl_hw_cidx = fl->hw_cidx; /* stable snapshot */ while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { @@ -1679,7 +1677,20 @@ service_iq_fl(struct sge_iq *iq, int budget) iq->gen ^= F_RSPD_GEN; d = &iq->desc[0]; } - if (__predict_false(++ndescs == limit)) { + ndescs++; + total_ndescs++; + if (__predict_false(total_ndescs == budget)) { + if (indirect) { + t4_write_reg(sc, sc->sge_gts_reg, + V_CIDXINC(ndescs) | + V_INGRESSQID(iq->cntxt_id) | + V_SEINTARM(V_QINTR_TIMER_IDX( + X_TIMERREG_UPDATE_CIDX))); + return (EINPROGRESS); + } + break; + } + if (__predict_false(ndescs == iq->qsize >> 4)) { t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); @@ -1691,8 +1702,6 @@ service_iq_fl(struct sge_iq *iq, int budget) tcp_lro_flush_inactive(lro, &lro_timeout); } #endif - if (budget) - return (EINPROGRESS); ndescs = 0; } }