Index: sys/dev/bge/if_bgereg.h =================================================================== --- sys/dev/bge/if_bgereg.h (revision 212191) +++ sys/dev/bge/if_bgereg.h (working copy) @@ -829,6 +829,7 @@ #define BGE_SDI_STATS_CTL 0x0C08 #define BGE_SDI_STATS_ENABLE_MASK 0x0C0C #define BGE_SDI_STATS_INCREMENT_MASK 0x0C10 +#define BGE_ISO_PKT_TX_SUPPORT 0x0C20 #define BGE_LOCSTATS_COS0 0x0C80 #define BGE_LOCSTATS_COS1 0x0C84 #define BGE_LOCSTATS_COS2 0x0C88 @@ -2257,6 +2258,7 @@ #define BGE_JUMBO_RX_RING_CNT 256 #define BGE_MINI_RX_RING_CNT 1024 #define BGE_RETURN_RING_CNT 1024 +#define BGE_RX_INTERNAL_RING_CNT_5906 32 /* 5705 has smaller return ring size */ @@ -2561,6 +2563,8 @@ struct mbuf *bge_tx_chain[BGE_TX_RING_CNT]; struct mbuf *bge_rx_std_chain[BGE_STD_RX_RING_CNT]; struct mbuf *bge_rx_jumbo_chain[BGE_JUMBO_RX_RING_CNT]; + int bge_rx_std_seglen[BGE_STD_RX_RING_CNT]; + int bge_rx_jumbo_seglen[BGE_JUMBO_RX_RING_CNT][4]; }; struct bge_dmamap_arg { @@ -2650,6 +2654,10 @@ int bge_link_evt; /* pending link event */ int bge_timer; int bge_forced_collapse; + int bge_rx_jumbo_pending; + int bge_rx_jumbo_thresh; + int bge_rx_std_pending; + int bge_rx_std_thresh; struct callout bge_stat_ch; uint32_t bge_rx_discards; uint32_t bge_tx_discards; Index: sys/dev/bge/if_bge.c =================================================================== --- sys/dev/bge/if_bge.c (revision 212191) +++ sys/dev/bge/if_bge.c (working copy) @@ -400,6 +400,10 @@ static void bge_setmulti(struct bge_softc *); static void bge_setvlan(struct bge_softc *); +static __inline void bge_rxpost_std(struct bge_softc *); +static __inline void bge_rxpost_jumbo(struct bge_softc *); +static __inline void bge_rxreuse_std(struct bge_softc *, int); +static __inline void bge_rxreuse_jumbo(struct bge_softc *, int); static int bge_newbuf_std(struct bge_softc *, int); static int bge_newbuf_jumbo(struct bge_softc *, int); static int bge_init_rx_ring_std(struct bge_softc *); @@ -949,6 +953,7 @@ sc->bge_cdata.bge_rx_std_dmamap[i] = sc->bge_cdata.bge_rx_std_sparemap; sc->bge_cdata.bge_rx_std_sparemap = map; sc->bge_cdata.bge_rx_std_chain[i] = m; + sc->bge_cdata.bge_rx_std_seglen[i] = segs[0].ds_len; r = &sc->bge_ldata.bge_rx_std_ring[sc->bge_std]; r->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[0].ds_addr); r->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[0].ds_addr); @@ -1006,6 +1011,11 @@ sc->bge_cdata.bge_rx_jumbo_sparemap; sc->bge_cdata.bge_rx_jumbo_sparemap = map; sc->bge_cdata.bge_rx_jumbo_chain[i] = m; + sc->bge_cdata.bge_rx_jumbo_seglen[i][0] = 0; + sc->bge_cdata.bge_rx_jumbo_seglen[i][1] = 0; + sc->bge_cdata.bge_rx_jumbo_seglen[i][2] = 0; + sc->bge_cdata.bge_rx_jumbo_seglen[i][3] = 0; + /* * Fill in the extended RX buffer descriptor. */ @@ -1018,18 +1028,22 @@ r->bge_addr3.bge_addr_lo = BGE_ADDR_LO(segs[3].ds_addr); r->bge_addr3.bge_addr_hi = BGE_ADDR_HI(segs[3].ds_addr); r->bge_len3 = segs[3].ds_len; + sc->bge_cdata.bge_rx_jumbo_seglen[i][3] = segs[3].ds_len; case 3: r->bge_addr2.bge_addr_lo = BGE_ADDR_LO(segs[2].ds_addr); r->bge_addr2.bge_addr_hi = BGE_ADDR_HI(segs[2].ds_addr); r->bge_len2 = segs[2].ds_len; + sc->bge_cdata.bge_rx_jumbo_seglen[i][2] = segs[2].ds_len; case 2: r->bge_addr1.bge_addr_lo = BGE_ADDR_LO(segs[1].ds_addr); r->bge_addr1.bge_addr_hi = BGE_ADDR_HI(segs[1].ds_addr); r->bge_len1 = segs[1].ds_len; + sc->bge_cdata.bge_rx_jumbo_seglen[i][1] = segs[1].ds_len; case 1: r->bge_addr0.bge_addr_lo = BGE_ADDR_LO(segs[0].ds_addr); r->bge_addr0.bge_addr_hi = BGE_ADDR_HI(segs[0].ds_addr); r->bge_len0 = segs[0].ds_len; + sc->bge_cdata.bge_rx_jumbo_seglen[i][0] = segs[0].ds_len; break; default: panic("%s: %d segments\n", __func__, nsegs); @@ -1041,12 +1055,6 @@ return (0); } -/* - * The standard receive ring has 512 entries in it. At 2K per mbuf cluster, - * that's 1MB or memory, which is a lot. For now, we fill only the first - * 256 ring entries and hope that our CPU is fast enough to keep up with - * the NIC. - */ static int bge_init_rx_ring_std(struct bge_softc *sc) { @@ -1054,18 +1062,17 @@ bzero(sc->bge_ldata.bge_rx_std_ring, BGE_STD_RX_RING_SZ); sc->bge_std = 0; - for (i = 0; i < BGE_SSLOTS; i++) { + sc->bge_rx_std_pending = 0; + bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag, + sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_PREWRITE); + for (i = 0; i < BGE_STD_RX_RING_CNT; i++) { if ((error = bge_newbuf_std(sc, i)) != 0) return (error); BGE_INC(sc->bge_std, BGE_STD_RX_RING_CNT); + bge_rxpost_std(sc); }; + bge_rxpost_std(sc); - bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag, - sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_PREWRITE); - - sc->bge_std = i - 1; - bge_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bge_std); - return (0); } @@ -1097,24 +1104,21 @@ bzero(sc->bge_ldata.bge_rx_jumbo_ring, BGE_JUMBO_RX_RING_SZ); sc->bge_jumbo = 0; + sc->bge_rx_jumbo_pending = 0; + bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag, + sc->bge_cdata.bge_rx_jumbo_ring_map, BUS_DMASYNC_PREWRITE); + rcb = &sc->bge_ldata.bge_info.bge_jumbo_rx_rcb; + rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, + BGE_RCB_FLAG_USE_EXT_RX_BD); + CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags); for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) { if ((error = bge_newbuf_jumbo(sc, i)) != 0) return (error); BGE_INC(sc->bge_jumbo, BGE_JUMBO_RX_RING_CNT); + bge_rxpost_jumbo(sc); }; + bge_rxpost_jumbo(sc); - bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag, - sc->bge_cdata.bge_rx_jumbo_ring_map, BUS_DMASYNC_PREWRITE); - - sc->bge_jumbo = i - 1; - - rcb = &sc->bge_ldata.bge_info.bge_jumbo_rx_rcb; - rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, - BGE_RCB_FLAG_USE_EXT_RX_BD); - CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags); - - bge_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bge_jumbo); - return (0); } @@ -1610,26 +1614,21 @@ rcb->bge_maxlen_flags); } - /* - * Set the BD ring replentish thresholds. The recommended - * values are 1/8th the number of descriptors allocated to - * each ring. - * XXX The 5754 requires a lower threshold, so it might be a - * requirement of all 575x family chips. The Linux driver sets - * the lower threshold for all 5705 family chips as well, but there - * are reports that it might not need to be so strict. - * - * XXX Linux does some extra fiddling here for the 5906 parts as - * well. - */ - if (BGE_IS_5705_PLUS(sc)) - val = 8; - else - val = BGE_STD_RX_RING_CNT / 8; - CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val); + if (sc->bge_asicrev == BGE_ASICREV_BCM5906 && + sc->bge_chipid == BGE_CHIPID_BCM5906_A1) { + /* + * Select de-pipeline method, 1 data packet + * inside TXMBUF at a time. + */ + CSR_WRITE_4(sc, BGE_ISO_PKT_TX_SUPPORT, + (CSR_READ_4(sc, BGE_ISO_PKT_TX_SUPPORT) & ~0x03) | 0x02); + } + + /* Set the BD ring replenish thresholds. */ + CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, sc->bge_rx_std_thresh); if (BGE_IS_JUMBO_CAPABLE(sc)) CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH, - BGE_JUMBO_RX_RING_CNT/8); + sc->bge_rx_jumbo_thresh); /* * Disable all unused send rings by setting the 'ring disabled' @@ -2692,7 +2691,39 @@ */ if (BGE_IS_5714_FAMILY(sc) && (sc->bge_flags & BGE_FLAG_PCIX)) sc->bge_flags |= BGE_FLAG_40BIT_BUG; + /* + * Set the BD ring replenish thresholds. The device will not + * initiate a DMA for fetching the Rx BDs until the number of + * BDs made available to the device by the host is at least the + * value programmed in this register. The recommended values + * are 1/8th the number of descriptors allocated to Mini ring + * and Jumbo ring. The Standard RX producer threshold value + * should be set very low. + */ + sc->bge_rx_std_thresh = BGE_STD_RX_RING_CNT / 16; + sc->bge_rx_jumbo_thresh = BGE_JUMBO_RX_RING_CNT / 8; + switch (sc->bge_asicrev) { + case BGE_ASICREV_BCM5750: + case BGE_ASICREV_BCM5752: + case BGE_ASICREV_BCM5755: + /* + * Limit burst size of RX BDs being DMA'ed for 5750, + * 5752 and 5755 to workaround hardware errata. + */ + sc->bge_rx_std_thresh = 8; + break; + case BGE_ASICREV_BCM5906: + if (sc->bge_chipid == BGE_CHIPID_BCM5906_A1 && + sc->bge_rx_std_thresh > BGE_RX_INTERNAL_RING_CNT_5906 / 2) + sc->bge_rx_std_thresh = + BGE_RX_INTERNAL_RING_CNT_5906 / 2; + break; + default: + break; + } + + /* * Allocate the interrupt, using MSI if possible. These devices * support 8 MSI messages, but only the first one is used in * normal operation. @@ -3255,6 +3286,67 @@ return(0); } +static __inline void +bge_rxpost_std(struct bge_softc *sc) +{ + + sc->bge_rx_std_pending++; + if (sc->bge_rx_std_pending < sc->bge_rx_std_thresh) + return; + bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag, + sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag, + sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_PREWRITE); + bge_writembx(sc, BGE_MBX_RX_STD_PROD_LO, (sc->bge_std + + BGE_STD_RX_RING_CNT - 1) % BGE_STD_RX_RING_CNT); + sc->bge_rx_std_pending = 0; +} + +static __inline void +bge_rxreuse_std(struct bge_softc *sc, int i) +{ + struct bge_rx_bd *r; + + r = &sc->bge_ldata.bge_rx_std_ring[sc->bge_std]; + r->bge_flags = BGE_RXBDFLAG_END; + r->bge_len = sc->bge_cdata.bge_rx_std_seglen[i]; + r->bge_idx = i; + BGE_INC(sc->bge_std, BGE_STD_RX_RING_CNT); + bge_rxpost_std(sc); +} + +static __inline void +bge_rxpost_jumbo(struct bge_softc *sc) +{ + + sc->bge_rx_jumbo_pending++; + if (sc->bge_rx_jumbo_pending < sc->bge_rx_jumbo_thresh) + return; + bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag, + sc->bge_cdata.bge_rx_jumbo_ring_map, BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag, + sc->bge_cdata.bge_rx_jumbo_ring_map, BUS_DMASYNC_PREWRITE); + bge_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, (sc->bge_jumbo + + BGE_JUMBO_RX_RING_CNT - 1) % BGE_JUMBO_RX_RING_CNT); + sc->bge_rx_jumbo_pending = 0; +} + +static __inline void +bge_rxreuse_jumbo(struct bge_softc *sc, int i) +{ + struct bge_extrx_bd *r; + + r = &sc->bge_ldata.bge_rx_jumbo_ring[sc->bge_jumbo]; + r->bge_flags = BGE_RXBDFLAG_JUMBO_RING | BGE_RXBDFLAG_END; + r->bge_len0 = sc->bge_cdata.bge_rx_jumbo_seglen[i][0]; + r->bge_len1 = sc->bge_cdata.bge_rx_jumbo_seglen[i][1]; + r->bge_len2 = sc->bge_cdata.bge_rx_jumbo_seglen[i][2]; + r->bge_len3 = sc->bge_cdata.bge_rx_jumbo_seglen[i][3]; + r->bge_idx = i; + BGE_INC(sc->bge_jumbo, BGE_JUMBO_RX_RING_CNT); + bge_rxpost_jumbo(sc); +} + /* * Frame reception handling. This is called if there's a frame * on the receive return list. @@ -3268,7 +3360,6 @@ bge_rxeof(struct bge_softc *sc, uint16_t rx_prod, int holdlck) { struct ifnet *ifp; - int stdcnt = 0, jumbocnt = 0; uint16_t rx_cons; rx_cons = sc->bge_rx_saved_considx; @@ -3281,12 +3372,6 @@ bus_dmamap_sync(sc->bge_cdata.bge_rx_return_ring_tag, sc->bge_cdata.bge_rx_return_ring_map, BUS_DMASYNC_POSTREAD); - bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag, - sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_POSTWRITE); - if (ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN > - (MCLBYTES - ETHER_ALIGN)) - bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag, - sc->bge_cdata.bge_rx_jumbo_ring_map, BUS_DMASYNC_POSTWRITE); while (rx_cons != rx_prod) { struct bge_rx_bd *cur_rx; @@ -3315,31 +3400,31 @@ } if (cur_rx->bge_flags & BGE_RXBDFLAG_JUMBO_RING) { - jumbocnt++; m = sc->bge_cdata.bge_rx_jumbo_chain[rxidx]; if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) { - BGE_INC(sc->bge_jumbo, BGE_JUMBO_RX_RING_CNT); + bge_rxreuse_jumbo(sc, rxidx); continue; } if (bge_newbuf_jumbo(sc, rxidx) != 0) { - BGE_INC(sc->bge_jumbo, BGE_JUMBO_RX_RING_CNT); + bge_rxreuse_jumbo(sc, rxidx); ifp->if_iqdrops++; continue; } BGE_INC(sc->bge_jumbo, BGE_JUMBO_RX_RING_CNT); + bge_rxpost_jumbo(sc); } else { - stdcnt++; + m = sc->bge_cdata.bge_rx_std_chain[rxidx]; if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) { - BGE_INC(sc->bge_std, BGE_STD_RX_RING_CNT); + bge_rxreuse_std(sc, rxidx); continue; } - m = sc->bge_cdata.bge_rx_std_chain[rxidx]; if (bge_newbuf_std(sc, rxidx) != 0) { - BGE_INC(sc->bge_std, BGE_STD_RX_RING_CNT); + bge_rxreuse_std(sc, rxidx); ifp->if_iqdrops++; continue; } BGE_INC(sc->bge_std, BGE_STD_RX_RING_CNT); + bge_rxpost_std(sc); } ifp->if_ipackets++; @@ -3400,20 +3485,8 @@ bus_dmamap_sync(sc->bge_cdata.bge_rx_return_ring_tag, sc->bge_cdata.bge_rx_return_ring_map, BUS_DMASYNC_PREREAD); - if (stdcnt > 0) - bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag, - sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_PREWRITE); - - if (jumbocnt > 0) - bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag, - sc->bge_cdata.bge_rx_jumbo_ring_map, BUS_DMASYNC_PREWRITE); - sc->bge_rx_saved_considx = rx_cons; bge_writembx(sc, BGE_MBX_RX_CONS0_LO, sc->bge_rx_saved_considx); - if (stdcnt) - bge_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bge_std); - if (jumbocnt) - bge_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bge_jumbo); #ifdef notyet /* * This register wraps very quickly under heavy packet drops.