--- fs/nfsserver/nfs_nfsdcache.c.sav 2012-10-10 18:56:01.000000000 -0400 +++ fs/nfsserver/nfs_nfsdcache.c 2012-10-12 21:04:21.000000000 -0400 @@ -160,7 +160,8 @@ __FBSDID("$FreeBSD: head/sys/fs/nfsserve #include extern struct nfsstats newnfsstats; -NFSCACHEMUTEX; +extern struct mtx nfsrc_tcpmtx[NFSRVCACHE_HASHSIZE]; +extern struct mtx nfsrc_udpmtx; int nfsrc_floodlevel = NFSRVCACHE_FLOODLEVEL, nfsrc_tcpsavedreplies = 0; #endif /* !APPLEKEXT */ @@ -208,10 +209,11 @@ static int newnfsv2_procid[NFS_V3NPROCS] NFSV2PROC_NOOP, }; +#define nfsrc_hash(xid) (((xid) + ((xid) >> 24)) % NFSRVCACHE_HASHSIZE) #define NFSRCUDPHASH(xid) \ - (&nfsrvudphashtbl[((xid) + ((xid) >> 24)) % NFSRVCACHE_HASHSIZE]) + (&nfsrvudphashtbl[nfsrc_hash(xid)]) #define NFSRCHASH(xid) \ - (&nfsrvhashtbl[((xid) + ((xid) >> 24)) % NFSRVCACHE_HASHSIZE]) + (&nfsrvhashtbl[nfsrc_hash(xid)]) #define TRUE 1 #define FALSE 0 #define NFSRVCACHE_CHECKLEN 100 @@ -262,6 +264,18 @@ static int nfsrc_getlenandcksum(mbuf_t m static void nfsrc_marksametcpconn(u_int64_t); /* + * Return the correct mutex for this cache entry. + */ +static __inline struct mtx * +nfsrc_cachemutex(struct nfsrvcache *rp) +{ + + if ((rp->rc_flag & RC_UDP) != 0) + return (&nfsrc_udpmtx); + return (&nfsrc_tcpmtx[nfsrc_hash(rp->rc_xid)]); +} + +/* * Initialize the server request cache list */ APPLESTATIC void @@ -336,10 +350,12 @@ nfsrc_getudp(struct nfsrv_descript *nd, struct sockaddr_in6 *saddr6; struct nfsrvhashhead *hp; int ret = 0; + struct mtx *mutex; + mutex = nfsrc_cachemutex(newrp); hp = NFSRCUDPHASH(newrp->rc_xid); loop: - NFSLOCKCACHE(); + mtx_lock(mutex); LIST_FOREACH(rp, hp, rc_hash) { if (newrp->rc_xid == rp->rc_xid && newrp->rc_proc == rp->rc_proc && @@ -347,8 +363,8 @@ loop: nfsaddr_match(NETFAMILY(rp), &rp->rc_haddr, nd->nd_nam)) { if ((rp->rc_flag & RC_LOCKED) != 0) { rp->rc_flag |= RC_WANTED; - (void)mtx_sleep(rp, NFSCACHEMUTEXPTR, - (PZERO - 1) | PDROP, "nfsrc", 10 * hz); + (void)mtx_sleep(rp, mutex, (PZERO - 1) | PDROP, + "nfsrc", 10 * hz); goto loop; } if (rp->rc_flag == 0) @@ -358,14 +374,14 @@ loop: TAILQ_INSERT_TAIL(&nfsrvudplru, rp, rc_lru); if (rp->rc_flag & RC_INPROG) { newnfsstats.srvcache_inproghits++; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); ret = RC_DROPIT; } else if (rp->rc_flag & RC_REPSTATUS) { /* * V2 only. */ newnfsstats.srvcache_nonidemdonehits++; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); nfsrvd_rephead(nd); *(nd->nd_errp) = rp->rc_status; ret = RC_REPLY; @@ -373,7 +389,7 @@ loop: NFSRVCACHE_UDPTIMEOUT; } else if (rp->rc_flag & RC_REPMBUF) { newnfsstats.srvcache_nonidemdonehits++; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); nd->nd_mreq = m_copym(rp->rc_reply, 0, M_COPYALL, M_WAIT); ret = RC_REPLY; @@ -403,7 +419,7 @@ loop: } LIST_INSERT_HEAD(hp, newrp, rc_hash); TAILQ_INSERT_TAIL(&nfsrvudplru, newrp, rc_lru); - NFSUNLOCKCACHE(); + mtx_unlock(mutex); nd->nd_rp = newrp; ret = RC_DOIT; @@ -421,12 +437,14 @@ nfsrvd_updatecache(struct nfsrv_descript struct nfsrvcache *rp; struct nfsrvcache *retrp = NULL; mbuf_t m; + struct mtx *mutex; rp = nd->nd_rp; if (!rp) panic("nfsrvd_updatecache null rp"); nd->nd_rp = NULL; - NFSLOCKCACHE(); + mutex = nfsrc_cachemutex(rp); + mtx_lock(mutex); nfsrc_lock(rp); if (!(rp->rc_flag & RC_INPROG)) panic("nfsrvd_updatecache not inprog"); @@ -441,7 +459,7 @@ nfsrvd_updatecache(struct nfsrv_descript */ if (nd->nd_repstat == NFSERR_REPLYFROMCACHE) { newnfsstats.srvcache_nonidemdonehits++; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); nd->nd_repstat = 0; if (nd->nd_mreq) mbuf_freem(nd->nd_mreq); @@ -474,21 +492,21 @@ nfsrvd_updatecache(struct nfsrv_descript nfsv2_repstat[newnfsv2_procid[nd->nd_procnum]]) { rp->rc_status = nd->nd_repstat; rp->rc_flag |= RC_REPSTATUS; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); } else { if (!(rp->rc_flag & RC_UDP)) { - nfsrc_tcpsavedreplies++; + atomic_add_int(&nfsrc_tcpsavedreplies, 1); if (nfsrc_tcpsavedreplies > newnfsstats.srvcache_tcppeak) newnfsstats.srvcache_tcppeak = nfsrc_tcpsavedreplies; } - NFSUNLOCKCACHE(); + mtx_unlock(mutex); m = m_copym(nd->nd_mreq, 0, M_COPYALL, M_WAIT); - NFSLOCKCACHE(); + mtx_lock(mutex); rp->rc_reply = m; rp->rc_flag |= RC_REPMBUF; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); } if (rp->rc_flag & RC_UDP) { rp->rc_timestamp = NFSD_MONOSEC + @@ -504,7 +522,7 @@ nfsrvd_updatecache(struct nfsrv_descript } } else { nfsrc_freecache(rp); - NFSUNLOCKCACHE(); + mtx_unlock(mutex); } out: @@ -520,14 +538,16 @@ out: APPLESTATIC void nfsrvd_delcache(struct nfsrvcache *rp) { + struct mtx *mutex; + mutex = nfsrc_cachemutex(rp); if (!(rp->rc_flag & RC_INPROG)) panic("nfsrvd_delcache not in prog"); - NFSLOCKCACHE(); + mtx_lock(mutex); rp->rc_flag &= ~RC_INPROG; if (rp->rc_refcnt == 0 && !(rp->rc_flag & RC_LOCKED)) nfsrc_freecache(rp); - NFSUNLOCKCACHE(); + mtx_unlock(mutex); } /* @@ -539,7 +559,9 @@ APPLESTATIC void nfsrvd_sentcache(struct nfsrvcache *rp, struct socket *so, int err) { tcp_seq tmp_seq; + struct mtx *mutex; + mutex = nfsrc_cachemutex(rp); if (!(rp->rc_flag & RC_LOCKED)) panic("nfsrvd_sentcache not locked"); if (!err) { @@ -548,10 +570,10 @@ nfsrvd_sentcache(struct nfsrvcache *rp, so->so_proto->pr_protocol != IPPROTO_TCP) panic("nfs sent cache"); if (nfsrv_getsockseqnum(so, &tmp_seq)) { - NFSLOCKCACHE(); + mtx_lock(mutex); rp->rc_tcpseq = tmp_seq; rp->rc_flag |= RC_TCPSEQ; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); } } nfsrc_unlock(rp); @@ -570,11 +592,13 @@ nfsrc_gettcp(struct nfsrv_descript *nd, struct nfsrvcache *hitrp; struct nfsrvhashhead *hp, nfsrc_templist; int hit, ret = 0; + struct mtx *mutex; + mutex = nfsrc_cachemutex(newrp); hp = NFSRCHASH(newrp->rc_xid); newrp->rc_reqlen = nfsrc_getlenandcksum(nd->nd_mrep, &newrp->rc_cksum); tryagain: - NFSLOCKCACHE(); + mtx_lock(mutex); hit = 1; LIST_INIT(&nfsrc_templist); /* @@ -632,8 +656,8 @@ tryagain: rp = hitrp; if ((rp->rc_flag & RC_LOCKED) != 0) { rp->rc_flag |= RC_WANTED; - (void)mtx_sleep(rp, NFSCACHEMUTEXPTR, - (PZERO - 1) | PDROP, "nfsrc", 10 * hz); + (void)mtx_sleep(rp, mutex, (PZERO - 1) | PDROP, + "nfsrc", 10 * hz); goto tryagain; } if (rp->rc_flag == 0) @@ -641,7 +665,7 @@ tryagain: rp->rc_flag |= RC_LOCKED; if (rp->rc_flag & RC_INPROG) { newnfsstats.srvcache_inproghits++; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); if (newrp->rc_sockref == rp->rc_sockref) nfsrc_marksametcpconn(rp->rc_sockref); ret = RC_DROPIT; @@ -650,7 +674,7 @@ tryagain: * V2 only. */ newnfsstats.srvcache_nonidemdonehits++; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); if (newrp->rc_sockref == rp->rc_sockref) nfsrc_marksametcpconn(rp->rc_sockref); ret = RC_REPLY; @@ -660,7 +684,7 @@ tryagain: NFSRVCACHE_TCPTIMEOUT; } else if (rp->rc_flag & RC_REPMBUF) { newnfsstats.srvcache_nonidemdonehits++; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); if (newrp->rc_sockref == rp->rc_sockref) nfsrc_marksametcpconn(rp->rc_sockref); ret = RC_REPLY; @@ -685,7 +709,7 @@ tryagain: newrp->rc_cachetime = NFSD_MONOSEC; newrp->rc_flag |= RC_INPROG; LIST_INSERT_HEAD(hp, newrp, rc_hash); - NFSUNLOCKCACHE(); + mtx_unlock(mutex); nd->nd_rp = newrp; ret = RC_DOIT; @@ -696,16 +720,17 @@ out: /* * Lock a cache entry. - * Also puts a mutex lock on the cache list. */ static void nfsrc_lock(struct nfsrvcache *rp) { - NFSCACHELOCKREQUIRED(); + struct mtx *mutex; + + mutex = nfsrc_cachemutex(rp); + mtx_assert(mutex, MA_OWNED); while ((rp->rc_flag & RC_LOCKED) != 0) { rp->rc_flag |= RC_WANTED; - (void)mtx_sleep(rp, NFSCACHEMUTEXPTR, PZERO - 1, - "nfsrc", 0); + (void)mtx_sleep(rp, mutex, PZERO - 1, "nfsrc", 0); } rp->rc_flag |= RC_LOCKED; } @@ -716,11 +741,13 @@ nfsrc_lock(struct nfsrvcache *rp) static void nfsrc_unlock(struct nfsrvcache *rp) { + struct mtx *mutex; - NFSLOCKCACHE(); + mutex = nfsrc_cachemutex(rp); + mtx_lock(mutex); rp->rc_flag &= ~RC_LOCKED; nfsrc_wanted(rp); - NFSUNLOCKCACHE(); + mtx_unlock(mutex); } /* @@ -743,7 +770,6 @@ static void nfsrc_freecache(struct nfsrvcache *rp) { - NFSCACHELOCKREQUIRED(); LIST_REMOVE(rp, rc_hash); if (rp->rc_flag & RC_UDP) { TAILQ_REMOVE(&nfsrvudplru, rp, rc_lru); @@ -753,7 +779,7 @@ nfsrc_freecache(struct nfsrvcache *rp) if (rp->rc_flag & RC_REPMBUF) { mbuf_freem(rp->rc_reply); if (!(rp->rc_flag & RC_UDP)) - nfsrc_tcpsavedreplies--; + atomic_add_int(&nfsrc_tcpsavedreplies, -1); } FREE((caddr_t)rp, M_NFSRVCACHE); newnfsstats.srvcache_size--; @@ -768,20 +794,22 @@ nfsrvd_cleancache(void) struct nfsrvcache *rp, *nextrp; int i; - NFSLOCKCACHE(); for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) { + mtx_lock(&nfsrc_tcpmtx[i]); LIST_FOREACH_SAFE(rp, &nfsrvhashtbl[i], rc_hash, nextrp) { nfsrc_freecache(rp); } + mtx_unlock(&nfsrc_tcpmtx[i]); } + mtx_lock(&nfsrc_udpmtx); for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) { LIST_FOREACH_SAFE(rp, &nfsrvudphashtbl[i], rc_hash, nextrp) { nfsrc_freecache(rp); } } newnfsstats.srvcache_size = 0; + mtx_unlock(&nfsrc_udpmtx); nfsrc_tcpsavedreplies = 0; - NFSUNLOCKCACHE(); } /* @@ -792,34 +820,42 @@ nfsrc_trimcache(u_int64_t sockref, struc { struct nfsrvcache *rp, *nextrp; int i; - static time_t lasttrim = 0; + static time_t udp_lasttrim = 0, tcp_lasttrim = 0; - if (NFSD_MONOSEC == lasttrim && - nfsrc_tcpsavedreplies < nfsrc_tcphighwater && - nfsrc_udpcachesize < (nfsrc_udphighwater + - nfsrc_udphighwater / 2)) - return; - NFSLOCKCACHE(); - lasttrim = NFSD_MONOSEC; - TAILQ_FOREACH_SAFE(rp, &nfsrvudplru, rc_lru, nextrp) { - if (!(rp->rc_flag & (RC_INPROG|RC_LOCKED|RC_WANTED)) - && rp->rc_refcnt == 0 - && ((rp->rc_flag & RC_REFCNT) || - NFSD_MONOSEC > rp->rc_timestamp || - nfsrc_udpcachesize > nfsrc_udphighwater)) - nfsrc_freecache(rp); - } - for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) { - LIST_FOREACH_SAFE(rp, &nfsrvhashtbl[i], rc_hash, nextrp) { + if (NFSD_MONOSEC != udp_lasttrim || + nfsrc_udpcachesize >= (nfsrc_udphighwater + + nfsrc_udphighwater / 2)) { + mtx_lock(&nfsrc_udpmtx); + udp_lasttrim = NFSD_MONOSEC; + TAILQ_FOREACH_SAFE(rp, &nfsrvudplru, rc_lru, nextrp) { if (!(rp->rc_flag & (RC_INPROG|RC_LOCKED|RC_WANTED)) && rp->rc_refcnt == 0 && ((rp->rc_flag & RC_REFCNT) || NFSD_MONOSEC > rp->rc_timestamp || - nfsrc_activesocket(rp, sockref, so))) + nfsrc_udpcachesize > nfsrc_udphighwater)) nfsrc_freecache(rp); } + mtx_unlock(&nfsrc_udpmtx); + } + if (NFSD_MONOSEC != tcp_lasttrim || + nfsrc_tcpsavedreplies >= nfsrc_tcphighwater) { + for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) { + mtx_lock(&nfsrc_tcpmtx[i]); + if (i == 0) + tcp_lasttrim = NFSD_MONOSEC; + LIST_FOREACH_SAFE(rp, &nfsrvhashtbl[i], rc_hash, + nextrp) { + if (!(rp->rc_flag & + (RC_INPROG|RC_LOCKED|RC_WANTED)) + && rp->rc_refcnt == 0 + && ((rp->rc_flag & RC_REFCNT) || + NFSD_MONOSEC > rp->rc_timestamp || + nfsrc_activesocket(rp, sockref, so))) + nfsrc_freecache(rp); + } + mtx_unlock(&nfsrc_tcpmtx[i]); + } } - NFSUNLOCKCACHE(); } /* @@ -828,12 +864,14 @@ nfsrc_trimcache(u_int64_t sockref, struc APPLESTATIC void nfsrvd_refcache(struct nfsrvcache *rp) { + struct mtx *mutex; - NFSLOCKCACHE(); + mutex = nfsrc_cachemutex(rp); + mtx_lock(mutex); if (rp->rc_refcnt < 0) panic("nfs cache refcnt"); rp->rc_refcnt++; - NFSUNLOCKCACHE(); + mtx_unlock(mutex); } /* @@ -842,14 +880,16 @@ nfsrvd_refcache(struct nfsrvcache *rp) APPLESTATIC void nfsrvd_derefcache(struct nfsrvcache *rp) { + struct mtx *mutex; - NFSLOCKCACHE(); + mutex = nfsrc_cachemutex(rp); + mtx_lock(mutex); if (rp->rc_refcnt <= 0) panic("nfs cache derefcnt"); rp->rc_refcnt--; if (rp->rc_refcnt == 0 && !(rp->rc_flag & (RC_LOCKED | RC_INPROG))) nfsrc_freecache(rp); - NFSUNLOCKCACHE(); + mtx_unlock(mutex); } /* --- fs/nfsserver/nfs_nfsdport.c.sav 2012-10-11 17:38:26.000000000 -0400 +++ fs/nfsserver/nfs_nfsdport.c 2012-10-11 17:43:16.000000000 -0400 @@ -60,7 +60,8 @@ extern SVCPOOL *nfsrvd_pool; extern struct nfsv4lock nfsd_suspend_lock; struct vfsoptlist nfsv4root_opt, nfsv4root_newopt; NFSDLOCKMUTEX; -struct mtx nfs_cache_mutex; +struct mtx nfsrc_tcpmtx[NFSRVCACHE_HASHSIZE]; +struct mtx nfsrc_udpmtx; struct mtx nfs_v4root_mutex; struct nfsrvfh nfs_rootfh, nfs_pubfh; int nfs_pubfhset = 0, nfs_rootfhset = 0; @@ -3288,7 +3289,7 @@ extern int (*nfsd_call_nfsd)(struct thre static int nfsd_modevent(module_t mod, int type, void *data) { - int error = 0; + int error = 0, i; static int loaded = 0; switch (type) { @@ -3296,7 +3297,10 @@ nfsd_modevent(module_t mod, int type, vo if (loaded) goto out; newnfs_portinit(); - mtx_init(&nfs_cache_mutex, "nfs_cache_mutex", NULL, MTX_DEF); + for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) + mtx_init(&nfsrc_tcpmtx[i], "nfs_tcpcache_mutex", NULL, + MTX_DEF); + mtx_init(&nfsrc_udpmtx, "nfs_udpcache_mutex", NULL, MTX_DEF); mtx_init(&nfs_v4root_mutex, "nfs_v4root_mutex", NULL, MTX_DEF); mtx_init(&nfsv4root_mnt.mnt_mtx, "struct mount mtx", NULL, MTX_DEF); @@ -3340,7 +3344,9 @@ nfsd_modevent(module_t mod, int type, vo svcpool_destroy(nfsrvd_pool); /* and get rid of the locks */ - mtx_destroy(&nfs_cache_mutex); + for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) + mtx_destroy(&nfsrc_tcpmtx[i]); + mtx_destroy(&nfsrc_udpmtx); mtx_destroy(&nfs_v4root_mutex); mtx_destroy(&nfsv4root_mnt.mnt_mtx); lockdestroy(&nfsv4root_mnt.mnt_explock); --- fs/nfs/nfsport.h.sav 2012-10-10 20:56:26.000000000 -0400 +++ fs/nfs/nfsport.h 2012-10-10 20:56:42.000000000 -0400 @@ -546,11 +546,6 @@ void nfsrvd_rcv(struct socket *, void *, #define NFSREQSPINLOCK extern struct mtx nfs_req_mutex #define NFSLOCKREQ() mtx_lock(&nfs_req_mutex) #define NFSUNLOCKREQ() mtx_unlock(&nfs_req_mutex) -#define NFSCACHEMUTEX extern struct mtx nfs_cache_mutex -#define NFSCACHEMUTEXPTR (&nfs_cache_mutex) -#define NFSLOCKCACHE() mtx_lock(&nfs_cache_mutex) -#define NFSUNLOCKCACHE() mtx_unlock(&nfs_cache_mutex) -#define NFSCACHELOCKREQUIRED() mtx_assert(&nfs_cache_mutex, MA_OWNED) #define NFSSOCKMUTEX extern struct mtx nfs_slock_mutex #define NFSSOCKMUTEXPTR (&nfs_slock_mutex) #define NFSLOCKSOCK() mtx_lock(&nfs_slock_mutex) --- fs/nfs/nfsrvcache.h.sav 2012-10-12 20:03:42.000000000 -0400 +++ fs/nfs/nfsrvcache.h 2012-10-12 20:03:55.000000000 -0400 @@ -41,7 +41,7 @@ #define NFSRVCACHE_MAX_SIZE 2048 #define NFSRVCACHE_MIN_SIZE 64 -#define NFSRVCACHE_HASHSIZE 20 +#define NFSRVCACHE_HASHSIZE 200 struct nfsrvcache { LIST_ENTRY(nfsrvcache) rc_hash; /* Hash chain */