diff -ruN /usr/src.org/sys/alpha/alpha/machdep.c /usr/src/sys/alpha/alpha/machdep.c --- /usr/src.org/sys/alpha/alpha/machdep.c Mon Sep 11 20:34:19 2000 +++ /usr/src/sys/alpha/alpha/machdep.c Mon Sep 11 21:59:04 2000 @@ -185,8 +185,6 @@ static void cpu_startup __P((void *)); SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL) -static MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf"); - struct msgbuf *msgbufp=0; int bootverbose = 0, Maxmem = 0; @@ -373,18 +371,12 @@ (16*(ARG_MAX+(PAGE_SIZE*3)))); /* - * Finally, allocate mbuf pool. + * Initialize mbuf system. + * Doing this early on (as opposed to through SYSINIT) is good as + * we want to make sure that the mutex locks are setup prior to + * network device drivers doing their stuff. */ - { - vm_offset_t mb_map_size; - - mb_map_size = nmbufs * MSIZE + nmbclusters * MCLBYTES + - (nmbclusters + nmbufs / 4) * sizeof(union mext_refcnt); - mb_map_size = roundup2(mb_map_size, max(MCLBYTES, PAGE_SIZE)); - mb_map = kmem_suballoc(kmem_map, (vm_offset_t *)&mbutl, - &maxaddr, mb_map_size); - mb_map->system_map = 1; - } + mbinit(); /* * Initialize callouts diff -ruN /usr/src.org/sys/i386/i386/machdep.c /usr/src/sys/i386/i386/machdep.c --- /usr/src.org/sys/i386/i386/machdep.c Mon Sep 11 20:35:23 2000 +++ /usr/src/sys/i386/i386/machdep.c Mon Sep 11 21:58:02 2000 @@ -139,8 +139,6 @@ static void cpu_startup __P((void *)); SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL) -static MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf"); - int _udatasel, _ucodesel; u_int atdevbase; @@ -399,18 +397,12 @@ (16*(ARG_MAX+(PAGE_SIZE*3)))); /* - * Finally, allocate mbuf pool. + * Initialize mbuf system. + * Doing this early on (as opposed to through SYSINIT) is good + * as we want to make sure that the mutex locks are setup prior to + * network device drivers doing their stuff. */ - { - vm_offset_t mb_map_size; - - mb_map_size = nmbufs * MSIZE + nmbclusters * MCLBYTES + - (nmbclusters + nmbufs / 4) * sizeof(union mext_refcnt); - mb_map_size = roundup2(mb_map_size, max(MCLBYTES, PAGE_SIZE)); - mb_map = kmem_suballoc(kmem_map, (vm_offset_t *)&mbutl, - &maxaddr, mb_map_size); - mb_map->system_map = 1; - } + mbinit(); /* * Initialize callouts diff -ruN /usr/src.org/sys/kern/uipc_mbuf.c /usr/src/sys/kern/uipc_mbuf.c --- /usr/src.org/sys/kern/uipc_mbuf.c Mon Sep 11 20:35:26 2000 +++ /usr/src/sys/kern/uipc_mbuf.c Mon Sep 11 21:50:07 2000 @@ -48,19 +48,19 @@ #include #include +#include +#include +#include +#include +#include + #ifdef INVARIANTS #include #endif -static void mbinit __P((void *)); -SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbinit, NULL) - struct mbuf *mbutl; struct mbstat mbstat; u_long mbtypes[MT_NTYPES]; -struct mbuf *mmbfree; -union mcluster *mclfree; -union mext_refcnt *mext_refcnt_free; int max_linkhdr; int max_protohdr; int max_hdr; @@ -70,6 +70,17 @@ u_int m_mballoc_wid = 0; u_int m_clalloc_wid = 0; +/* + * freelist header structures... + * mbffree_lst, mclfree_lst, mcntfree_lst + */ +struct mbffree_lst mbffree_lst_hdr, *mmbfree; +struct mclfree_lst mclfree_lst_hdr, *mclfree; +struct mcntfree_lst mcntfree_lst_hdr, *mcntfree; + +/* + * sysctl(8) exported objects + */ SYSCTL_DECL(_kern_ipc); SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW, &max_linkhdr, 0, ""); @@ -95,41 +106,90 @@ static void m_reclaim __P((void)); +/* Initial allocation numbers */ #define NCL_INIT 2 #define NMB_INIT 16 #define REF_INIT (NMBCLUSTERS * 2) -/* ARGSUSED*/ -static void -mbinit(dummy) - void *dummy; +/* + * Full mbuf subsystem initialization done here. + * + * XXX: If ever we have system specific map setups to do, then move them to + * machdep.c - for now, there is no reason for this stuff to go there. + * We just call this explicitly, as most of the stuff that needs to get + * done here should be done early on (i.e. from cpu_startup) anyway. + */ +void +mbinit(void) { - int s; + vm_offset_t maxaddr, mb_map_size; - mmbfree = NULL; - mclfree = NULL; - mext_refcnt_free = NULL; + /* + * Setup the mb_map, allocate requested VM space. + */ + mb_map_size = nmbufs * MSIZE + nmbclusters * MCLBYTES + (nmbclusters + + nmbufs / 4) * sizeof(union mext_refcnt); + mb_map_size = roundup2(mb_map_size, max(MCLBYTES, PAGE_SIZE)); + mb_map = kmem_suballoc(kmem_map, (vm_offset_t *)&mbutl, &maxaddr, + mb_map_size); + /* XXX: mb_map->system_map = 1; */ + /* + * Initialize the free list headers, and setup locks for lists. + */ + mmbfree = (struct mbffree_lst *)&mbffree_lst_hdr; + mclfree = (struct mclfree_lst *)&mclfree_lst_hdr; + mcntfree = (struct mcntfree_lst *)&mcntfree_lst_hdr; + mmbfree->m_head = NULL; + mclfree->m_head = NULL; + mcntfree->m_head = NULL; + mtx_init(&mmbfree->m_mtx, "mbuf free list lock", MTX_DEF); + mtx_init(&mclfree->m_mtx, "mcluster free list lock", MTX_DEF); + mtx_init(&mcntfree->m_mtx, "m_ext counter free list lock", MTX_DEF); + + /* + * Initialize mbuf subsystem (sysctl exported) statistics structure. + */ mbstat.m_msize = MSIZE; mbstat.m_mclbytes = MCLBYTES; mbstat.m_minclsize = MINCLSIZE; mbstat.m_mlen = MLEN; mbstat.m_mhlen = MHLEN; - s = splimp(); + /* + * Perform some initial allocations. + * If allocations are succesful, locks are obtained in the allocation + * routines, so we must release them if it works. + * + * XXX: We try to allocate as many reference counters as we'll + * most need throughout the system's lifespan. + * XXXXXX: Make sure we check whether the MCLBYTES > PAGE_SIZE + * is still useful before bringing this in. + */ if (m_alloc_ref(REF_INIT) == 0) goto bad; + else + mtx_exit(&mcntfree->m_mtx, MTX_DEF); + if (m_mballoc(NMB_INIT, M_DONTWAIT) == 0) goto bad; + else + mtx_exit(&mmbfree->m_mtx, MTX_DEF); + #if MCLBYTES <= PAGE_SIZE if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0) goto bad; + else + mtx_exit(&mclfree->m_mtx, MTX_DEF); #else + /* XXXXXX */ /* It's OK to call contigmalloc in this context. */ if (m_clalloc(16, M_WAIT) == 0) goto bad; + else + mtx_exit(&mclfree->m_mtx, MTX_DEF); #endif - splx(s); + return; bad: panic("mbinit: failed to initialize mbuf subsystem!"); @@ -138,7 +198,6 @@ /* * Allocate at least nmb reference count structs and place them * on the ref cnt free list. - * Must be called at splimp. */ int m_alloc_ref(nmb) @@ -157,18 +216,28 @@ * and if we are, we're probably not out of the woods anyway, * so leave this way for now. */ - if (mb_map_full) return (0); nbytes = round_page(nmb * sizeof(union mext_refcnt)); - if ((p = (caddr_t)kmem_malloc(mb_map, nbytes, M_NOWAIT)) == NULL) + mtx_enter(&Giant, MTX_DEF); + if ((p = (caddr_t)kmem_malloc(mb_map, nbytes, M_NOWAIT)) == NULL) { + mtx_exit(&Giant, MTX_DEF); return (0); + } + mtx_exit(&Giant, MTX_DEF); nmb = nbytes / sizeof(union mext_refcnt); + /* + * We don't let go of the mutex in order to avoid a race. + * It is up to the caller to let go of the mutex if the call + * was successful or just do nothing if it failed, because in that + * case, we wouldn't have grabbed the mutex at all. + */ + mtx_enter(&mcntfree->m_mtx, MTX_DEF); for (i = 0; i < nmb; i++) { - ((union mext_refcnt *)p)->next_ref = mext_refcnt_free; - mext_refcnt_free = (union mext_refcnt *)p; + ((union mext_refcnt *)p)->next_ref = mcntfree->m_head; + mcntfree->m_head = (union mext_refcnt *)p; p += sizeof(union mext_refcnt); mbstat.m_refree++; } @@ -179,9 +248,7 @@ /* * Allocate at least nmb mbufs and place on mbuf free list. - * Must be called at splimp. */ -/* ARGSUSED */ int m_mballoc(nmb, how) register int nmb; @@ -192,32 +259,23 @@ int nbytes; /* - * If we've hit the mbuf limit, stop allocating from mb_map, - * (or trying to) in order to avoid dipping into the section of - * mb_map which we've "reserved" for clusters. - */ - if ((nmb + mbstat.m_mbufs) > nmbufs) - return (0); - - /* - * Once we run out of map space, it will be impossible to get - * any more (nothing is ever freed back to the map) - * -- however you are not dead as m_reclaim might - * still be able to free a substantial amount of space. - * - * XXX Furthermore, we can also work with "recycled" mbufs (when - * we're calling with M_WAIT the sleep procedure will be woken - * up when an mbuf is freed. See m_mballoc_wait()). + * If we've hit the mbuf limit, stop allocating from mb_map. + * Also, once we run out of map space, it will be impossible to + * get any more (nothing is ever freed back to the map). */ - if (mb_map_full) + if (mb_map_full || ((nmb + mbstat.m_mbufs) > nmbufs)) { + mbstat.m_drops++; return (0); + } nbytes = round_page(nmb * MSIZE); + mtx_enter(&Giant, MTX_DEF); p = (caddr_t)kmem_malloc(mb_map, nbytes, M_NOWAIT); if (p == 0 && how == M_WAIT) { mbstat.m_wait++; p = (caddr_t)kmem_malloc(mb_map, nbytes, M_WAITOK); } + mtx_exit(&Giant, MTX_DEF); /* * Either the map is now full, or `how' is M_NOWAIT and there @@ -227,9 +285,17 @@ return (0); nmb = nbytes / MSIZE; + + /* + * We don't let go of the mutex in order to avoid a race. + * It is up to the caller to let go of the mutex if the call + * was successful or just do nothing if it failed, because in that + * case, we wouldn't have grabbed the mutex at all. + */ + mtx_enter(&mmbfree->m_mtx, MTX_DEF); for (i = 0; i < nmb; i++) { - ((struct mbuf *)p)->m_next = mmbfree; - mmbfree = (struct mbuf *)p; + ((struct mbuf *)p)->m_next = mmbfree->m_head; + mmbfree->m_head = (struct mbuf *)p; p += MSIZE; } mbstat.m_mbufs += nmb; @@ -242,25 +308,36 @@ * (or, in some cases, functions) is with M_WAIT, then it is necessary to rely * solely on reclaimed mbufs. Here we wait for an mbuf to be freed for a * designated (mbuf_wait) time. + * + * Must be called with no held mutexes... may block. */ struct mbuf * m_mballoc_wait(int caller, int type) { struct mbuf *p; - int s; - s = splimp(); + /* + * This avoids a potential race. What we do is first place ourselves + * in the queue (with asleep) and then later do an actual await() so + * that if we happen to get a wakeup() in between, that the await() + * does effectively nothing. Otherwise, what could happen is that + * we increment m_mballoc_wid, at which point it will be decremented + * by a (racing) MMBWAKEUP(), yet we will sleep on it nonetheless and + * risk never being woken up (i.e. sleep on a m_mballoc_wid of 0)! + */ + asleep(&m_mballoc_wid, PVM, "mballc", mbuf_wait); m_mballoc_wid++; - if ((tsleep(&m_mballoc_wid, PVM, "mballc", mbuf_wait)) == EWOULDBLOCK) + if (await(PVM, mbuf_wait) == EWOULDBLOCK) m_mballoc_wid--; - splx(s); /* * Now that we (think) that we've got something, we will redo an * MGET, but avoid getting into another instance of m_mballoc_wait() + * * XXX: We retry to fetch _even_ if the sleep timed out. This is left * this way, purposely, in the [unlikely] case that an mbuf was - * freed but the sleep was not awakened in time. + * freed but the sleep was not awakened in time; then we are + * willing to race for it. */ p = NULL; switch (caller) { @@ -274,17 +351,25 @@ panic("m_mballoc_wait: invalid caller (%d)", caller); } - s = splimp(); - if (p != NULL) { /* We waited and got something... */ + /* If we waited and got something... */ + if (p != NULL) { + /* + * We don't need to grab a mutex here, since even if + * mmbfree->m_head changes to NULL (and it wasn't NULL right + * after our allocation) right here, then so let it be, + * we just won't wakeup() anybody. + * + * XXX: In other words, we willing race here for now. + */ mbstat.m_wait++; - /* Wake up another if we have more free. */ - if (mmbfree != NULL) + if (mmbfree->m_head != NULL) MMBWAKEUP(); } - splx(s); + return (p); } +/* XXXXXX */ #if MCLBYTES > PAGE_SIZE static int i_want_my_mcl; @@ -312,13 +397,12 @@ SYSINIT(mclallocproc, SI_SUB_KTHREAD_UPDATE, SI_ORDER_ANY, kproc_start, &mclalloc_kp); #endif +/* XXXXXX */ /* * Allocate some number of mbuf clusters * and place on cluster free list. - * Must be called at splimp. */ -/* ARGSUSED */ int m_clalloc(ncl, how) register int ncl; @@ -329,26 +413,16 @@ int npg; /* + * If the map is now full (nothing will ever be freed to it). * If we've hit the mcluster number limit, stop allocating from - * mb_map, (or trying to) in order to avoid dipping into the section - * of mb_map which we've "reserved" for mbufs. - */ - if ((ncl + mbstat.m_clusters) > nmbclusters) { - mbstat.m_drops++; - return (0); - } - - /* - * Once we run out of map space, it will be impossible - * to get any more (nothing is ever freed back to the - * map). From this point on, we solely rely on freed - * mclusters. + * mb_map. */ - if (mb_map_full) { + if (mb_map_full || ((ncl + mbstat.m_clusters) > nmbclusters)) { mbstat.m_drops++; return (0); } +/* XXXXXX */ #if MCLBYTES > PAGE_SIZE if (how != M_WAIT) { i_want_my_mcl += ncl; @@ -359,10 +433,12 @@ p = contigmalloc1(MCLBYTES * ncl, M_DEVBUF, M_WAITOK, 0ul, ~0ul, PAGE_SIZE, 0, mb_map); } -#else +#else /* XXXXXX */ npg = ncl; + mtx_enter(&Giant, MTX_DEF); p = (caddr_t)kmem_malloc(mb_map, ctob(npg), how != M_WAIT ? M_NOWAIT : M_WAITOK); + mtx_exit(&Giant, MTX_DEF); ncl = ncl * PAGE_SIZE / MCLBYTES; #endif /* @@ -374,9 +450,16 @@ return (0); } + /* + * We don't let go of the mutex in order to avoid a race. + * It is up to the caller to let go of the mutex if the call + * was successful or just do nothing if it failed, because in that + * case, we wouldn't have grabbed the mutex at all. + */ + mtx_enter(&mclfree->m_mtx, MTX_DEF); for (i = 0; i < ncl; i++) { - ((union mcluster *)p)->mcl_next = mclfree; - mclfree = (union mcluster *)p; + ((union mcluster *)p)->mcl_next = mclfree->m_head; + mclfree->m_head = (union mcluster *)p; p += MCLBYTES; mbstat.m_clfree++; } @@ -389,21 +472,31 @@ * M_WAIT, we rely on the mclfree union pointers. If nothing is free, we will * sleep for a designated amount of time (mbuf_wait) or until we're woken up * due to sudden mcluster availability. + * + * Must be called with no held mutexes... may block. */ caddr_t m_clalloc_wait(void) { caddr_t p; - int s; #ifdef __i386__ /* If in interrupt context, and INVARIANTS, maintain sanity and die. */ KASSERT(intr_nesting_level == 0, ("CLALLOC: CANNOT WAIT IN INTERRUPT")); #endif - /* Sleep until something's available or until we expire. */ + /* + * This avoids a potential race. What we do is first place ourselves + * in the queue (with asleep) and then later do an actual await() so + * that if we happen to get a wakeup() in between, that the await() + * does effectively nothing. Otherwise, what could happen is that + * we increment m_clalloc_wid, at which point it will be decremented + * by a (racing) MCLWAKEUP(), yet we will sleep on it nonetheless and + * risk never being woken up (i.e. sleep on a m_clalloc_wid of 0)! + */ + asleep(&m_clalloc_wid, PVM, "mclalc", mbuf_wait); m_clalloc_wid++; - if ((tsleep(&m_clalloc_wid, PVM, "mclalc", mbuf_wait)) == EWOULDBLOCK) + if (await(PVM, mbuf_wait) == EWOULDBLOCK) m_clalloc_wid--; /* @@ -413,15 +506,21 @@ p = NULL; _MCLALLOC(p, M_DONTWAIT); - s = splimp(); - if (p != NULL) { /* We waited and got something... */ + /* If we waited and got something ... */ + if (p != NULL) { + /* + * We don't need to grab a mutex here, since even if + * mclfree->m_head changes to NULL (and it wasn't NULL right + * after our allocation) right here, then so let it be, + * we just won't wakeup() anybody. + * + * XXX: In other words, we willing race here for now. + */ mbstat.m_wait++; - /* Wake up another if we have more free. */ - if (mclfree != NULL) + if (mclfree->m_head != NULL) MCLWAKEUP(); } - splx(s); return (p); } @@ -504,13 +603,15 @@ { register struct domain *dp; register struct protosw *pr; - int s = splimp(); + /* + * XXX: MAKE SURE DRAINERS DONT DIRECTLY TOUCH LISTS UNLESS THEY GRAB + * APPROPRIATE MUTEX!!! + */ for (dp = domains; dp; dp = dp->dom_next) for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) if (pr->pr_drain) (*pr->pr_drain)(); - splx(s); mbstat.m_drain++; } diff -ruN /usr/src.org/sys/pc98/i386/machdep.c /usr/src/sys/pc98/i386/machdep.c --- /usr/src.org/sys/pc98/i386/machdep.c Mon Sep 11 20:35:46 2000 +++ /usr/src/sys/pc98/i386/machdep.c Mon Sep 11 21:57:40 2000 @@ -145,8 +145,6 @@ static void cpu_startup __P((void *)); SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL) -static MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf"); - #ifdef PC98 int need_pre_dma_flush; /* If 1, use wbinvd befor DMA transfer. */ int need_post_dma_flush; /* If 1, use invd after DMA transfer. */ @@ -413,18 +411,12 @@ (16*(ARG_MAX+(PAGE_SIZE*3)))); /* - * Finally, allocate mbuf pool. + * Initialize mbuf system. + * Doing this early on (as opposed to through SYSINIT) is good + * as we want to make sure that the mutex locks are setup prior to + * network device drivers doing their stuff. */ - { - vm_offset_t mb_map_size; - - mb_map_size = nmbufs * MSIZE + nmbclusters * MCLBYTES + - (nmbclusters + nmbufs / 4) * sizeof(union mext_refcnt); - mb_map_size = roundup2(mb_map_size, max(MCLBYTES, PAGE_SIZE)); - mb_map = kmem_suballoc(kmem_map, (vm_offset_t *)&mbutl, - &maxaddr, mb_map_size); - mb_map->system_map = 1; - } + mbinit(); /* * Initialize callouts diff -ruN /usr/src.org/sys/sys/mbuf.h /usr/src/sys/sys/mbuf.h --- /usr/src.org/sys/sys/mbuf.h Mon Sep 11 20:35:50 2000 +++ /usr/src/sys/sys/mbuf.h Mon Sep 11 21:05:29 2000 @@ -37,6 +37,11 @@ #ifndef _SYS_MBUF_H_ #define _SYS_MBUF_H_ +#ifdef _KERNEL +#include +#include +#endif + /* * Mbufs are of a single size, MSIZE (machine/param.h), which * includes overhead. An mbuf may add a single "mbuf cluster" of size @@ -203,8 +208,7 @@ #define M_DONTWAIT 1 #define M_WAIT 0 -/* Freelists: - * +/* * Normal mbuf clusters are normally treated as character arrays * after allocation, but use the first word of the buffer as a free list * pointer while on the free list. @@ -232,8 +236,26 @@ }; /* + * free list header definitions: mbffree_lst, mclfree_lst, mcntfree_lst + */ +struct mbffree_lst { + struct mbuf *m_head; + mtx_t m_mtx; +}; + +struct mclfree_lst { + union mcluster *m_head; + mtx_t m_mtx; +}; + +struct mcntfree_lst { + union mext_refcnt *m_head; + mtx_t m_mtx; +}; + +/* * Wake up the next instance (if any) of m_mballoc_wait() which is - * waiting for an mbuf to be freed. This should be called at splimp(). + * waiting for an mbuf to be freed. * * XXX: If there is another free mbuf, this routine will be called [again] * from the m_mballoc_wait routine in order to wake another sleep instance. @@ -256,20 +278,6 @@ } while (0) /* - * mbuf utility macros: - * - * MBUFLOCK(code) - * prevents a section of code from from being interrupted by network - * drivers. - */ -#define MBUFLOCK(code) do { \ - int _ms = splimp(); \ - \ - { code } \ - splx(_ms); \ -} while (0) - -/* * mbuf external reference count management macros: * * MEXT_IS_REF(m): true if (m) is not the only mbuf referencing @@ -286,24 +294,31 @@ #define MEXT_ADD_REF(m) atomic_add_long(&((m)->m_ext.ref_cnt->refcnt), 1) -#define _MEXT_ALLOC_CNT(m_cnt) MBUFLOCK( \ +#define _MEXT_ALLOC_CNT(m_cnt) do { \ union mext_refcnt *__mcnt; \ \ - if ((mext_refcnt_free == NULL) && (m_alloc_ref(1) == 0)) \ - panic("mbuf subsystem: out of ref counts!"); \ - __mcnt = mext_refcnt_free; \ - mext_refcnt_free = __mcnt->next_ref; \ - __mcnt->refcnt = 0; \ - (m_cnt) = __mcnt; \ + mtx_enter(&mcntfree->m_mtx, MTX_DEF); \ + if (mcntfree->m_head == NULL) { \ + mtx_exit(&mcntfree->m_mtx, MTX_DEF); \ + if (m_alloc_ref(1) == 0) \ + panic("mbuf subsystem: out of ref counts!"); \ + } \ + __mcnt = mcntfree->m_head; \ + mcntfree->m_head = __mcnt->next_ref; \ mbstat.m_refree--; \ -) + mtx_exit(&mcntfree->m_mtx, MTX_DEF); \ + __mcnt->refcnt = 0; \ + (m_cnt) = __mcnt; \ +} while (0) #define _MEXT_DEALLOC_CNT(m_cnt) do { \ union mext_refcnt *__mcnt = (m_cnt); \ \ - __mcnt->next_ref = mext_refcnt_free; \ - mext_refcnt_free = __mcnt; \ + mtx_enter(&mcntfree->m_mtx, MTX_DEF); \ + __mcnt->next_ref = mcntfree->m_head; \ + mcntfree->m_head = __mcnt; \ mbstat.m_refree++; \ + mtx_exit(&mcntfree->m_mtx, MTX_DEF); \ } while (0) #define MEXT_INIT_REF(m) do { \ @@ -327,23 +342,26 @@ struct mbuf *_mm; \ int _mhow = (how); \ int _mtype = (type); \ - int _ms = splimp(); \ \ - if (mmbfree == NULL) \ - (void)m_mballoc(1, _mhow); \ - _mm = mmbfree; \ + mtx_enter(&mmbfree->m_mtx, MTX_DEF); \ + if (mmbfree->m_head == NULL) { \ + mtx_exit(&mmbfree->m_mtx, MTX_DEF); \ + if (m_mballoc(1, _mhow) == 0) \ + mtx_enter(&mmbfree->m_mtx, MTX_DEF); \ + } \ + _mm = mmbfree->m_head; \ if (_mm != NULL) { \ - mmbfree = _mm->m_next; \ + mmbfree->m_head = _mm->m_next; \ mbtypes[MT_FREE]--; \ mbtypes[_mtype]++; \ - splx(_ms); \ + mtx_exit(&mmbfree->m_mtx, MTX_DEF); \ _mm->m_type = _mtype; \ _mm->m_next = NULL; \ _mm->m_nextpkt = NULL; \ _mm->m_data = _mm->m_dat; \ _mm->m_flags = 0; \ } else { \ - splx(_ms); \ + mtx_exit(&mmbfree->m_mtx, MTX_DEF); \ _mm = m_retry(_mhow, _mtype); \ if (_mm == NULL && _mhow == M_WAIT) \ _mm = m_mballoc_wait(MGET_C, _mtype); \ @@ -355,16 +373,19 @@ struct mbuf *_mm; \ int _mhow = (how); \ int _mtype = (type); \ - int _ms = splimp(); \ \ - if (mmbfree == NULL) \ - (void)m_mballoc(1, _mhow); \ - _mm = mmbfree; \ + mtx_enter(&mmbfree->m_mtx, MTX_DEF); \ + if (mmbfree->m_head == NULL) { \ + mtx_exit(&mmbfree->m_mtx, MTX_DEF); \ + if (m_mballoc(1, _mhow) == 0) \ + mtx_enter(&mmbfree->m_mtx, MTX_DEF); \ + } \ + _mm = mmbfree->m_head; \ if (_mm != NULL) { \ - mmbfree = _mm->m_next; \ + mmbfree->m_head = _mm->m_next; \ mbtypes[MT_FREE]--; \ mbtypes[_mtype]++; \ - splx(_ms); \ + mtx_exit(&mmbfree->m_mtx, MTX_DEF); \ _mm->m_type = _mtype; \ _mm->m_next = NULL; \ _mm->m_nextpkt = NULL; \ @@ -374,7 +395,7 @@ _mm->m_pkthdr.csum_flags = 0; \ _mm->m_pkthdr.aux = NULL; \ } else { \ - splx(_ms); \ + mtx_exit(&mmbfree->m_mtx, MTX_DEF); \ _mm = m_retryhdr(_mhow, _mtype); \ if (_mm == NULL && _mhow == M_WAIT) \ _mm = m_mballoc_wait(MGETHDR_C, _mtype); \ @@ -393,17 +414,20 @@ #define _MCLALLOC(p, how) do { \ caddr_t _mp; \ int _mhow = (how); \ - int _ms = splimp(); \ \ - if (mclfree == NULL) \ - (void)m_clalloc(1, _mhow); \ - _mp = (caddr_t)mclfree; \ + mtx_enter(&mclfree->m_mtx, MTX_DEF); \ + if (mclfree->m_head == NULL) { \ + mtx_exit(&mclfree->m_mtx, MTX_DEF); \ + if (m_clalloc(1, _mhow) == 0) \ + mtx_enter(&mclfree->m_mtx, MTX_DEF); \ + } \ + _mp = (caddr_t)mclfree->m_head; \ if (_mp != NULL) { \ mbstat.m_clfree--; \ - mclfree = ((union mcluster *)_mp)->mcl_next; \ - splx(_ms); \ + mclfree->m_head = ((union mcluster *)_mp)->mcl_next; \ + mtx_exit(&mclfree->m_mtx, MTX_DEF); \ } else { \ - splx(_ms); \ + mtx_exit(&mclfree->m_mtx, MTX_DEF); \ if (_mhow == M_WAIT) \ _mp = m_clalloc_wait(); \ } \ @@ -436,16 +460,18 @@ MEXT_INIT_REF(_mm); \ } while (0) -#define _MCLFREE(p) MBUFLOCK( \ +#define _MCLFREE(p) do { \ union mcluster *_mp = (union mcluster *)(p); \ \ - _mp->mcl_next = mclfree; \ - mclfree = _mp; \ + mtx_enter(&mclfree->m_mtx, MTX_DEF); \ + _mp->mcl_next = mclfree->m_head; \ + mclfree->m_head = _mp; \ mbstat.m_clfree++; \ + mtx_exit(&mclfree->m_mtx, MTX_DEF); \ MCLWAKEUP(); \ -) +} while (0) -#define _MEXTFREE(m) do { \ +#define MEXTFREE(m) do { \ struct mbuf *_mmm = (m); \ \ if (MEXT_IS_REF(_mmm)) \ @@ -461,29 +487,27 @@ _mmm->m_flags &= ~M_EXT; \ } while (0) -#define MEXTFREE(m) MBUFLOCK( \ - _MEXTFREE(m); \ -) - /* * MFREE(struct mbuf *m, struct mbuf *n) * Free a single mbuf and associated external storage. * Place the successor, if any, in n. */ -#define MFREE(m, n) MBUFLOCK( \ +#define MFREE(m, n) do { \ struct mbuf *_mm = (m); \ \ KASSERT(_mm->m_type != MT_FREE, ("freeing free mbuf")); \ if (_mm->m_flags & M_EXT) \ - _MEXTFREE(_mm); \ + MEXTFREE(_mm); \ + mtx_enter(&mmbfree->m_mtx, MTX_DEF); \ mbtypes[_mm->m_type]--; \ _mm->m_type = MT_FREE; \ mbtypes[MT_FREE]++; \ (n) = _mm->m_next; \ - _mm->m_next = mmbfree; \ - mmbfree = _mm; \ + _mm->m_next = mmbfree->m_head; \ + mmbfree->m_head = _mm; \ + mtx_exit(&mmbfree->m_mtx, MTX_DEF); \ MMBWAKEUP(); \ -) +} while (0) /* * Copy mbuf pkthdr from "from" to "to". @@ -557,15 +581,20 @@ *_mmp = _mm; \ } while (0) -/* change mbuf to new type */ +/* change mbuf to new type + * + * XXX: + * mbtypes is protected by the same mutex that protects the mmbfree list, + * since only mbufs are involved anyway... + */ #define MCHTYPE(m, t) do { \ struct mbuf *_mm = (m); \ int _mt = (t); \ - int _ms = splimp(); \ \ + mtx_enter(&mmbfree->m_mtx, MTX_DEF); \ mbtypes[_mm->m_type]--; \ mbtypes[_mt]++; \ - splx(_ms); \ + mtx_exit(&mmbfree->m_mtx, MTX_DEF); \ _mm->m_type = (_mt); \ } while (0) @@ -594,15 +623,16 @@ extern u_long mbtypes[MT_NTYPES]; /* per-type mbuf allocations */ extern int mbuf_wait; /* mbuf sleep time */ extern struct mbuf *mbutl; /* virtual address of mclusters */ -extern union mcluster *mclfree; -extern struct mbuf *mmbfree; -extern union mext_refcnt *mext_refcnt_free; +extern struct mclfree_lst mclfree_lst_hdr, *mclfree; +extern struct mbffree_lst mbffree_lst_hdr, *mmbfree; +extern struct mcntfree_lst mcntfree_lst_hdr, *mcntfree; extern int nmbclusters; extern int nmbufs; extern int nsfbufs; void m_adj __P((struct mbuf *, int)); int m_alloc_ref __P((u_int)); +void mbinit __P((void)); void m_cat __P((struct mbuf *,struct mbuf *)); int m_clalloc __P((int, int)); caddr_t m_clalloc_wait __P((void));