Index: conf/files =================================================================== --- conf/files (revision 256184) +++ conf/files (working copy) @@ -2829,6 +2829,7 @@ kern/kern_sema.c standard kern/kern_sharedpage.c standard kern/kern_shutdown.c standard kern/kern_sig.c standard +kern/kern_spinqueue.c standard kern/kern_switch.c standard kern/kern_sx.c standard kern/kern_synch.c standard Index: kern/kern_spinqueue.c =================================================================== --- kern/kern_spinqueue.c (revision 0) +++ kern/kern_spinqueue.c (working copy) @@ -0,0 +1,101 @@ +/*- + * Copyright (c) 2013 Davide Italiano + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include +#include + +/* + * TODO: inline fast path for acquisition. + */ + +void +spinqueue_init(struct spinqueue *spq) +{ + + /* Initialize spinqueue */ + spq = NULL; +} + +void +spinqueue_lock(struct spinqueue **spq, struct spinqueue *me) +{ + struct spinqueue *tail; + + if (*spq == me) { + printf("recursing"); + me->spinqueue_recurse++; + return; + } + me->next = NULL; + me->spin = 0; + tail = (struct spinqueue *)atomic_swap_ptr((uintptr_t *)spq, + (uintptr_t)me); + if (tail == NULL) + return; + tail->next = me; + mb(); + while (me->spin == 0) + cpu_spinwait(); + return; +} + +int +spinqueue_trylock(struct spinqueue **spq, struct spinqueue *me) +{ + + me->next = NULL; + me->spin = 0; + if(atomic_cmpset_acq_ptr((uintptr_t *)spq, + (uintptr_t)0, (uintptr_t)me)) + return (0); + else + return (EBUSY); +} + +void +spinqueue_unlock(struct spinqueue **spq, struct spinqueue *me) +{ + + if (me->spinqueue_recurse > 0) { + me->spinqueue_recurse--; + return; + } + if (me->next == NULL) { + if (atomic_cmpset_rel_ptr((uintptr_t *)spq, (uintptr_t)me, + (uintptr_t)0)) { + return; + } + while (me->next == NULL) + cpu_spinwait(); + } + me->next->spin = 1; +} Index: sys/spinqueue.h =================================================================== --- sys/spinqueue.h (revision 0) +++ sys/spinqueue.h (working copy) @@ -0,0 +1,42 @@ +/*- + * Copyright (c) 2013 Davide Italiano + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_SPINQUEUE_H_ +#define _SYS_SPINQUEUE_H_ + +struct spinqueue { + /* struct lock_object lock_object */ + struct spinqueue *next; + int spin; + int spinqueue_recurse; +}; + +void spinqueue_init(struct spinqueue *spq); +void spinqueue_lock(struct spinqueue **spq, struct spinqueue *self); +void spinqueue_unlock(struct spinqueue **spq, struct spinqueue *self); +int spinqueue_trylock(struct spinqueue **spq, struct spinqueue *self); + +#endif /* _SYS_SPINQUEUE_H_ */ Index: vm/swap_pager.c =================================================================== --- vm/swap_pager.c (revision 256184) +++ vm/swap_pager.c (working copy) @@ -90,6 +90,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -506,6 +507,7 @@ swap_pager_init(void) void swap_pager_swap_init(void) { + struct spinqueue me; int n, n2; /* @@ -532,12 +534,12 @@ swap_pager_swap_init(void) */ nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER); - mtx_lock(&pbuf_mtx); + spinqueue_lock(&pbuf_mtx, &me); nsw_rcount = (nswbuf + 1) / 2; nsw_wcount_sync = (nswbuf + 3) / 4; nsw_wcount_async = 4; nsw_wcount_async_max = nsw_wcount_async; - mtx_unlock(&pbuf_mtx); + spinqueue_unlock(&pbuf_mtx, &me); /* * Initialize our zone. Right now I'm just guessing on the number @@ -1276,6 +1278,7 @@ swap_pager_putpages(vm_object_t object, vm_page_t { int i; int n = 0; + struct spinqueue me; if (count && m[0]->object != object) { panic("swap_pager_putpages: object mismatch %p/%p", @@ -1304,7 +1307,7 @@ swap_pager_putpages(vm_object_t object, vm_page_t * Update nsw parameters from swap_async_max sysctl values. * Do not let the sysop crash the machine with bogus numbers. */ - mtx_lock(&pbuf_mtx); + spinqueue_lock(&pbuf_mtx, &me); if (swap_async_max != nsw_wcount_async_max) { int n; @@ -1329,7 +1332,7 @@ swap_pager_putpages(vm_object_t object, vm_page_t wakeup(&nsw_wcount_async); } } - mtx_unlock(&pbuf_mtx); + spinqueue_unlock(&pbuf_mtx, &me); /* * Step 3 Index: vm/vm_pager.c =================================================================== --- vm/vm_pager.c (revision 256184) +++ vm/vm_pager.c (working copy) @@ -74,6 +74,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -175,7 +176,7 @@ static const int npagers = sizeof(pagertab) / size * cleaning requests (NPENDINGIO == 64) * the maximum swap cluster size * (MAXPHYS == 64k) if you want to get the most efficiency. */ -struct mtx_padalign pbuf_mtx; +struct spinqueue *pbuf_mtx; static TAILQ_HEAD(swqueue, buf) bswlist; static int bswneeded; vm_offset_t swapbkva; /* swap buffers kva */ @@ -200,7 +201,7 @@ vm_pager_bufferinit() struct buf *bp; int i; - mtx_init(&pbuf_mtx, "pbuf mutex", NULL, MTX_DEF); + spinqueue_init(pbuf_mtx); bp = swbuf; /* * Now set up swap and physical I/O buffer headers. @@ -327,15 +328,10 @@ struct buf * getpbuf(int *pfreecnt) { struct buf *bp; + struct spinqueue me; - mtx_lock(&pbuf_mtx); - + spinqueue_lock(&pbuf_mtx, &me); for (;;) { - if (pfreecnt) { - while (*pfreecnt == 0) { - msleep(pfreecnt, &pbuf_mtx, PVM, "wswbuf0", 0); - } - } /* get a bp from the swap buffer header pool */ if ((bp = TAILQ_FIRST(&bswlist)) != NULL) @@ -342,13 +338,15 @@ getpbuf(int *pfreecnt) break; bswneeded = 1; - msleep(&bswneeded, &pbuf_mtx, PVM, "wswbuf1", 0); + /* + * XXX: This shouldn't be here. + */ + spinqueue_unlock(&pbuf_mtx, &me); + spinqueue_lock(&pbuf_mtx, &me); /* loop in case someone else grabbed one */ } TAILQ_REMOVE(&bswlist, bp, b_freelist); - if (pfreecnt) - --*pfreecnt; - mtx_unlock(&pbuf_mtx); + spinqueue_unlock(&pbuf_mtx, &me); initpbuf(bp); return bp; @@ -364,10 +362,11 @@ struct buf * trypbuf(int *pfreecnt) { struct buf *bp; + struct spinqueue me; - mtx_lock(&pbuf_mtx); + spinqueue_lock(&pbuf_mtx, &me); if (*pfreecnt == 0 || (bp = TAILQ_FIRST(&bswlist)) == NULL) { - mtx_unlock(&pbuf_mtx); + spinqueue_unlock(&pbuf_mtx, &me); return NULL; } TAILQ_REMOVE(&bswlist, bp, b_freelist); @@ -374,7 +373,7 @@ trypbuf(int *pfreecnt) --*pfreecnt; - mtx_unlock(&pbuf_mtx); + spinqueue_unlock(&pbuf_mtx, &me); initpbuf(bp); @@ -390,6 +389,7 @@ trypbuf(int *pfreecnt) void relpbuf(struct buf *bp, int *pfreecnt) { + struct spinqueue me; if (bp->b_rcred != NOCRED) { crfree(bp->b_rcred); @@ -405,7 +405,7 @@ relpbuf(struct buf *bp, int *pfreecnt) BUF_UNLOCK(bp); - mtx_lock(&pbuf_mtx); + spinqueue_lock(&pbuf_mtx, &me); TAILQ_INSERT_HEAD(&bswlist, bp, b_freelist); if (bswneeded) { @@ -416,7 +416,7 @@ relpbuf(struct buf *bp, int *pfreecnt) if (++*pfreecnt == 1) wakeup(pfreecnt); } - mtx_unlock(&pbuf_mtx); + spinqueue_unlock(&pbuf_mtx, &me); } /* Index: vm/vm_pager.h =================================================================== --- vm/vm_pager.h (revision 256184) +++ vm/vm_pager.h (working copy) @@ -96,7 +96,7 @@ extern struct pagerops mgtdevicepagerops; #ifdef _KERNEL extern struct pagerops *pagertab[]; -extern struct mtx_padalign pbuf_mtx; +extern struct spinqueue *pbuf_mtx; vm_object_t vm_pager_allocate(objtype_t, void *, vm_ooffset_t, vm_prot_t, vm_ooffset_t, struct ucred *);