diff --git a/sys/sys/bitset.h b/sys/sys/bitset.h index 47c4106d4127..6d60e40a3f7e 100644 --- a/sys/sys/bitset.h +++ b/sys/sys/bitset.h @@ -286,6 +286,9 @@ break; \ } \ __bits = op((p)->__bits[__i]); \ + if (__i == __bitset_words(_s) - 1 && \ + (_s) % _BITSET_BITS != 0) \ + __bits &= (1ul << (_s % _BITSET_BITS)) - 1; \ } \ __found != 0; \ }) diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index 5a9d9bb9fbf4..123d3095d869 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -4013,6 +4013,36 @@ slab_alloc_item(uma_keg_t keg, uma_slab_t slab) return (item); } +static int +slab_alloc_items(uma_keg_t keg, uma_slab_t slab, void **bucket, int max) +{ + uma_domain_t dom; + int count, freei; + + KASSERT(max >= slab->us_freecount, + ("%s: less than %d items in %p", __func__, max, slab)); + KEG_LOCK_ASSERT(keg, slab->us_domain); + dom = &keg->uk_domain[slab->us_domain]; + + count = 0; + BIT_FOREACH_ISSET(keg->uk_ipers, freei, &slab->us_free) { + bucket[count++] = slab_item(slab, keg, freei); + } + slab->us_freecount = 0; + dom->ud_free_items -= count; + BIT_ZERO(keg->uk_ipers, &slab->us_free); + + /* + * Move this slab to the full list. It must be on the partial list, so + * we do not need to update the free slab count. In particular, + * keg_fetch_slab() always returns slabs on the partial list. + */ + LIST_REMOVE(slab, us_link); + LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link); + + return (count); +} + static int zone_import(void *arg, void **bucket, int max, int domain, int flags) { @@ -4032,6 +4062,21 @@ zone_import(void *arg, void **bucket, int max, int domain, int flags) for (i = 0; i < max; ) { if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) break; + + /* + * As an optimization, try to pull all of the items out of the + * slab at once. + * + * XXXMJ this can and should go further: we can make a copy of + * the bitmap and slab base address, then drop the per-domain + * lock before the loop. + */ + if (max - i >= slab->us_freecount && keg->uk_reserve == 0 && + (zone->uz_flags & UMA_ZONE_ROUNDROBIN) == 0) { + i += slab_alloc_items(keg, slab, bucket + i, max - i); + goto next; + } + #ifdef NUMA stripe = howmany(max, vm_ndomains); #endif @@ -4062,6 +4107,7 @@ zone_import(void *arg, void **bucket, int max, int domain, int flags) break; #endif } while (slab->us_freecount != 0 && i < max); +next: KEG_UNLOCK(keg, slab->us_domain); /* Don't block if we allocated any successfully. */