Index: uma_core.c =================================================================== --- uma_core.c (revision 260328) +++ uma_core.c (working copy) @@ -237,10 +237,13 @@ static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zo static void cache_drain(uma_zone_t); static void bucket_drain(uma_zone_t, uma_bucket_t); static void bucket_cache_drain(uma_zone_t zone); +static void bucket_cache_drain_unused(uma_zone_t zone); static int keg_ctor(void *, int, void *, int); static void keg_dtor(void *, int, void *); +static void keg_drain(uma_keg_t keg); static int zone_ctor(void *, int, void *, int); static void zone_dtor(void *, int, void *); +static void zone_drain_wait(uma_zone_t zone, int waitok, int unused); static int zero_init(void *, int, int); static void keg_small_init(uma_keg_t keg); static void keg_large_init(uma_keg_t keg); @@ -511,6 +514,7 @@ static void zone_timeout(uma_zone_t zone) { + zone_drain_wait(zone, M_NOWAIT, 1); zone_foreach_keg(zone, &keg_timeout); } @@ -775,6 +779,36 @@ cache_drain_safe(uma_zone_t zone) } /* + * Drain the unused cached buckets from a zone. Expects a locked zone on entry. + */ +static void +bucket_cache_drain_unused(uma_zone_t zone) +{ + uma_bucket_t bucket; + + /* + * Drain the bucket queues and free the buckets, we just keep two per + * cpu (alloc/free). + */ + while ((bucket = zone->uz_unused) != NULL) { + zone->uz_unused = LIST_NEXT(bucket, ub_link); + LIST_REMOVE(bucket, ub_link); + ZONE_UNLOCK(zone); + bucket_drain(zone, bucket); + bucket_free(zone, bucket, NULL); + ZONE_LOCK(zone); + } + zone->uz_unused = LIST_FIRST(&zone->uz_buckets); + + /* + * Shrink further bucket sizes. Price of single zone lock collision + * is probably lower then price of global cache drain. + */ + if (zone->uz_count > zone->uz_count_min) + zone->uz_count--; +} + +/* * Drain the cached buckets from a zone. Expects a locked zone on entry. */ static void @@ -787,6 +821,7 @@ bucket_cache_drain(uma_zone_t zone) * cpu (alloc/free). */ while ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) { + zone->uz_unused = NULL; LIST_REMOVE(bucket, ub_link); ZONE_UNLOCK(zone); bucket_drain(zone, bucket); @@ -884,7 +919,7 @@ finished: } static void -zone_drain_wait(uma_zone_t zone, int waitok) +zone_drain_wait(uma_zone_t zone, int waitok, int unused) { /* @@ -902,7 +937,10 @@ static void mtx_lock(&uma_mtx); } zone->uz_flags |= UMA_ZFLAG_DRAINING; - bucket_cache_drain(zone); + if (unused) + bucket_cache_drain_unused(zone); + else + bucket_cache_drain(zone); ZONE_UNLOCK(zone); /* * The DRAINING flag protects us from being freed while @@ -921,7 +959,7 @@ void zone_drain(uma_zone_t zone) { - zone_drain_wait(zone, M_NOWAIT); + zone_drain_wait(zone, M_NOWAIT, 0); } /* @@ -1729,7 +1767,7 @@ zone_dtor(void *arg, int size, void *udata) * released and then refilled before we * remove it... we dont care for now */ - zone_drain_wait(zone, M_WAITOK); + zone_drain_wait(zone, M_WAITOK, 0); /* * Unlink all of our kegs. */ @@ -2240,7 +2278,8 @@ zalloc_start: if ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) { KASSERT(bucket->ub_cnt != 0, ("uma_zalloc_arg: Returning an empty bucket.")); - + if (bucket == zone->uz_unused) + zone->uz_unused = LIST_NEXT(bucket, ub_link); LIST_REMOVE(bucket, ub_link); cache->uc_allocbucket = bucket; ZONE_UNLOCK(zone); Index: uma_int.h =================================================================== --- uma_int.h (revision 260328) +++ uma_int.h (working copy) @@ -280,6 +280,7 @@ struct uma_zone { LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ LIST_HEAD(,uma_bucket) uz_buckets; /* full buckets */ + struct uma_bucket *uz_unused; /* First unused full bucket */ LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */ struct uma_klink uz_klink; /* klink for first keg. */