diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index 455a8bcf49f9..98962fa5d032 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -663,8 +663,8 @@ zone_domain_imax_set(uma_zone_domain_t zdom, int nitems) * Attempt to satisfy an allocation by retrieving a full bucket from one of the * zone's caches. If a bucket is found the zone is not locked on return. */ -static uma_bucket_t -zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim) +static inline uma_bucket_t +zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, const bool reclaim) { uma_bucket_t bucket; int i; @@ -685,8 +685,13 @@ zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim) if (STAILQ_NEXT(bucket, ub_link) != NULL) zdom->uzd_seq = STAILQ_NEXT(bucket, ub_link)->ub_seq; } - MPASS(zdom->uzd_nitems >= bucket->ub_cnt); STAILQ_REMOVE_HEAD(&zdom->uzd_buckets, ub_link); + + KASSERT(zdom->uzd_nitems >= bucket->ub_cnt, + ("%s: item count underflow (%ld, %d)", + __func__, zdom->uzd_nitems, bucket->ub_cnt)); + KASSERT(bucket->ub_cnt > 0, + ("%s: empty bucket in bucket cache", __func__)); zdom->uzd_nitems -= bucket->ub_cnt; /* @@ -713,7 +718,7 @@ zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim) * whether the bucket's contents should be counted as part of the zone's working * set. The bucket may be freed if it exceeds the bucket limit. */ -static void +static inline void zone_put_bucket(uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata, const bool ws) { @@ -731,9 +736,19 @@ zone_put_bucket(uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata, if (__predict_true(zdom->uzd_nitems < zone->uz_bucket_max)) { if (ws) zone_domain_imax_set(zdom, zdom->uzd_nitems); - if (STAILQ_EMPTY(&zdom->uzd_buckets)) - zdom->uzd_seq = bucket->ub_seq; - STAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link); + + /* + * In non-SMR zones place the bucket at the head of the list to + * promote reuse of cache-hot items. Reuse of SMR items should + * be deferred to give readers time to finish. + */ + if ((zone->uz_flags & UMA_ZONE_SMR) != 0) { + if (STAILQ_EMPTY(&zdom->uzd_buckets)) + zdom->uzd_seq = bucket->ub_seq; + STAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link); + } else { + STAILQ_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link); + } ZDOM_UNLOCK(zdom); return; } @@ -906,11 +921,8 @@ cache_fetch_bucket(uma_zone_t zone, uma_cache_t cache, int domain) * Check the zone's cache of buckets. */ zdom = zone_domain_lock(zone, domain); - if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) { - KASSERT(bucket->ub_cnt != 0, - ("cache_fetch_bucket: Returning an empty bucket.")); + if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) return (bucket); - } ZDOM_UNLOCK(zdom); return (NULL); @@ -3331,7 +3343,7 @@ uma_zalloc_smr(uma_zone_t zone, int flags) void *item; KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, - ("uma_zalloc_arg: called with non-SMR zone.\n")); + ("uma_zalloc_arg: called with non-SMR zone.")); if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) return (item); #endif @@ -3362,7 +3374,7 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) void *item; KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, - ("uma_zalloc_arg: called with SMR zone.\n")); + ("uma_zalloc_arg: called with SMR zone.")); if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) return (item); #endif @@ -3487,6 +3499,11 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) void * uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) { +#ifdef NUMA + uma_bucket_t bucket; + uma_zone_domain_t zdom; + void *item; +#endif /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); @@ -3501,8 +3518,45 @@ uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) } KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), ("uma_zalloc_domain: called with spinlock or critical section held")); + KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, + ("uma_zalloc_domain: called with SMR zone.")); +#ifdef NUMA + KASSERT((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0, + ("uma_zalloc_domain: called with non-FIRSTTOUCH zone.")); + + if (vm_ndomains == 1) + return (uma_zalloc_arg(zone, udata, flags)); + /* + * Try to allocate from the bucket cache before falling back to the keg. + * We could try harder and attempt to allocate from per-CPU caches or + * the per-domain cross-domain buckets, but the complexity is probably + * not worth it. It is more important that frees of previous + * cross-domain allocations do not blow up the cache. + */ + zdom = zone_domain_lock(zone, domain); + if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) { + item = bucket->ub_bucket[bucket->ub_cnt - 1]; +#ifdef INVARIANTS + bucket->ub_bucket[bucket->ub_cnt - 1] = NULL; +#endif + bucket->ub_cnt--; + zone_put_bucket(zone, domain, bucket, udata, true); + item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, + flags, item); + if (item != NULL) { + KASSERT(item_domain(item) == domain, + ("%s: bucket cache item %p from wrong domain", + __func__, item)); + counter_u64_add(zone->uz_allocs, 1); + } + return (item); + } + ZDOM_UNLOCK(zdom); return (zone_alloc_item(zone, udata, domain, flags)); +#else + return (uma_zalloc_arg(zone, udata, flags)); +#endif } /* @@ -4007,7 +4061,7 @@ uma_zfree_smr(uma_zone_t zone, void *item) #ifdef UMA_ZALLOC_DEBUG KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, - ("uma_zfree_smr: called with non-SMR zone.\n")); + ("uma_zfree_smr: called with non-SMR zone.")); KASSERT(item != NULL, ("uma_zfree_smr: Called with NULL pointer.")); SMR_ASSERT_NOT_ENTERED(zone->uz_smr); if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN) @@ -4060,7 +4114,7 @@ uma_zfree_arg(uma_zone_t zone, void *item, void *udata) #ifdef UMA_ZALLOC_DEBUG KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, - ("uma_zfree_arg: called with SMR zone.\n")); + ("uma_zfree_arg: called with SMR zone.")); if (uma_zfree_debug(zone, item, udata) == EJUSTRETURN) return; #endif