Index: amd64/conf/GENERIC =================================================================== --- amd64/conf/GENERIC (revision 274898) +++ amd64/conf/GENERIC (working copy) @@ -87,6 +87,8 @@ options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones +options DEBUG_MEMGUARD + # Make an SMP-capable kernel by default options SMP # Symmetric MultiProcessor Kernel Index: kern/subr_vmem.c =================================================================== --- kern/subr_vmem.c (revision 274898) +++ kern/subr_vmem.c (working copy) @@ -80,10 +80,11 @@ #define VMEM_QCACHE_IDX_MAX 16 -#define VMEM_FITMASK (M_BESTFIT | M_FIRSTFIT) +#define VMEM_FITMASK (M_BESTFIT | M_FIRSTFIT | M_NEXTFIT) #define VMEM_FLAGS \ - (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | M_BESTFIT | M_FIRSTFIT) + (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | M_BESTFIT | \ + M_FIRSTFIT | M_NEXTFIT) #define BT_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM) @@ -111,6 +112,28 @@ #define VMEM_NAME_MAX 16 +/* boundary tag */ +struct vmem_btag { + TAILQ_ENTRY(vmem_btag) bt_seglist; + union { + LIST_ENTRY(vmem_btag) u_freelist; /* BT_TYPE_FREE */ + LIST_ENTRY(vmem_btag) u_hashlist; /* BT_TYPE_BUSY */ + } bt_u; +#define bt_hashlist bt_u.u_hashlist +#define bt_freelist bt_u.u_freelist + vmem_addr_t bt_start; + vmem_size_t bt_size; + int bt_type; +}; + +#define BT_TYPE_SPAN 1 /* Allocated from importfn */ +#define BT_TYPE_SPAN_STATIC 2 /* vmem_add() or create. */ +#define BT_TYPE_FREE 3 /* Available space. */ +#define BT_TYPE_BUSY 4 /* Used space. */ +#define BT_ISSPAN_P(bt) ((bt)->bt_type <= BT_TYPE_SPAN_STATIC) + +#define BT_END(bt) ((bt)->bt_start + (bt)->bt_size - 1) + /* vmem arena */ struct vmem { struct mtx_padalign vm_lock; @@ -146,30 +169,11 @@ /* quantum cache */ qcache_t vm_qcache[VMEM_QCACHE_IDX_MAX]; -}; -/* boundary tag */ -struct vmem_btag { - TAILQ_ENTRY(vmem_btag) bt_seglist; - union { - LIST_ENTRY(vmem_btag) u_freelist; /* BT_TYPE_FREE */ - LIST_ENTRY(vmem_btag) u_hashlist; /* BT_TYPE_BUSY */ - } bt_u; -#define bt_hashlist bt_u.u_hashlist -#define bt_freelist bt_u.u_freelist - vmem_addr_t bt_start; - vmem_size_t bt_size; - int bt_type; + /* rotor for VM_NEXTFIT allocations */ + bt_t vm_rotor; }; -#define BT_TYPE_SPAN 1 /* Allocated from importfn */ -#define BT_TYPE_SPAN_STATIC 2 /* vmem_add() or create. */ -#define BT_TYPE_FREE 3 /* Available space. */ -#define BT_TYPE_BUSY 4 /* Used space. */ -#define BT_ISSPAN_P(bt) ((bt)->bt_type <= BT_TYPE_SPAN_STATIC) - -#define BT_END(bt) ((bt)->bt_start + (bt)->bt_size - 1) - #if defined(DIAGNOSTIC) static int enable_vmem_check = 1; SYSCTL_INT(_debug, OID_AUTO, vmem_check, CTLFLAG_RW, @@ -489,6 +493,23 @@ LIST_INSERT_HEAD(list, bt, bt_freelist); } +/* + * Allocate a boundary tag and initializs it. + */ +static bt_t * +bt_create(vmem_t *vm, bt_t *prev, vmem_addr_t start, vmem_size_t size, int type) +{ + bt_t *newbt; + + VMEM_ASSERT_LOCKED(vm); + newbt = bt_alloc(vm); + newbt->bt_start = start; + newbt->bt_size = size; + newbt->bt_type = type; + bt_insseg(vm, newbt, prev); + return (newbt); +} + /* ---- vmem internal functions */ /* @@ -954,6 +975,46 @@ bt->bt_type = BT_TYPE_BUSY; } +/* + * vmem_advance: Advance a walker from its previous position to 'after'. + * Note: this may drop and reacquire vmem lock. + */ +static void +vmem_advance(vmem_t *vm, bt_t *walker, bt_t *after) +{ + bt_t *btnext, *btprev, *bt; + + btprev = TAILQ_PREV(walker, vmem_seglist, bt_seglist); + btnext = TAILQ_NEXT(walker, bt_seglist); + bt = NULL; + + /* + * Coalesce walker's neighbors that were prevented from coalescing. + */ + if (btprev->bt_type == BT_TYPE_FREE) { + if (bt->bt_type == BT_TYPE_FREE) { + KASSERT(btprev->bt_start + btprev->bt_size == + btnext->bt_start, + ("Trying to coalesce non-adjacent boundary tags")); + bt_remfree(vm, btnext); + bt_remfree(vm, btprev); + btprev->bt_size += btnext->bt_size; + bt_insfree(vm, btprev); + // XXXDAVIDE: FIXME + // vm_seg_destroy(vm, vnext); + } + bt = btprev; + } else if (btnext->bt_type == BT_TYPE_FREE) + bt = btnext; + + /* + * vsp could represent a complete imported span, in which case + * we must return it to the source. + * XXXDAVIDE: FIXME. + */ + return; +} + /* ---- vmem API */ void @@ -1058,6 +1119,9 @@ vmem_destroy1(vm); } +/* + * XXX: Probably the following two functions should belong to param.h + */ vmem_size_t vmem_roundup_size(vmem_t *vm, vmem_size_t size) { @@ -1065,6 +1129,110 @@ return (size + vm->vm_quantum_mask) & ~vm->vm_quantum_mask; } +static int +vmem_samehighbit(vmem_size_t lhs, vmem_size_t rhs) +{ + + return ((lhs ^ rhs) < (lhs & rhs)); +} + +int +vmem_alloc_nextfit(vmem_t *vm, vmem_size_t size, int flags, vmem_addr_t *addrp) +{ + bt_t *rotor, *nbt; + vmem_addr_t addr; + vmem_size_t nbt_size, roundup_size; + int error, strat; + + flags &= VMEM_FLAGS; + strat = flags & VMEM_FITMASK; + MPASS(size > 0); + MPASS(strat == M_NEXTFIT); + MPASS((flags & (M_NOWAIT | M_WAITOK)) != (M_NOWAIT | M_WAITOK)); + if ((flags & M_NOWAIT) == 0) + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "vmem_alloc_nextfit"); + VMEM_LOCK(vm); + + /* + * Make sure we have enough tags to complete the operation. + */ + if (vm->vm_nfreetags < BT_MAXALLOC && !bt_fill(vm, flags)) { + VMEM_UNLOCK(vm); + return (ENOMEM); + } + roundup_size = vmem_roundup_size(vm, vm->vm_import_quantum); + + /* + * Fast path. The segment after the rotor is free and large enough + * that it won't change which freelist it's on. + */ + rotor = &vm->vm_rotor; + nbt = TAILQ_NEXT(rotor, bt_seglist); + if (nbt->bt_type == BT_TYPE_FREE && (nbt_size = nbt->bt_size) > + roundup_size && + vmem_samehighbit(nbt_size, nbt_size - roundup_size)) { + addr = nbt->bt_start; + nbt->bt_start = addr + roundup_size; + bt_insbusy(vm, bt_create(vm, + TAILQ_PREV(rotor, vmem_seglist, bt_seglist), addr, size, + BT_TYPE_BUSY)); + goto success; + } + + /* + * Slow-path. Starting at the rotor, look for a segment large + * enough to satisfy the allocation. + */ + for (;;) { + if (nbt->bt_type == BT_TYPE_FREE && nbt->bt_size >= size) + break; + nbt = TAILQ_NEXT(nbt, bt_seglist); + if (nbt == rotor) { + + /* + * Full scan. There might be enough space but + * the rotor itself is preventing the allocation to + * succeed because it's sitting in between two free + * segments. Advance the rotor and see if this + * liberates enough space. + */ + vmem_advance(vm, rotor, TAILQ_NEXT(rotor, bt_seglist)); + nbt = TAILQ_PREV(rotor, vmem_seglist, bt_seglist); + if (nbt->bt_type == BT_TYPE_FREE && + nbt->bt_size >= size) + break; + + /* + * Otherwise let's wait until another thread + * frees something. + */ + VMEM_CONDVAR_WAIT(vm); + nbt = TAILQ_NEXT(nbt, bt_seglist); + } + } + + /* + * We found a segment. Extract space to satisfy an allocation. + */ + addr = nbt->bt_start; + error = vmem_fit(nbt, size, 0, 0, 0, 0, 0, addrp); + if (error == 0) + vmem_clip(vm, nbt, *addrp, size); + KASSERT(nbt->bt_type = BT_TYPE_BUSY && nbt->bt_start == addr && + nbt->bt_size == size, ("boundary tag allocation failed")); + + /* + * Advance the rotor right after the newly allocated segment. + * That's where the next VM_NEXTFIT allocation will begin searching. + */ + vmem_advance(vm, rotor, nbt); +success: + VMEM_UNLOCK(vm); + addrp = ((vmem_addr_t *) addr); + return (0); +} + /* * vmem_alloc: allocate resource from the arena. */ @@ -1071,12 +1239,12 @@ int vmem_alloc(vmem_t *vm, vmem_size_t size, int flags, vmem_addr_t *addrp) { - const int strat __unused = flags & VMEM_FITMASK; + const int strat = flags & VMEM_FITMASK; qcache_t *qc; flags &= VMEM_FLAGS; MPASS(size > 0); - MPASS(strat == M_BESTFIT || strat == M_FIRSTFIT); + MPASS(strat == M_BESTFIT || strat == M_FIRSTFIT || strat == M_NEXTFIT); if ((flags & M_NOWAIT) == 0) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "vmem_alloc"); @@ -1088,8 +1256,11 @@ return (0); } - return vmem_xalloc(vm, size, 0, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, - flags, addrp); + if (strat == M_NEXTFIT) + return vmem_alloc_nextfit(vm, size, flags, addrp); + else /* M_BEST_FIT | M_FIRSTFIT */ + return vmem_xalloc(vm, size, 0, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, + flags, addrp); } int Index: sys/malloc.h =================================================================== --- sys/malloc.h (revision 274898) +++ sys/malloc.h (working copy) @@ -53,6 +53,7 @@ #define M_NODUMP 0x0800 /* don't dump pages in this allocation */ #define M_FIRSTFIT 0x1000 /* Only for vmem, fast fit. */ #define M_BESTFIT 0x2000 /* Only for vmem, low fragmentation. */ +#define M_NEXTFIT 0x4000 /* Only for vmem, next-fit policy. */ #define M_MAGIC 877983977 /* time when first defined :-) */ Index: sys/vmem.h =================================================================== --- sys/vmem.h (revision 274898) +++ sys/vmem.h (working copy) @@ -80,10 +80,12 @@ /* * Allocate and free linear regions from a vmem. Must specify - * BESTFIT or FIRSTFIT. Free is non-blocking. These routines + * BESTFIT, FIRSTFIT or NEXTFIT. Free is non-blocking. These routines * respect the quantum caches. */ int vmem_alloc(vmem_t *vm, vmem_size_t size, int flags, vmem_addr_t *addrp); +int vmem_alloc_nextfit(vmem_t *vm, vmem_size_t size, int flags, + vmem_addr_t *addrp); void vmem_free(vmem_t *vm, vmem_addr_t addr, vmem_size_t size); /* Index: vm/memguard.c =================================================================== --- vm/memguard.c (revision 274898) +++ vm/memguard.c (working copy) @@ -204,7 +204,7 @@ { vm_offset_t base; - vmem_alloc(parent, memguard_mapsize, M_BESTFIT | M_WAITOK, &base); + vmem_alloc(parent, memguard_mapsize, M_NEXTFIT | M_WAITOK, &base); vmem_init(memguard_arena, "memguard arena", base, memguard_mapsize, PAGE_SIZE, 0, M_WAITOK); memguard_cursor = base; @@ -328,7 +328,7 @@ for (;;) { if (vmem_xalloc(memguard_arena, size_v, 0, 0, 0, memguard_cursor, VMEM_ADDR_MAX, - M_BESTFIT | M_NOWAIT, &addr) == 0) + M_NEXTFIT | M_NOWAIT, &addr) == 0) break; /* * The map has no space. This may be due to