diff --git a/sys/compat/linux/common/linux_misc.c b/sys/compat/linux/common/linux_misc.c index 071a7cda6e1f..7976dd4690ef 100644 --- a/sys/compat/linux/common/linux_misc.c +++ b/sys/compat/linux/common/linux_misc.c @@ -1353,7 +1353,7 @@ linux_sys_sysinfo(struct lwp *l, const struct linux_sys_sysinfo_args *uap, regis si.totalram = ctob((u_long)physmem); si.freeram = (u_long)uvmexp.free * uvmexp.pagesize; si.sharedram = 0; /* XXX */ - si.bufferram = (u_long)uvmexp.filepages * uvmexp.pagesize; + si.bufferram = (u_long)uvmexp_filepages * uvmexp.pagesize; si.totalswap = (u_long)uvmexp.swpages * uvmexp.pagesize; si.freeswap = (u_long)(uvmexp.swpages - uvmexp.swpginuse) * uvmexp.pagesize; diff --git a/sys/compat/linux32/common/linux32_sysinfo.c b/sys/compat/linux32/common/linux32_sysinfo.c index accab1b88cbb..d224982d3f31 100644 --- a/sys/compat/linux32/common/linux32_sysinfo.c +++ b/sys/compat/linux32/common/linux32_sysinfo.c @@ -78,7 +78,7 @@ linux32_sys_sysinfo(struct lwp *l, const struct linux32_sys_sysinfo_args *uap, r si.totalram = ctob((u_long)physmem); si.freeram = (u_long)uvmexp.free * uvmexp.pagesize; si.sharedram = 0; /* XXX */ - si.bufferram = (u_long)uvmexp.filepages * uvmexp.pagesize; + si.bufferram = (u_long)uvmexp_filepages * uvmexp.pagesize; si.totalswap = (u_long)uvmexp.swpages * uvmexp.pagesize; si.freeswap = (u_long)(uvmexp.swpages - uvmexp.swpginuse) * uvmexp.pagesize; diff --git a/sys/fs/tmpfs/tmpfs_mem.c b/sys/fs/tmpfs/tmpfs_mem.c index bd021a19ba2e..30a0eeb6f94a 100644 --- a/sys/fs/tmpfs/tmpfs_mem.c +++ b/sys/fs/tmpfs/tmpfs_mem.c @@ -103,7 +103,7 @@ tmpfs_mem_info(bool total) size -= uvmexp.swpgonly; } size += uvmexp.free; - size += uvmexp.filepages; + size += uvmexp_filepages; if (size > uvmexp.wired) { size -= uvmexp.wired; } else { diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 63e115ae4b31..ee71f52196ae 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -247,6 +247,8 @@ static void configure2(void); static void configure3(void); void main(void); +void mutex_backoff_init(void); + /* * System startup; initialize the world, create process 0, mount root * filesystem, and fork to create init and pagedaemon. Most of the @@ -320,6 +322,8 @@ main(void) /* Initialize the sysctl subsystem. */ sysctl_init(); + mutex_backoff_init(); + /* Initialize callouts, part 1. */ callout_startup(); diff --git a/sys/kern/kern_idle.c b/sys/kern/kern_idle.c index ffe29bff38da..f1976633866d 100644 --- a/sys/kern/kern_idle.c +++ b/sys/kern/kern_idle.c @@ -80,9 +80,11 @@ idle_loop(void *dummy) sched_idle(); if (!sched_curcpu_runnable_p()) { +#if 0 if ((spc->spc_flags & SPCF_OFFLINE) == 0) { uvm_pageidlezero(); } +#endif if (!sched_curcpu_runnable_p()) { cpu_idle(); if (!sched_curcpu_runnable_p() && diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index 83cfa2096f4f..4112d120d790 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -49,6 +49,7 @@ __KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.65 2017/05/01 21:35:25 pgoyette Exp #include #include #include +#include #include #include #include @@ -210,6 +211,37 @@ do { \ #define MUTEX_INHERITDEBUG(n, o) /* nothing */ #endif /* defined(LOCKDEBUG) */ +static u_int __read_mostly mutex_backoff_min = 4; +static u_int __read_mostly mutex_backoff_max = 128; +static u_int __read_mostly mutex_backoff_style = 0; +struct sysctllog *mutex_backoff_log; + +void mutex_backoff_init(void); + +void +mutex_backoff_init(void) +{ + + sysctl_createv(&mutex_backoff_log, 0, NULL, NULL, + CTLFLAG_PERMANENT | CTLFLAG_READWRITE, + CTLTYPE_INT, "mutex_backoff_min", + SYSCTL_DESCR("backoff min"), + NULL, 0, &mutex_backoff_min, 0, + CTL_CREATE, CTL_EOL); + sysctl_createv(&mutex_backoff_log, 0, NULL, NULL, + CTLFLAG_PERMANENT | CTLFLAG_READWRITE, + CTLTYPE_INT, "mutex_backoff_max", + SYSCTL_DESCR("backoff max"), + NULL, 0, &mutex_backoff_max, 0, + CTL_CREATE, CTL_EOL); + sysctl_createv(&mutex_backoff_log, 0, NULL, NULL, + CTLFLAG_PERMANENT | CTLFLAG_READWRITE, + CTLTYPE_INT, "mutex_backoff_style", + SYSCTL_DESCR("backoff style"), + NULL, 0, &mutex_backoff_style, 0, + CTL_CREATE, CTL_EOL); +} + static inline int MUTEX_ACQUIRE(kmutex_t *mtx, uintptr_t curthread) { @@ -519,6 +551,7 @@ mutex_vector_enter(kmutex_t *mtx) * determine that the owner is not running on a processor, * then we stop spinning, and sleep instead. */ + count = mutex_backoff_min; KPREEMPT_DISABLE(curlwp); for (owner = mtx->mtx_owner;;) { if (!MUTEX_OWNED(owner)) { @@ -551,17 +584,27 @@ mutex_vector_enter(kmutex_t *mtx) */ if (mutex_oncpu(owner)) { LOCKSTAT_START_TIMER(lsflag, spintime); - count = SPINLOCK_BACKOFF_MIN; - do { - KPREEMPT_ENABLE(curlwp); - SPINLOCK_BACKOFF(count); - KPREEMPT_DISABLE(curlwp); - owner = mtx->mtx_owner; - } while (mutex_oncpu(owner)); + if (mutex_backoff_style == 0) { + count = mutex_backoff_min; + do { + KPREEMPT_ENABLE(curlwp); + spinlock_backoff_capped(&count, mutex_backoff_max); + KPREEMPT_DISABLE(curlwp); + owner = mtx->mtx_owner; + } while (mutex_oncpu(owner)); + } else { + do { + KPREEMPT_ENABLE(curlwp); + spinlock_backoff_capped(&count, mutex_backoff_max); + KPREEMPT_DISABLE(curlwp); + owner = mtx->mtx_owner; + } while (mutex_oncpu(owner)); + } LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_COUNT(spincnt, 1); if (!MUTEX_OWNED(owner)) continue; + } #endif diff --git a/sys/kern/vfs_vnode.c b/sys/kern/vfs_vnode.c index 2de60e6dd49c..7e60d2bc5357 100644 --- a/sys/kern/vfs_vnode.c +++ b/sys/kern/vfs_vnode.c @@ -772,9 +772,9 @@ vrelel(vnode_t *vp, int flags) /* Take care of space accounting. */ if (vp->v_iflag & VI_EXECMAP) { - atomic_add_int(&uvmexp.execpages, + atomic_add_int(&uvmexp_execpages, -vp->v_uobj.uo_npages); - atomic_add_int(&uvmexp.filepages, + atomic_add_int(&uvmexp_filepages, vp->v_uobj.uo_npages); } vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP); @@ -1553,8 +1553,8 @@ vcache_reclaim(vnode_t *vp) */ VSTATE_CHANGE(vp, VS_LOADED, VS_RECLAIMING); if (vp->v_iflag & VI_EXECMAP) { - atomic_add_int(&uvmexp.execpages, -vp->v_uobj.uo_npages); - atomic_add_int(&uvmexp.filepages, vp->v_uobj.uo_npages); + atomic_add_int(&uvmexp_execpages, -vp->v_uobj.uo_npages); + atomic_add_int(&uvmexp_filepages, vp->v_uobj.uo_npages); } vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP); mutex_exit(vp->v_interlock); diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c index 6a1d55e6891e..66952282ac5e 100644 --- a/sys/kern/vfs_vnops.c +++ b/sys/kern/vfs_vnops.c @@ -331,8 +331,8 @@ vn_markexec(struct vnode *vp) mutex_enter(vp->v_interlock); if ((vp->v_iflag & VI_EXECMAP) == 0) { - atomic_add_int(&uvmexp.filepages, -vp->v_uobj.uo_npages); - atomic_add_int(&uvmexp.execpages, vp->v_uobj.uo_npages); + atomic_add_int(&uvmexp_filepages, -vp->v_uobj.uo_npages); + atomic_add_int(&uvmexp_execpages, vp->v_uobj.uo_npages); vp->v_iflag |= VI_EXECMAP; } mutex_exit(vp->v_interlock); @@ -358,8 +358,8 @@ vn_marktext(struct vnode *vp) return (ETXTBSY); } if ((vp->v_iflag & VI_EXECMAP) == 0) { - atomic_add_int(&uvmexp.filepages, -vp->v_uobj.uo_npages); - atomic_add_int(&uvmexp.execpages, vp->v_uobj.uo_npages); + atomic_add_int(&uvmexp_filepages, -vp->v_uobj.uo_npages); + atomic_add_int(&uvmexp_execpages, vp->v_uobj.uo_npages); } vp->v_iflag |= (VI_TEXT | VI_EXECMAP); mutex_exit(vp->v_interlock); diff --git a/sys/miscfs/procfs/procfs_linux.c b/sys/miscfs/procfs/procfs_linux.c index b8f93ed86cac..5e5fb18fccea 100644 --- a/sys/miscfs/procfs/procfs_linux.c +++ b/sys/miscfs/procfs/procfs_linux.c @@ -160,16 +160,16 @@ procfs_domeminfo(struct lwp *curl, struct proc *p, PGTOB(uvmexp.npages - uvmexp.free), PGTOB(uvmexp.free), 0L, - PGTOB(uvmexp.filepages), - PGTOB(uvmexp.anonpages + uvmexp.filepages + uvmexp.execpages), + PGTOB(uvmexp_filepages), + PGTOB(uvmexp_anonpages + uvmexp_filepages + uvmexp_execpages), PGTOB(uvmexp.swpages), PGTOB(uvmexp.swpginuse), PGTOB(uvmexp.swpages - uvmexp.swpginuse), PGTOKB(uvmexp.npages), PGTOKB(uvmexp.free), 0L, - PGTOKB(uvmexp.filepages), - PGTOKB(uvmexp.anonpages + uvmexp.filepages + uvmexp.execpages), + PGTOKB(uvmexp_filepages), + PGTOKB(uvmexp_anonpages + uvmexp_filepages + uvmexp_execpages), PGTOKB(uvmexp.swpages), PGTOKB(uvmexp.swpages - uvmexp.swpginuse)); diff --git a/sys/sys/lock.h b/sys/sys/lock.h index 42c772f1b10e..b49ec1d0fb83 100644 --- a/sys/sys/lock.h +++ b/sys/sys/lock.h @@ -101,6 +101,17 @@ do { \ (count) += (count); \ } while (/* CONSTCOND */ 0); +static inline void +spinlock_backoff_capped(int *count, int max) +{ + int i; + for (i = *count; i != 0; i--) { + __asm __volatile("pause"); + } + if (*count < max) + *count += *count; +} + #define SPINLOCK_RUN_HOOK(count) ((count) >= SPINLOCK_BACKOFF_MAX) #ifdef LOCKDEBUG diff --git a/sys/sys/param.h b/sys/sys/param.h index 6273d44a6e28..ce4a1be357ee 100644 --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -455,7 +455,7 @@ #define UBC_WINSHIFT 13 #endif #ifndef UBC_NWINS -#define UBC_NWINS 1024 +#define UBC_NWINS 65536 #endif #ifdef _KERNEL diff --git a/sys/uvm/uvm_extern.h b/sys/uvm/uvm_extern.h index 58c6b674495d..f75d080cd49c 100644 --- a/sys/uvm/uvm_extern.h +++ b/sys/uvm/uvm_extern.h @@ -328,9 +328,6 @@ struct uvmexp { int zeropages; /* number of zero'd pages */ int reserve_pagedaemon; /* number of pages reserved for pagedaemon */ int reserve_kernel; /* number of pages reserved for kernel */ - unsigned anonpages; /* number of pages used by anon mappings */ - unsigned filepages; /* number of pages used by cached file data */ - unsigned execpages; /* number of pages used by cached exec data */ /* pageout params */ int freemin; /* min number of free pages */ @@ -410,6 +407,11 @@ struct uvmexp { int pdreexec; /* executable pages reactivated due to thresholds */ }; +extern unsigned uvmexp_anonpages; /* number of pages used by anon mappings */ +extern unsigned uvmexp_filepages; /* number of pages used by cached file data */ +extern unsigned uvmexp_execpages; /* number of pages used by cached exec data */ + + /* * The following structure is 64-bit alignment safe. New elements * should only be added to the end of this structure so binary diff --git a/sys/uvm/uvm_init.c b/sys/uvm/uvm_init.c index f571a33d0374..c4846a111a5d 100644 --- a/sys/uvm/uvm_init.c +++ b/sys/uvm/uvm_init.c @@ -58,16 +58,21 @@ struct uvm uvm; /* decl */ struct uvmexp uvmexp; /* decl */ struct uvm_object *uvm_kernel_object; +unsigned uvmexp_anonpages __cacheline_aligned; +unsigned uvmexp_filepages __cacheline_aligned; +unsigned uvmexp_execpages __cacheline_aligned; + + #if defined(__uvmexp_pagesize) const int * const uvmexp_pagesize = &uvmexp.pagesize; const int * const uvmexp_pagemask = &uvmexp.pagemask; const int * const uvmexp_pageshift = &uvmexp.pageshift; #endif -kmutex_t uvm_pageqlock; -kmutex_t uvm_fpageqlock; -kmutex_t uvm_kentry_lock; -kmutex_t uvm_swap_data_lock; +kmutex_t uvm_pageqlock __cacheline_aligned; +kmutex_t uvm_fpageqlock __cacheline_aligned; +kmutex_t uvm_kentry_lock __cacheline_aligned; +kmutex_t uvm_swap_data_lock __cacheline_aligned; /* * uvm_md_init: Init dependant on the MD boot context. diff --git a/sys/uvm/uvm_meter.c b/sys/uvm/uvm_meter.c index cd89ef60bdb3..7b6954f2c5e4 100644 --- a/sys/uvm/uvm_meter.c +++ b/sys/uvm/uvm_meter.c @@ -171,9 +171,9 @@ sysctl_vm_uvmexp2(SYSCTLFN_ARGS) u.pdpageouts = uvmexp.pdpageouts; u.pdpending = uvmexp.pdpending; u.pddeact = uvmexp.pddeact; - u.anonpages = uvmexp.anonpages; - u.filepages = uvmexp.filepages; - u.execpages = uvmexp.execpages; + u.anonpages = uvmexp_anonpages; + u.filepages = uvmexp_filepages; + u.execpages = uvmexp_execpages; u.colorhit = uvmexp.colorhit; u.colormiss = uvmexp.colormiss; u.ncolors = uvmexp.ncolors; diff --git a/sys/uvm/uvm_page.c b/sys/uvm/uvm_page.c index 660d3f5f2c53..1d619a07a369 100644 --- a/sys/uvm/uvm_page.c +++ b/sys/uvm/uvm_page.c @@ -224,12 +224,12 @@ uvm_pageinsert_list(struct uvm_object *uobj, struct vm_page *pg, vholdl(vp); } if (UVM_OBJ_IS_VTEXT(uobj)) { - atomic_inc_uint(&uvmexp.execpages); + atomic_inc_uint(&uvmexp_execpages); } else { - atomic_inc_uint(&uvmexp.filepages); + atomic_inc_uint(&uvmexp_filepages); } } else if (UVM_OBJ_IS_AOBJ(uobj)) { - atomic_inc_uint(&uvmexp.anonpages); + atomic_inc_uint(&uvmexp_anonpages); } if (where) @@ -282,12 +282,12 @@ uvm_pageremove_list(struct uvm_object *uobj, struct vm_page *pg) holdrelel(vp); } if (UVM_OBJ_IS_VTEXT(uobj)) { - atomic_dec_uint(&uvmexp.execpages); + atomic_dec_uint(&uvmexp_execpages); } else { - atomic_dec_uint(&uvmexp.filepages); + atomic_dec_uint(&uvmexp_filepages); } } else if (UVM_OBJ_IS_AOBJ(uobj)) { - atomic_dec_uint(&uvmexp.anonpages); + atomic_dec_uint(&uvmexp_anonpages); } /* object should be locked */ @@ -1073,7 +1073,7 @@ uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon, if (anon) { anon->an_page = pg; pg->pqflags = PQ_ANON; - atomic_inc_uint(&uvmexp.anonpages); + atomic_inc_uint(&uvmexp_anonpages); } else { if (obj) { uvm_pageinsert(obj, pg); @@ -1253,7 +1253,7 @@ uvm_pagefree(struct vm_page *pg) pg->loan_count--; } else { pg->pqflags &= ~PQ_ANON; - atomic_dec_uint(&uvmexp.anonpages); + atomic_dec_uint(&uvmexp_anonpages); } pg->uanon->an_page = NULL; pg->uanon = NULL; @@ -1283,7 +1283,7 @@ uvm_pagefree(struct vm_page *pg) uvm_pageremove(pg->uobject, pg); } else if (pg->uanon != NULL) { pg->uanon->an_page = NULL; - atomic_dec_uint(&uvmexp.anonpages); + atomic_dec_uint(&uvmexp_anonpages); } /* diff --git a/sys/uvm/uvm_pdaemon.c b/sys/uvm/uvm_pdaemon.c index e652590f1009..b9d6c4aaf5d9 100644 --- a/sys/uvm/uvm_pdaemon.c +++ b/sys/uvm/uvm_pdaemon.c @@ -1001,7 +1001,7 @@ uvm_reclaimable(void) * XXX ie. pools, traditional buffer cache. */ - filepages = uvmexp.filepages + uvmexp.execpages - uvmexp.wired; + filepages = uvmexp_filepages + uvmexp_execpages - uvmexp.wired; uvm_estimatepageable(&active, &inactive); if (filepages >= MIN((active + inactive) >> 4, 5 * 1024 * 1024 >> PAGE_SHIFT)) { diff --git a/sys/uvm/uvm_pdpolicy_clock.c b/sys/uvm/uvm_pdpolicy_clock.c index 47b50cf460fc..406332d90f47 100644 --- a/sys/uvm/uvm_pdpolicy_clock.c +++ b/sys/uvm/uvm_pdpolicy_clock.c @@ -145,12 +145,12 @@ uvmpdpol_scaninit(void) */ t = s->s_active + s->s_inactive + uvmexp.free; - anonunder = uvmexp.anonpages <= UVM_PCTPARAM_APPLY(&s->s_anonmin, t); - fileunder = uvmexp.filepages <= UVM_PCTPARAM_APPLY(&s->s_filemin, t); - execunder = uvmexp.execpages <= UVM_PCTPARAM_APPLY(&s->s_execmin, t); - anonover = uvmexp.anonpages > UVM_PCTPARAM_APPLY(&s->s_anonmax, t); - fileover = uvmexp.filepages > UVM_PCTPARAM_APPLY(&s->s_filemax, t); - execover = uvmexp.execpages > UVM_PCTPARAM_APPLY(&s->s_execmax, t); + anonunder = uvmexp_anonpages <= UVM_PCTPARAM_APPLY(&s->s_anonmin, t); + fileunder = uvmexp_filepages <= UVM_PCTPARAM_APPLY(&s->s_filemin, t); + execunder = uvmexp_execpages <= UVM_PCTPARAM_APPLY(&s->s_execmin, t); + anonover = uvmexp_anonpages > UVM_PCTPARAM_APPLY(&s->s_anonmax, t); + fileover = uvmexp_filepages > UVM_PCTPARAM_APPLY(&s->s_filemax, t); + execover = uvmexp_execpages > UVM_PCTPARAM_APPLY(&s->s_execmax, t); anonreact = anonunder || (!anonover && (fileover || execover)); filereact = fileunder || (!fileover && (anonover || execover)); execreact = execunder || (!execover && (anonover || fileover)); diff --git a/sys/uvm/uvm_stat.c b/sys/uvm/uvm_stat.c index a9c622985201..fdf9f8e724e3 100644 --- a/sys/uvm/uvm_stat.c +++ b/sys/uvm/uvm_stat.c @@ -67,7 +67,7 @@ uvmexp_print(void (*pr)(const char *, ...) uvmexp.npages, active, inactive, uvmexp.wired, uvmexp.free); (*pr)(" pages %d anon, %d file, %d exec\n", - uvmexp.anonpages, uvmexp.filepages, uvmexp.execpages); + uvmexp_anonpages, uvmexp_filepages, uvmexp_execpages); (*pr)(" freemin=%d, free-target=%d, wired-max=%d\n", uvmexp.freemin, uvmexp.freetarg, uvmexp.wiredmax); (*pr)(" resv-pg=%d, resv-kernel=%d, zeropages=%d\n",