diff -urN src.orig/sys/kern/subr_pcpu.c src/sys/kern/subr_pcpu.c --- src.orig/sys/kern/subr_pcpu.c Sat Jan 14 13:15:09 2006 +++ src/sys/kern/subr_pcpu.c Sat Jan 14 17:23:17 2006 @@ -73,6 +73,7 @@ pcpu->pc_cpumask = 1 << cpuid; cpuid_to_pcpu[cpuid] = pcpu; SLIST_INSERT_HEAD(&cpuhead, pcpu, pc_allcpu); + pcpu->pc_pq_inited = 0; cpu_pcpu_init(pcpu, cpuid, size); } diff -urN src.orig/sys/sys/pcpu.h src/sys/sys/pcpu.h --- src.orig/sys/sys/pcpu.h Sat Jan 14 13:15:23 2006 +++ src/sys/sys/pcpu.h Sat Jan 14 21:02:47 2006 @@ -43,6 +43,8 @@ #include #include +#include +#include #include struct pcb; @@ -74,6 +76,10 @@ PCPU_MD_FIELDS; struct vmmeter pc_cnt; /* VM stats counters */ struct device *pc_device; + struct vpgqueues pc_free_pages; /* Free list */ + int pc_free_pages_cnt; + int pc_pq_inited; + int pc_cur_color; }; SLIST_HEAD(cpuhead, pcpu); diff -urN src.orig/sys/vm/vm_page.c src/sys/vm/vm_page.c --- src.orig/sys/vm/vm_page.c Sat Jan 14 13:14:42 2006 +++ src/sys/vm/vm_page.c Sun Jan 15 11:30:51 2006 @@ -766,7 +766,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) { vm_page_t m = NULL; - int color, flags, page_req; + int color, flags, page_req, found_pcpu; page_req = req & VM_ALLOC_CLASS_MASK; KASSERT(curthread->td_intr_nesting_level == 0 || @@ -789,7 +789,21 @@ }; loop: + found_pcpu = 0; + if (page_req != VM_ALLOC_INTERRUPT) { + critical_enter(); + if ((m = vm_pageq_find(PQ_PCPU, 0, 0))) { + vm_pageq_remove_nowakeup(m); + critical_exit(); + found_pcpu = 1; + goto found; + } + critical_exit(); + } else + atomic_add_int(&pq_pcpu_intr, 1); + mtx_lock_spin(&vm_page_queue_free_mtx); + vm_pageq_refill_pcpu(); if (cnt.v_free_count > cnt.v_free_reserved || (page_req == VM_ALLOC_SYSTEM && cnt.v_cache_count == 0 && @@ -844,6 +858,7 @@ */ vm_pageq_remove_nowakeup(m); +found: /* * Initialize structure. Only the PG_ZERO flag is inherited. */ @@ -866,7 +881,8 @@ m->busy = 0; m->valid = 0; KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m)); - mtx_unlock_spin(&vm_page_queue_free_mtx); + if (!found_pcpu) + mtx_unlock_spin(&vm_page_queue_free_mtx); if ((req & VM_ALLOC_NOOBJ) == 0) vm_page_insert(m, object, pindex); diff -urN src.orig/sys/vm/vm_page.h src/sys/vm/vm_page.h --- src.orig/sys/vm/vm_page.h Sat Jan 14 13:14:42 2006 +++ src/sys/vm/vm_page.h Sun Jan 15 11:31:18 2006 @@ -153,6 +153,7 @@ #define PQ_ACTIVE (page_queue_coloring.active) #define PQ_CACHE (page_queue_coloring.cache) #define PQ_HOLD (page_queue_coloring.hold) +#define PQ_PCPU (page_queue_coloring.pcpu) #define PQ_COUNT (page_queue_coloring.count) #define PQ_MAXCOLORS 1024 #define PQ_MAXCOUNT (4 + 2 * PQ_MAXCOLORS) @@ -162,6 +163,9 @@ #define PQ_COLORMASK (page_queue_coloring.colormask) #define PQ_MAXLENGTH (page_queue_coloring.maxlength) +#define PQ_REFILL_PCPU_NUM 32 +extern int pq_pcpu_intr; + /* Returns the real queue a page is on. */ #define VM_PAGE_GETQUEUE(m) ((m)->queue) @@ -195,6 +199,7 @@ int active; int cache; int hold; + int pcpu; int count; int maxlength; }; @@ -321,6 +326,7 @@ void vm_pageq_remove(vm_page_t m); vm_page_t vm_pageq_find(int basequeue, int index, boolean_t prefer_zero); void vm_pageq_requeue(vm_page_t m); +void vm_pageq_refill_pcpu(void); void vm_page_activate (vm_page_t); vm_page_t vm_page_alloc (vm_object_t, vm_pindex_t, int); diff -urN src.orig/sys/vm/vm_pageq.c src/sys/vm/vm_pageq.c --- src.orig/sys/vm/vm_pageq.c Sat Jan 14 13:14:42 2006 +++ src/sys/vm/vm_pageq.c Sun Jan 15 11:34:17 2006 @@ -38,6 +38,8 @@ #include #include #include +#include +#include #include #include @@ -57,6 +59,16 @@ static int pq_cachesize = 0; /* size of the cache in KB */ static int pq_cachenways = 0; /* associativity of the cache */ +static int pq_refill_pcpu_num = PQ_REFILL_PCPU_NUM; +static int pq_pcpu_max_len = 2*PQ_REFILL_PCPU_NUM; +int pq_pcpu_hit = 0; +int pq_pcpu_miss = 0; +int pq_pcpu_not_inited = 0; +int pq_pcpu_intr = 0; +int pq_pcpu_partial_refill = 0; +int pq_pcpu_full_refill = 0; +int pq_pcpu_no_refill = 0; + SYSCTL_DECL(_vm_stats); SYSCTL_NODE(_vm_stats, OID_AUTO, pagequeue, CTLFLAG_RW, 0, "VM meter stats"); SYSCTL_INT(_vm_stats_pagequeue, OID_AUTO, page_colors, CTLFLAG_RD, @@ -70,6 +82,19 @@ SYSCTL_INT(_vm_stats_pagequeue, OID_AUTO, prime2, CTLFLAG_RD, &(PQ_PRIME2), 0, "Cache tuning value"); +SYSCTL_NODE(_vm, OID_AUTO, pcpu, CTLFLAG_RW, 0, ""); +SYSCTL_INT(_vm_pcpu, OID_AUTO, refill_num, CTLFLAG_RW, &pq_refill_pcpu_num, 0, ""); +SYSCTL_INT(_vm_pcpu, OID_AUTO, max_len, CTLFLAG_RW, &pq_pcpu_max_len, 0, ""); +SYSCTL_NODE(_vm_pcpu, OID_AUTO, stats, CTLFLAG_RW, 0, ""); +SYSCTL_INT(_vm_pcpu_stats, OID_AUTO, hit, CTLFLAG_RW, &pq_pcpu_hit, 0, ""); +SYSCTL_INT(_vm_pcpu_stats, OID_AUTO, miss, CTLFLAG_RW, &pq_pcpu_miss, 0, ""); +SYSCTL_INT(_vm_pcpu_stats, OID_AUTO, not_inited, CTLFLAG_RW, &pq_pcpu_not_inited, 0, ""); +SYSCTL_INT(_vm_pcpu_stats, OID_AUTO, intr, CTLFLAG_RW, &pq_pcpu_intr, 0, ""); +SYSCTL_INT(_vm_pcpu_stats, OID_AUTO, partial_refill, CTLFLAG_RW, &pq_pcpu_partial_refill, 0, ""); +SYSCTL_INT(_vm_pcpu_stats, OID_AUTO, full_refill, CTLFLAG_RW, &pq_pcpu_full_refill, 0, ""); +SYSCTL_INT(_vm_pcpu_stats, OID_AUTO, no_refill, CTLFLAG_RW, &pq_pcpu_no_refill, 0, ""); + + static void vm_coloring_init(void) { @@ -117,7 +142,8 @@ PQ_ACTIVE = 2 + PQ_NUMCOLORS; PQ_CACHE = 3 + PQ_NUMCOLORS; PQ_HOLD = 3 + 2 * PQ_NUMCOLORS; - PQ_COUNT = 4 + 2 * PQ_NUMCOLORS; + PQ_PCPU = 4 + 2 * PQ_NUMCOLORS; + PQ_COUNT = 5 + 2 * PQ_NUMCOLORS; PQ_MAXLENGTH = PQ_NUMCOLORS / 3 + PQ_PRIME1; #if 0 @@ -159,6 +185,8 @@ int queue = VM_PAGE_GETQUEUE(m); struct vpgqueues *vpq; + KASSERT(queue != PQ_PCPU, ("requeuing page on per-cpu free list.")); + if (queue != PQ_NONE) { vpq = &vm_page_queues[queue]; TAILQ_REMOVE(&vpq->pl, m, pageq); @@ -174,9 +202,15 @@ { struct vpgqueues *vpq; - vpq = &vm_page_queues[queue]; + if (queue == PQ_PCPU) + vpq = PCPU_PTR(free_pages); + else + vpq = &vm_page_queues[queue]; VM_PAGE_SETQUEUE2(m, queue); - TAILQ_INSERT_TAIL(&vpq->pl, m, pageq); + if (m->flags & PG_ZERO) + TAILQ_INSERT_TAIL(&vpq->pl, m, pageq); + else + TAILQ_INSERT_HEAD(&vpq->pl, m, pageq); ++*vpq->cnt; ++vpq->lcnt; } @@ -247,7 +281,10 @@ int queue = VM_PAGE_GETQUEUE(m); struct vpgqueues *pq; if (queue != PQ_NONE) { - pq = &vm_page_queues[queue]; + if (queue == PQ_PCPU) + pq = PCPU_PTR(free_pages); + else + pq = &vm_page_queues[queue]; VM_PAGE_SETQUEUE2(m, PQ_NONE); TAILQ_REMOVE(&pq->pl, m, pageq); (*pq->cnt)--; @@ -271,7 +308,10 @@ if (queue != PQ_NONE) { VM_PAGE_SETQUEUE2(m, PQ_NONE); - pq = &vm_page_queues[queue]; + if (queue == PQ_PCPU) + pq = PCPU_PTR(free_pages); + else + pq = &vm_page_queues[queue]; TAILQ_REMOVE(&pq->pl, m, pageq); (*pq->cnt)--; pq->lcnt--; @@ -333,6 +373,24 @@ { vm_page_t m; + if (basequeue == PQ_PCPU) { + if (!PCPU_GET(pq_inited)) { + atomic_add_int(&pq_pcpu_not_inited, 1); + return (NULL); + } + if (TAILQ_EMPTY(&PCPU_PTR(free_pages)->pl)) { + atomic_add_int(&pq_pcpu_miss, 1); + return (NULL); + } + atomic_add_int(&pq_pcpu_hit, 1); + if (prefer_zero) + m = TAILQ_LAST(&PCPU_PTR(free_pages)->pl, pglist); + else + m = TAILQ_FIRST(&PCPU_PTR(free_pages)->pl); + + return (m); + } + #ifndef PQ_NOOPT if (PQ_NUMCOLORS > 1) { if (prefer_zero) { @@ -357,3 +415,39 @@ return (m); } +void +vm_pageq_refill_pcpu(void) +{ + int i, color; + vm_page_t m; + + if (!PCPU_GET(pq_inited)) { + printf("%s: initializing\n", __func__); + PCPU_PTR(free_pages)->cnt = PCPU_PTR(free_pages_cnt); + PCPU_SET(free_pages_cnt, 0); + TAILQ_INIT(&PCPU_PTR(free_pages)->pl); + PCPU_SET(cur_color, 0); + PCPU_SET(pq_inited, 1); + } + + /* Don't do anything if the pcpu list is big enough. */ + if (*(PCPU_PTR(free_pages)->cnt) > pq_pcpu_max_len) { + atomic_add_int(&pq_pcpu_no_refill, 1); + return; + } + + color = PCPU_GET(cur_color); + PCPU_SET(cur_color, color + 1); + for(i = 0; i < pq_refill_pcpu_num; i++) { + if (cnt.v_free_count <= cnt.v_free_reserved) { + atomic_add_int(&pq_pcpu_partial_refill, 1); + break; + } + m = vm_pageq_find(PQ_FREE, color % PQ_NUMCOLORS, i % 2); + if (m) { + vm_pageq_remove_nowakeup(m); + vm_pageq_enqueue(PQ_PCPU, m); + } + } + atomic_add_int(&pq_pcpu_full_refill, 1); +}