diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 7f9b3299cc12..1a28038c56b4 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -379,7 +379,8 @@ static void vm_page_domain_init(struct vm_domain *vmd) { struct vm_pagequeue *pq; - int i; + vm_page_t marker; + u_short queue; *__DECONST(char **, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_name) = "vm inactive pagequeue"; @@ -397,11 +398,15 @@ vm_page_domain_init(struct vm_domain *vmd) vmd->vmd_free_count = 0; vmd->vmd_segs = 0; vmd->vmd_oom = FALSE; - for (i = 0; i < PQ_COUNT; i++) { - pq = &vmd->vmd_pagequeues[i]; + for (queue = 0; queue < PQ_COUNT; queue++) { + pq = &vmd->vmd_pagequeues[queue]; TAILQ_INIT(&pq->pq_pl); mtx_init(&pq->pq_mutex, pq->pq_name, "vm pagequeue", MTX_DEF | MTX_DUPOK); + + marker = &pq->pq_insmarker; + vm_pageout_init_marker(marker, queue); + TAILQ_INSERT_TAIL(&pq->pq_pl, marker, plinks.q); } } @@ -2558,7 +2563,7 @@ vm_page_enqueue(uint8_t queue, vm_page_t m) pq = &vm_phys_domain(m)->vmd_pagequeues[queue]; vm_pagequeue_lock(pq); m->queue = queue; - TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); + TAILQ_INSERT_BEFORE(&pq->pq_insmarker, m, plinks.q); vm_pagequeue_cnt_inc(pq); vm_pagequeue_unlock(pq); } @@ -2581,7 +2586,7 @@ vm_page_requeue(vm_page_t m) pq = vm_page_pagequeue(m); vm_pagequeue_lock(pq); TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); - TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); + TAILQ_INSERT_BEFORE(&pq->pq_insmarker, m, plinks.q); vm_pagequeue_unlock(pq); } @@ -2602,7 +2607,7 @@ vm_page_requeue_locked(vm_page_t m) pq = vm_page_pagequeue(m); vm_pagequeue_assert_locked(pq); TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); - TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); + TAILQ_INSERT_BEFORE(&pq->pq_insmarker, m, plinks.q); } /* @@ -2867,7 +2872,7 @@ _vm_page_deactivate(vm_page_t m, boolean_t noreuse) TAILQ_INSERT_BEFORE(&vm_phys_domain(m)->vmd_inacthead, m, plinks.q); else - TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); + TAILQ_INSERT_BEFORE(&pq->pq_insmarker, m, plinks.q); vm_pagequeue_cnt_inc(pq); vm_pagequeue_unlock(pq); } diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 6e177194a53e..1d065b3b3997 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -214,13 +214,13 @@ SLIST_HEAD(spglist, vm_page); struct vm_pagequeue { struct mtx pq_mutex; - struct pglist pq_pl; int pq_cnt; u_int * const pq_vcnt; + struct vm_page pq_insmarker; + struct pglist pq_pl; const char * const pq_name; } __aligned(CACHE_LINE_SIZE); - struct vm_domain { struct vm_pagequeue vmd_pagequeues[PQ_COUNT]; u_int vmd_page_count; diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index a62aebdd28e3..a2ac4eb6732f 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -275,7 +275,7 @@ static boolean_t vm_pageout_page_lock(vm_page_t, vm_page_t *); * PG_MARKER. Nonetheless, it write busies and initializes the hold count * to one as safety precautions. */ -static void +void vm_pageout_init_marker(vm_page_t marker, u_short queue) { @@ -1266,7 +1266,7 @@ dolaundry: static bool vm_pageout_scan(struct vm_domain *vmd, int pass) { - vm_page_t m, next; + vm_page_t m, next, ins; struct vm_pagequeue *pq; vm_object_t object; long min_scan; @@ -1444,7 +1444,8 @@ unlock_page: vm_pagequeue_lock(pq); queue_locked = TRUE; m->queue = PQ_INACTIVE; - TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); + TAILQ_INSERT_BEFORE(&pq->pq_insmarker, m, + plinks.q); vm_pagequeue_cnt_inc(pq); goto drop_page; } @@ -1562,9 +1563,11 @@ drop_page: * the per-page activity counter and use it to identify deactivation * candidates. Held pages may be deactivated. */ - for (m = TAILQ_FIRST(&pq->pq_pl), scanned = 0; m != NULL && (scanned < - min_scan || (inactq_shortage > 0 && scanned < maxscan)); m = next, - scanned++) { + ins = &pq->pq_insmarker; + next = NULL; + for (m = TAILQ_NEXT(ins, plinks.q), scanned = 0; m != NULL && + (scanned < min_scan || (inactq_shortage > 0 && scanned < maxscan)); + m = next, scanned++) { KASSERT(m->queue == PQ_ACTIVE, ("vm_pageout_scan: page %p isn't active", m)); next = TAILQ_NEXT(m, plinks.q); @@ -1659,9 +1662,22 @@ drop_page: inactq_shortage--; } } - } else - vm_page_requeue_locked(m); + } vm_page_unlock(m); + if (next == NULL) + next = TAILQ_FIRST(&pq->pq_pl); + } + /* + * Reposition the insertion marker so that the next scan starts at + * "next." It's possible that we just scanned the entire queue, in + * which case "next" will be the marker itself. + */ + if (next != ins) { + TAILQ_REMOVE(&pq->pq_pl, ins, plinks.q); + if (next == NULL) + TAILQ_INSERT_HEAD(&pq->pq_pl, ins, plinks.q); + else + TAILQ_INSERT_BEFORE(next, ins, plinks.q); } vm_pagequeue_unlock(pq); #if !defined(NO_SWAPPING) diff --git a/sys/vm/vm_pageout.h b/sys/vm/vm_pageout.h index c67719747b5c..47c8a5c4ebe9 100644 --- a/sys/vm/vm_pageout.h +++ b/sys/vm/vm_pageout.h @@ -101,6 +101,7 @@ extern void vm_wait(void); extern void vm_waitpfault(void); #ifdef _KERNEL +void vm_pageout_init_marker(vm_page_t, u_short); int vm_pageout_flush(vm_page_t *, int, int, int, int *, boolean_t *); void vm_pageout_oom(int shortage); #endif