Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src Break the global uvm_pageqlock into a per-page identity lock...
details: https://anonhg.NetBSD.org/src/rev/758dd8a3dfe2
branches: trunk
changeset: 466138:758dd8a3dfe2
user: ad <ad%NetBSD.org@localhost>
date: Fri Dec 13 20:10:21 2019 +0000
description:
Break the global uvm_pageqlock into a per-page identity lock and a private
lock for use of the pagedaemon policy code. Discussed on tech-kern.
PR kern/54209: NetBSD 8 large memory performance extremely low
PR kern/54210: NetBSD-8 processes presumably not exiting
PR kern/54727: writing a large file causes unreasonable system behaviour
diffstat:
external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c | 2 -
sys/arch/amd64/amd64/machdep.c | 6 +-
sys/miscfs/genfs/genfs_io.c | 16 +-
sys/nfs/nfs_bio.c | 6 +-
sys/rump/librump/rumpkern/vm.c | 25 +-
sys/rump/librump/rumpvfs/vm_vfs.c | 6 +-
sys/ufs/lfs/lfs_pages.c | 11 +-
sys/ufs/lfs/lfs_vfsops.c | 8 +-
sys/ufs/lfs/ulfs_inode.c | 6 +-
sys/ufs/ufs/ufs_inode.c | 6 +-
sys/uvm/uvm.h | 3 +-
sys/uvm/uvm_amap.c | 16 +-
sys/uvm/uvm_anon.c | 111 ++------
sys/uvm/uvm_aobj.c | 41 +--
sys/uvm/uvm_bio.c | 13 +-
sys/uvm/uvm_fault.c | 37 +--
sys/uvm/uvm_init.c | 5 +-
sys/uvm/uvm_km.c | 6 +-
sys/uvm/uvm_loan.c | 96 +++----
sys/uvm/uvm_map.c | 7 +-
sys/uvm/uvm_object.c | 10 +-
sys/uvm/uvm_page.c | 180 ++++++++-------
sys/uvm/uvm_page.h | 80 +++---
sys/uvm/uvm_pager.c | 17 +-
sys/uvm/uvm_pdaemon.c | 195 ++++++++--------
sys/uvm/uvm_pdpolicy.h | 4 +-
sys/uvm/uvm_pdpolicy_clock.c | 200 +++++++++++++---
sys/uvm/uvm_pdpolicy_clockpro.c | 155 ++++++++++--
sys/uvm/uvm_pglist.c | 7 +-
sys/uvm/uvm_physseg.c | 10 +-
30 files changed, 664 insertions(+), 621 deletions(-)
diffs (truncated from 3764 to 300 lines):
diff -r 86a777af3d6b -r 758dd8a3dfe2 external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c
--- a/external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c Fri Dec 13 16:15:54 2019 +0000
+++ b/external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c Fri Dec 13 20:10:21 2019 +0000
@@ -6081,9 +6081,7 @@
out_unbusy:
mutex_enter(mtx);
- mutex_enter(&uvm_pageqlock);
uvm_page_unbusy(pp, count);
- mutex_exit(&uvm_pageqlock);
mutex_exit(mtx);
out:
diff -r 86a777af3d6b -r 758dd8a3dfe2 sys/arch/amd64/amd64/machdep.c
--- a/sys/arch/amd64/amd64/machdep.c Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/arch/amd64/amd64/machdep.c Fri Dec 13 20:10:21 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.343 2019/12/10 02:06:07 manu Exp $ */
+/* $NetBSD: machdep.c,v 1.344 2019/12/13 20:14:25 ad Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.343 2019/12/10 02:06:07 manu Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.344 2019/12/13 20:14:25 ad Exp $");
#include "opt_modular.h"
#include "opt_multiboot.h"
@@ -865,7 +865,7 @@
pfn++) {
pg = PHYS_TO_VM_PAGE(ptoa(pfn));
- if (pg->uanon || (pg->pqflags & PQ_FREE) ||
+ if (pg->uanon || (pg->flags & PG_FREE) ||
(pg->uobject && pg->uobject->pgops)) {
p = VM_PAGE_TO_PHYS(pg) / PAGE_SIZE;
clrbit(sparse_dump_physmap, p);
diff -r 86a777af3d6b -r 758dd8a3dfe2 sys/miscfs/genfs/genfs_io.c
--- a/sys/miscfs/genfs/genfs_io.c Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/miscfs/genfs/genfs_io.c Fri Dec 13 20:10:21 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: genfs_io.c,v 1.76 2019/10/06 05:48:00 mlelstv Exp $ */
+/* $NetBSD: genfs_io.c,v 1.77 2019/12/13 20:10:21 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.76 2019/10/06 05:48:00 mlelstv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.77 2019/12/13 20:10:21 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -79,9 +79,7 @@
pg->flags |= PG_RELEASED;
}
}
- mutex_enter(&uvm_pageqlock);
uvm_page_unbusy(pgs, npages);
- mutex_exit(&uvm_pageqlock);
}
static void
@@ -466,7 +464,6 @@
out:
UVMHIST_LOG(ubchist, "succeeding, npages %jd", npages,0,0,0);
error = 0;
- mutex_enter(&uvm_pageqlock);
for (i = 0; i < npages; i++) {
struct vm_page *pg = pgs[i];
if (pg == NULL) {
@@ -498,7 +495,6 @@
UVM_PAGE_OWN(pg, NULL);
}
}
- mutex_exit(&uvm_pageqlock);
if (memwrite) {
genfs_markdirty(vp);
}
@@ -1201,9 +1197,6 @@
* apply FREE or DEACTIVATE options if requested.
*/
- if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
- mutex_enter(&uvm_pageqlock);
- }
for (i = 0; i < npages; i++) {
tpg = pgs[i];
KASSERT(tpg->uobject == uobj);
@@ -1236,9 +1229,6 @@
}
}
}
- if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
- mutex_exit(&uvm_pageqlock);
- }
if (needs_clean) {
modified = true;
@@ -1646,7 +1636,6 @@
}
uvm_pagermapout(kva, npages);
mutex_enter(uobj->vmobjlock);
- mutex_enter(&uvm_pageqlock);
for (i = 0; i < npages; i++) {
pg = pgs[i];
if (error && (pg->flags & PG_FAKE) != 0) {
@@ -1659,7 +1648,6 @@
if (error) {
uvm_page_unbusy(pgs, npages);
}
- mutex_exit(&uvm_pageqlock);
if (error == 0 && memwrite) {
genfs_markdirty(vp);
}
diff -r 86a777af3d6b -r 758dd8a3dfe2 sys/nfs/nfs_bio.c
--- a/sys/nfs/nfs_bio.c Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/nfs/nfs_bio.c Fri Dec 13 20:10:21 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: nfs_bio.c,v 1.191 2015/07/15 03:28:55 manu Exp $ */
+/* $NetBSD: nfs_bio.c,v 1.192 2019/12/13 20:10:21 ad Exp $ */
/*
* Copyright (c) 1989, 1993
@@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.191 2015/07/15 03:28:55 manu Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.192 2019/12/13 20:10:21 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_nfs.h"
@@ -1340,9 +1340,7 @@
* available and put back original pgs array.
*/
- mutex_enter(&uvm_pageqlock);
uvm_page_unbusy(pgs, npages);
- mutex_exit(&uvm_pageqlock);
*ap->a_count = 0;
memcpy(pgs, opgs,
npages * sizeof(struct vm_pages *));
diff -r 86a777af3d6b -r 758dd8a3dfe2 sys/rump/librump/rumpkern/vm.c
--- a/sys/rump/librump/rumpkern/vm.c Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/rump/librump/rumpkern/vm.c Fri Dec 13 20:10:21 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: vm.c,v 1.173 2017/05/14 13:49:55 nat Exp $ */
+/* $NetBSD: vm.c,v 1.174 2019/12/13 20:10:21 ad Exp $ */
/*
* Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved.
@@ -41,7 +41,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.173 2017/05/14 13:49:55 nat Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.174 2019/12/13 20:10:21 ad Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -67,7 +67,7 @@
#include <rump/rumpuser.h>
-kmutex_t uvm_pageqlock; /* non-free page lock */
+kmutex_t vmpage_lruqueue_lock; /* non-free page lock */
kmutex_t uvm_fpageqlock; /* free page lock, non-gpl license */
kmutex_t uvm_swap_data_lock;
@@ -213,9 +213,9 @@
*/
if (!UVM_OBJ_IS_AOBJ(uobj)) {
atomic_inc_uint(&vmpage_onqueue);
- mutex_enter(&uvm_pageqlock);
+ mutex_enter(&vmpage_lruqueue_lock);
TAILQ_INSERT_TAIL(&vmpage_lruqueue, pg, pageq.queue);
- mutex_exit(&uvm_pageqlock);
+ mutex_exit(&vmpage_lruqueue_lock);
}
uobj->uo_npages++;
@@ -233,7 +233,6 @@
{
struct uvm_object *uobj = pg->uobject;
- KASSERT(mutex_owned(&uvm_pageqlock));
KASSERT(mutex_owned(uobj->vmobjlock));
if (pg->flags & PG_WANTED)
@@ -245,7 +244,9 @@
rb_tree_remove_node(&uobj->rb_tree, pg);
if (!UVM_OBJ_IS_AOBJ(uobj)) {
+ mutex_enter(&vmpage_lruqueue_lock);
TAILQ_REMOVE(&vmpage_lruqueue, pg, pageq.queue);
+ mutex_exit(&vmpage_lruqueue_lock);
atomic_dec_uint(&vmpage_onqueue);
}
@@ -366,7 +367,7 @@
#endif
mutex_init(&pagermtx, MUTEX_DEFAULT, IPL_NONE);
- mutex_init(&uvm_pageqlock, MUTEX_DEFAULT, IPL_NONE);
+ mutex_init(&vmpage_lruqueue_lock, MUTEX_DEFAULT, IPL_NONE);
mutex_init(&uvm_swap_data_lock, MUTEX_DEFAULT, IPL_NONE);
/* just to appease linkage */
@@ -619,10 +620,10 @@
pg = rb_tree_find_node(&uobj->rb_tree, &off);
if (pg && !UVM_OBJ_IS_AOBJ(pg->uobject) && !ispagedaemon) {
- mutex_enter(&uvm_pageqlock);
+ mutex_enter(&vmpage_lruqueue_lock);
TAILQ_REMOVE(&vmpage_lruqueue, pg, pageq.queue);
TAILQ_INSERT_TAIL(&vmpage_lruqueue, pg, pageq.queue);
- mutex_exit(&uvm_pageqlock);
+ mutex_exit(&vmpage_lruqueue_lock);
}
return pg;
@@ -1052,7 +1053,7 @@
uobj = pg->uobject;
if (mutex_tryenter(uobj->vmobjlock)) {
if ((pg->flags & PG_BUSY) == 0) {
- mutex_exit(&uvm_pageqlock);
+ mutex_exit(&vmpage_lruqueue_lock);
uobj->pgops->pgo_put(uobj, pg->offset,
pg->offset + PAGE_SIZE,
PGO_CLEANIT|PGO_FREE);
@@ -1130,7 +1131,7 @@
skip = 0;
lockrunning = false;
again:
- mutex_enter(&uvm_pageqlock);
+ mutex_enter(&vmpage_lruqueue_lock);
while (cleaned < PAGEDAEMON_OBJCHUNK) {
skipped = 0;
TAILQ_FOREACH(pg, &vmpage_lruqueue, pageq.queue) {
@@ -1153,7 +1154,7 @@
}
break;
}
- mutex_exit(&uvm_pageqlock);
+ mutex_exit(&vmpage_lruqueue_lock);
/*
* Ok, someone is running with an object lock held.
diff -r 86a777af3d6b -r 758dd8a3dfe2 sys/rump/librump/rumpvfs/vm_vfs.c
--- a/sys/rump/librump/rumpvfs/vm_vfs.c Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/rump/librump/rumpvfs/vm_vfs.c Fri Dec 13 20:10:21 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: vm_vfs.c,v 1.34 2013/10/18 19:56:11 christos Exp $ */
+/* $NetBSD: vm_vfs.c,v 1.35 2019/12/13 20:10:22 ad Exp $ */
/*
* Copyright (c) 2008-2011 Antti Kantee. All Rights Reserved.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm_vfs.c,v 1.34 2013/10/18 19:56:11 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vm_vfs.c,v 1.35 2019/12/13 20:10:22 ad Exp $");
#include <sys/param.h>
@@ -73,9 +73,7 @@
}
KASSERT(mutex_owned(uobj->vmobjlock));
- mutex_enter(&uvm_pageqlock);
uvm_page_unbusy(pgs, npages);
- mutex_exit(&uvm_pageqlock);
mutex_exit(uobj->vmobjlock);
uvm_pagermapout((vaddr_t)bp->b_data, npages);
diff -r 86a777af3d6b -r 758dd8a3dfe2 sys/ufs/lfs/lfs_pages.c
--- a/sys/ufs/lfs/lfs_pages.c Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/ufs/lfs/lfs_pages.c Fri Dec 13 20:10:21 2019 +0000
@@ -1,7 +1,7 @@
-/* $NetBSD: lfs_pages.c,v 1.15 2017/08/19 14:22:49 maya Exp $ */
+/* $NetBSD: lfs_pages.c,v 1.16 2019/12/13 20:10:22 ad Exp $ */
/*-
- * Copyright (c) 1999, 2000, 2001, 2002, 2003 The NetBSD Foundation, Inc.
+ * Copyright (c) 1999, 2000, 2001, 2002, 2003, 2019 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -60,7 +60,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.15 2017/08/19 14:22:49 maya Exp $");
Home |
Main Index |
Thread Index |
Old Index