Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/uvm Break the global uvm_pageqlock into a per-page ident...



details:   https://anonhg.NetBSD.org/src/rev/4392d605e7c9
branches:  trunk
changeset: 1005566:4392d605e7c9
user:      ad <ad%NetBSD.org@localhost>
date:      Fri Dec 13 20:10:21 2019 +0000

description:
Break the global uvm_pageqlock into a per-page identity lock and a private
lock for use of the pagedaemon policy code.  Discussed on tech-kern.

PR kern/54209: NetBSD 8 large memory performance extremely low
PR kern/54210: NetBSD-8 processes presumably not exiting
PR kern/54727: writing a large file causes unreasonable system behaviour

diffstat:

 external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c |    2 -
 sys/arch/amd64/amd64/machdep.c                         |    6 +-
 sys/miscfs/genfs/genfs_io.c                            |   16 +-
 sys/nfs/nfs_bio.c                                      |    6 +-
 sys/rump/librump/rumpkern/vm.c                         |   25 +-
 sys/rump/librump/rumpvfs/vm_vfs.c                      |    6 +-
 sys/ufs/lfs/lfs_pages.c                                |   11 +-
 sys/ufs/lfs/lfs_vfsops.c                               |    8 +-
 sys/ufs/lfs/ulfs_inode.c                               |    6 +-
 sys/ufs/ufs/ufs_inode.c                                |    6 +-
 sys/uvm/uvm.h                                          |    3 +-
 sys/uvm/uvm_amap.c                                     |   16 +-
 sys/uvm/uvm_anon.c                                     |  111 ++------
 sys/uvm/uvm_aobj.c                                     |   41 +--
 sys/uvm/uvm_bio.c                                      |   13 +-
 sys/uvm/uvm_fault.c                                    |   37 +--
 sys/uvm/uvm_init.c                                     |    5 +-
 sys/uvm/uvm_km.c                                       |    6 +-
 sys/uvm/uvm_loan.c                                     |   96 +++----
 sys/uvm/uvm_map.c                                      |    7 +-
 sys/uvm/uvm_object.c                                   |   10 +-
 sys/uvm/uvm_page.c                                     |  180 ++++++++-------
 sys/uvm/uvm_page.h                                     |   80 +++---
 sys/uvm/uvm_pager.c                                    |   17 +-
 sys/uvm/uvm_pdaemon.c                                  |  195 ++++++++--------
 sys/uvm/uvm_pdpolicy.h                                 |    4 +-
 sys/uvm/uvm_pdpolicy_clock.c                           |  200 +++++++++++++---
 sys/uvm/uvm_pdpolicy_clockpro.c                        |  155 ++++++++++--
 sys/uvm/uvm_pglist.c                                   |    7 +-
 sys/uvm/uvm_physseg.c                                  |   10 +-
 30 files changed, 664 insertions(+), 621 deletions(-)

diffs (truncated from 3764 to 300 lines):

diff -r a86d3e6c1b60 -r 4392d605e7c9 external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c
--- a/external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c    Fri Dec 13 16:15:54 2019 +0000
+++ b/external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c    Fri Dec 13 20:10:21 2019 +0000
@@ -6081,9 +6081,7 @@
 
 out_unbusy:
        mutex_enter(mtx);
-       mutex_enter(&uvm_pageqlock);
        uvm_page_unbusy(pp, count);
-       mutex_exit(&uvm_pageqlock);
        mutex_exit(mtx);
 
 out:
diff -r a86d3e6c1b60 -r 4392d605e7c9 sys/arch/amd64/amd64/machdep.c
--- a/sys/arch/amd64/amd64/machdep.c    Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/arch/amd64/amd64/machdep.c    Fri Dec 13 20:10:21 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: machdep.c,v 1.343 2019/12/10 02:06:07 manu Exp $       */
+/*     $NetBSD: machdep.c,v 1.344 2019/12/13 20:14:25 ad Exp $ */
 
 /*
  * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.343 2019/12/10 02:06:07 manu Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.344 2019/12/13 20:14:25 ad Exp $");
 
 #include "opt_modular.h"
 #include "opt_multiboot.h"
@@ -865,7 +865,7 @@
                     pfn++) {
                        pg = PHYS_TO_VM_PAGE(ptoa(pfn));
 
-                       if (pg->uanon || (pg->pqflags & PQ_FREE) ||
+                       if (pg->uanon || (pg->flags & PG_FREE) ||
                            (pg->uobject && pg->uobject->pgops)) {
                                p = VM_PAGE_TO_PHYS(pg) / PAGE_SIZE;
                                clrbit(sparse_dump_physmap, p);
diff -r a86d3e6c1b60 -r 4392d605e7c9 sys/miscfs/genfs/genfs_io.c
--- a/sys/miscfs/genfs/genfs_io.c       Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/miscfs/genfs/genfs_io.c       Fri Dec 13 20:10:21 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: genfs_io.c,v 1.76 2019/10/06 05:48:00 mlelstv Exp $    */
+/*     $NetBSD: genfs_io.c,v 1.77 2019/12/13 20:10:21 ad Exp $ */
 
 /*
  * Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.76 2019/10/06 05:48:00 mlelstv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.77 2019/12/13 20:10:21 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -79,9 +79,7 @@
                        pg->flags |= PG_RELEASED;
                }
        }
-       mutex_enter(&uvm_pageqlock);
        uvm_page_unbusy(pgs, npages);
-       mutex_exit(&uvm_pageqlock);
 }
 
 static void
@@ -466,7 +464,6 @@
 out:
        UVMHIST_LOG(ubchist, "succeeding, npages %jd", npages,0,0,0);
        error = 0;
-       mutex_enter(&uvm_pageqlock);
        for (i = 0; i < npages; i++) {
                struct vm_page *pg = pgs[i];
                if (pg == NULL) {
@@ -498,7 +495,6 @@
                        UVM_PAGE_OWN(pg, NULL);
                }
        }
-       mutex_exit(&uvm_pageqlock);
        if (memwrite) {
                genfs_markdirty(vp);
        }
@@ -1201,9 +1197,6 @@
                 * apply FREE or DEACTIVATE options if requested.
                 */
 
-               if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
-                       mutex_enter(&uvm_pageqlock);
-               }
                for (i = 0; i < npages; i++) {
                        tpg = pgs[i];
                        KASSERT(tpg->uobject == uobj);
@@ -1236,9 +1229,6 @@
                                }
                        }
                }
-               if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
-                       mutex_exit(&uvm_pageqlock);
-               }
                if (needs_clean) {
                        modified = true;
 
@@ -1646,7 +1636,6 @@
        }
        uvm_pagermapout(kva, npages);
        mutex_enter(uobj->vmobjlock);
-       mutex_enter(&uvm_pageqlock);
        for (i = 0; i < npages; i++) {
                pg = pgs[i];
                if (error && (pg->flags & PG_FAKE) != 0) {
@@ -1659,7 +1648,6 @@
        if (error) {
                uvm_page_unbusy(pgs, npages);
        }
-       mutex_exit(&uvm_pageqlock);
        if (error == 0 && memwrite) {
                genfs_markdirty(vp);
        }
diff -r a86d3e6c1b60 -r 4392d605e7c9 sys/nfs/nfs_bio.c
--- a/sys/nfs/nfs_bio.c Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/nfs/nfs_bio.c Fri Dec 13 20:10:21 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: nfs_bio.c,v 1.191 2015/07/15 03:28:55 manu Exp $       */
+/*     $NetBSD: nfs_bio.c,v 1.192 2019/12/13 20:10:21 ad Exp $ */
 
 /*
  * Copyright (c) 1989, 1993
@@ -35,7 +35,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.191 2015/07/15 03:28:55 manu Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.192 2019/12/13 20:10:21 ad Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_nfs.h"
@@ -1340,9 +1340,7 @@
                                 * available and put back original pgs array.
                                 */
 
-                               mutex_enter(&uvm_pageqlock);
                                uvm_page_unbusy(pgs, npages);
-                               mutex_exit(&uvm_pageqlock);
                                *ap->a_count = 0;
                                memcpy(pgs, opgs,
                                    npages * sizeof(struct vm_pages *));
diff -r a86d3e6c1b60 -r 4392d605e7c9 sys/rump/librump/rumpkern/vm.c
--- a/sys/rump/librump/rumpkern/vm.c    Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/rump/librump/rumpkern/vm.c    Fri Dec 13 20:10:21 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: vm.c,v 1.173 2017/05/14 13:49:55 nat Exp $     */
+/*     $NetBSD: vm.c,v 1.174 2019/12/13 20:10:21 ad Exp $      */
 
 /*
  * Copyright (c) 2007-2011 Antti Kantee.  All Rights Reserved.
@@ -41,7 +41,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.173 2017/05/14 13:49:55 nat Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.174 2019/12/13 20:10:21 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -67,7 +67,7 @@
 
 #include <rump/rumpuser.h>
 
-kmutex_t uvm_pageqlock; /* non-free page lock */
+kmutex_t vmpage_lruqueue_lock; /* non-free page lock */
 kmutex_t uvm_fpageqlock; /* free page lock, non-gpl license */
 kmutex_t uvm_swap_data_lock;
 
@@ -213,9 +213,9 @@
         */
        if (!UVM_OBJ_IS_AOBJ(uobj)) {
                atomic_inc_uint(&vmpage_onqueue);
-               mutex_enter(&uvm_pageqlock);
+               mutex_enter(&vmpage_lruqueue_lock);
                TAILQ_INSERT_TAIL(&vmpage_lruqueue, pg, pageq.queue);
-               mutex_exit(&uvm_pageqlock);
+               mutex_exit(&vmpage_lruqueue_lock);
        }
 
        uobj->uo_npages++;
@@ -233,7 +233,6 @@
 {
        struct uvm_object *uobj = pg->uobject;
 
-       KASSERT(mutex_owned(&uvm_pageqlock));
        KASSERT(mutex_owned(uobj->vmobjlock));
 
        if (pg->flags & PG_WANTED)
@@ -245,7 +244,9 @@
        rb_tree_remove_node(&uobj->rb_tree, pg);
 
        if (!UVM_OBJ_IS_AOBJ(uobj)) {
+               mutex_enter(&vmpage_lruqueue_lock);
                TAILQ_REMOVE(&vmpage_lruqueue, pg, pageq.queue);
+               mutex_exit(&vmpage_lruqueue_lock);
                atomic_dec_uint(&vmpage_onqueue);
        }
 
@@ -366,7 +367,7 @@
 #endif
 
        mutex_init(&pagermtx, MUTEX_DEFAULT, IPL_NONE);
-       mutex_init(&uvm_pageqlock, MUTEX_DEFAULT, IPL_NONE);
+       mutex_init(&vmpage_lruqueue_lock, MUTEX_DEFAULT, IPL_NONE);
        mutex_init(&uvm_swap_data_lock, MUTEX_DEFAULT, IPL_NONE);
 
        /* just to appease linkage */
@@ -619,10 +620,10 @@
 
        pg = rb_tree_find_node(&uobj->rb_tree, &off);
        if (pg && !UVM_OBJ_IS_AOBJ(pg->uobject) && !ispagedaemon) {
-               mutex_enter(&uvm_pageqlock);
+               mutex_enter(&vmpage_lruqueue_lock);
                TAILQ_REMOVE(&vmpage_lruqueue, pg, pageq.queue);
                TAILQ_INSERT_TAIL(&vmpage_lruqueue, pg, pageq.queue);
-               mutex_exit(&uvm_pageqlock);
+               mutex_exit(&vmpage_lruqueue_lock);
        }
 
        return pg;
@@ -1052,7 +1053,7 @@
        uobj = pg->uobject;
        if (mutex_tryenter(uobj->vmobjlock)) {
                if ((pg->flags & PG_BUSY) == 0) {
-                       mutex_exit(&uvm_pageqlock);
+                       mutex_exit(&vmpage_lruqueue_lock);
                        uobj->pgops->pgo_put(uobj, pg->offset,
                            pg->offset + PAGE_SIZE,
                            PGO_CLEANIT|PGO_FREE);
@@ -1130,7 +1131,7 @@
                skip = 0;
                lockrunning = false;
  again:
-               mutex_enter(&uvm_pageqlock);
+               mutex_enter(&vmpage_lruqueue_lock);
                while (cleaned < PAGEDAEMON_OBJCHUNK) {
                        skipped = 0;
                        TAILQ_FOREACH(pg, &vmpage_lruqueue, pageq.queue) {
@@ -1153,7 +1154,7 @@
                        }
                        break;
                }
-               mutex_exit(&uvm_pageqlock);
+               mutex_exit(&vmpage_lruqueue_lock);
 
                /*
                 * Ok, someone is running with an object lock held.
diff -r a86d3e6c1b60 -r 4392d605e7c9 sys/rump/librump/rumpvfs/vm_vfs.c
--- a/sys/rump/librump/rumpvfs/vm_vfs.c Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/rump/librump/rumpvfs/vm_vfs.c Fri Dec 13 20:10:21 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: vm_vfs.c,v 1.34 2013/10/18 19:56:11 christos Exp $     */
+/*     $NetBSD: vm_vfs.c,v 1.35 2019/12/13 20:10:22 ad Exp $   */
 
 /*
  * Copyright (c) 2008-2011 Antti Kantee.  All Rights Reserved.
@@ -26,7 +26,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm_vfs.c,v 1.34 2013/10/18 19:56:11 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vm_vfs.c,v 1.35 2019/12/13 20:10:22 ad Exp $");
 
 #include <sys/param.h>
 
@@ -73,9 +73,7 @@
        }
        KASSERT(mutex_owned(uobj->vmobjlock));
 
-       mutex_enter(&uvm_pageqlock);
        uvm_page_unbusy(pgs, npages);
-       mutex_exit(&uvm_pageqlock);
        mutex_exit(uobj->vmobjlock);
 
        uvm_pagermapout((vaddr_t)bp->b_data, npages);
diff -r a86d3e6c1b60 -r 4392d605e7c9 sys/ufs/lfs/lfs_pages.c
--- a/sys/ufs/lfs/lfs_pages.c   Fri Dec 13 16:15:54 2019 +0000
+++ b/sys/ufs/lfs/lfs_pages.c   Fri Dec 13 20:10:21 2019 +0000
@@ -1,7 +1,7 @@
-/*     $NetBSD: lfs_pages.c,v 1.15 2017/08/19 14:22:49 maya Exp $      */
+/*     $NetBSD: lfs_pages.c,v 1.16 2019/12/13 20:10:22 ad Exp $        */
 
 /*-
- * Copyright (c) 1999, 2000, 2001, 2002, 2003 The NetBSD Foundation, Inc.
+ * Copyright (c) 1999, 2000, 2001, 2002, 2003, 2019 The NetBSD Foundation, Inc.
  * All rights reserved.
  *
  * This code is derived from software contributed to The NetBSD Foundation
@@ -60,7 +60,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.15 2017/08/19 14:22:49 maya Exp $");



Home | Main Index | Thread Index | Old Index