Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/usr.bin/vmstat Process concurrent page faults on individual ...



details:   https://anonhg.NetBSD.org/src/rev/01e1f264c9c4
branches:  trunk
changeset: 970391:01e1f264c9c4
user:      ad <ad%NetBSD.org@localhost>
date:      Sun Mar 22 18:32:41 2020 +0000

description:
Process concurrent page faults on individual uvm_objects / vm_amaps in
parallel, where the relevant pages are already in-core.  Proposed on
tech-kern.

Temporarily disabled on MP architectures with __HAVE_UNLOCKED_PMAP until
adjustments are made to their pmaps.

diffstat:

 sys/miscfs/genfs/genfs_io.c |   48 +++-
 sys/nfs/nfs_bio.c           |   20 +-
 sys/sys/cpu_data.h          |    6 +-
 sys/sys/vnode.h             |    5 +-
 sys/uvm/uvm_anon.c          |    6 +-
 sys/uvm/uvm_aobj.c          |   28 +-
 sys/uvm/uvm_extern.h        |    5 +-
 sys/uvm/uvm_fault.c         |  467 ++++++++++++++++++++++++++++++++++++-------
 sys/uvm/uvm_loan.c          |    5 +-
 sys/uvm/uvm_map.c           |   20 +-
 sys/uvm/uvm_meter.c         |    6 +-
 sys/uvm/uvm_pager.h         |    3 +-
 sys/uvm/uvm_vnode.c         |   20 +-
 usr.bin/vmstat/vmstat.c     |    8 +-
 14 files changed, 521 insertions(+), 126 deletions(-)

diffs (truncated from 1395 to 300 lines):

diff -r cc2fb35f5c66 -r 01e1f264c9c4 sys/miscfs/genfs/genfs_io.c
--- a/sys/miscfs/genfs/genfs_io.c       Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/miscfs/genfs/genfs_io.c       Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: genfs_io.c,v 1.94 2020/03/17 18:31:38 ad Exp $ */
+/*     $NetBSD: genfs_io.c,v 1.95 2020/03/22 18:32:41 ad Exp $ */
 
 /*
  * Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.94 2020/03/17 18:31:38 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.95 2020/03/22 18:32:41 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -43,6 +43,7 @@
 #include <sys/kauth.h>
 #include <sys/fstrans.h>
 #include <sys/buf.h>
+#include <sys/atomic.h>
 
 #include <miscfs/genfs/genfs.h>
 #include <miscfs/genfs/genfs_node.h>
@@ -103,7 +104,7 @@
        } */ * const ap = v;
 
        off_t diskeof, memeof;
-       int i, error, npages;
+       int i, error, npages, iflag;
        const int flags = ap->a_flags;
        struct vnode * const vp = ap->a_vp;
        struct uvm_object * const uobj = &vp->v_uobj;
@@ -125,18 +126,35 @@
        KASSERT(vp->v_type == VREG || vp->v_type == VDIR ||
            vp->v_type == VLNK || vp->v_type == VBLK);
 
+       /*
+        * the object must be locked.  it can only be a read lock when
+        * processing a read fault with PGO_LOCKED | PGO_NOBUSY.
+        */
+
+       KASSERT(rw_lock_held(uobj->vmobjlock));
+       KASSERT(rw_write_held(uobj->vmobjlock) ||
+          ((~flags & (PGO_LOCKED | PGO_NOBUSY)) == 0 && !memwrite));
+
 #ifdef DIAGNOSTIC
        if ((flags & PGO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl)
                 WAPBL_JLOCK_ASSERT(vp->v_mount);
 #endif
 
-       mutex_enter(vp->v_interlock);
-       error = vdead_check(vp, VDEAD_NOWAIT);
-       mutex_exit(vp->v_interlock);
-       if (error) {
-               if ((flags & PGO_LOCKED) == 0)
-                       rw_exit(uobj->vmobjlock);
-               return error;
+       /*
+        * check for reclaimed vnode.  v_interlock is not held here, but
+        * VI_DEADCHECK is set with vmobjlock held.
+        */
+
+       iflag = atomic_load_relaxed(&vp->v_iflag);
+       if (__predict_false((iflag & VI_DEADCHECK) != 0)) {
+               mutex_enter(vp->v_interlock);
+               error = vdead_check(vp, VDEAD_NOWAIT);
+               mutex_exit(vp->v_interlock);
+               if (error) {
+                       if ((flags & PGO_LOCKED) == 0)
+                               rw_exit(uobj->vmobjlock);
+                       return error;
+               }
        }
 
 startover:
@@ -217,9 +235,11 @@
                        KASSERT(pg == NULL || pg == PGO_DONTCARE);
                }
 #endif /* defined(DEBUG) */
-               nfound = uvn_findpages(uobj, origoffset, &npages,
+               nfound = uvn_findpages(uobj, origoffset, &npages,
                    ap->a_m, NULL,
-                   UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0));
+                   UFP_NOWAIT | UFP_NOALLOC |
+                   (memwrite ? UFP_NORDONLY : 0) |
+                   ((flags & PGO_NOBUSY) != 0 ? UFP_NOBUSY : 0));
                KASSERT(npages == *ap->a_count);
                if (nfound == 0) {
                        error = EBUSY;
@@ -230,7 +250,9 @@
                 * the file behind us.
                 */
                if (!genfs_node_rdtrylock(vp)) {
-                       genfs_rel_pages(ap->a_m, npages);
+                       if ((flags & PGO_NOBUSY) == 0) {
+                               genfs_rel_pages(ap->a_m, npages);
+                       }
 
                        /*
                         * restore the array.
diff -r cc2fb35f5c66 -r 01e1f264c9c4 sys/nfs/nfs_bio.c
--- a/sys/nfs/nfs_bio.c Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/nfs/nfs_bio.c Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: nfs_bio.c,v 1.194 2020/02/23 15:46:41 ad Exp $ */
+/*     $NetBSD: nfs_bio.c,v 1.195 2020/03/22 18:32:42 ad Exp $ */
 
 /*
  * Copyright (c) 1989, 1993
@@ -35,7 +35,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.194 2020/02/23 15:46:41 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.195 2020/03/22 18:32:42 ad Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_nfs.h"
@@ -1260,6 +1260,19 @@
        bool v3 = NFS_ISV3(vp);
        bool write = (ap->a_access_type & VM_PROT_WRITE) != 0;
        bool locked = (ap->a_flags & PGO_LOCKED) != 0;
+       bool nobusy = (ap->a_flags & PGO_NOBUSY);
+
+       /*
+        * XXX NFS wants to modify the pages below and that can't be done
+        * with a read lock.  We can't upgrade the lock here because it
+        * would screw up UVM fault processing.  Have NFS take the I/O
+        * path.
+        */
+       if (locked && rw_lock_op(uobj->vmobjlock) == RW_READER) {
+               *ap->a_count = 0;
+               ap->a_m[ap->a_centeridx] = NULL;
+               return EBUSY;
+       }
 
        /*
         * If we are not locked we are not really using opgs,
@@ -1341,7 +1354,8 @@
                                 * available and put back original pgs array.
                                 */
 
-                               uvm_page_unbusy(pgs, npages);
+                               if (nobusy == false)
+                                       uvm_page_unbusy(pgs, npages);
                                *ap->a_count = 0;
                                memcpy(pgs, opgs,
                                    npages * sizeof(struct vm_pages *));
diff -r cc2fb35f5c66 -r 01e1f264c9c4 sys/sys/cpu_data.h
--- a/sys/sys/cpu_data.h        Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/sys/cpu_data.h        Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpu_data.h,v 1.49 2020/01/15 17:55:44 ad Exp $ */
+/*     $NetBSD: cpu_data.h,v 1.50 2020/03/22 18:32:42 ad Exp $ */
 
 /*-
  * Copyright (c) 2004, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -93,8 +93,8 @@
        CPU_COUNT_FILEUNKNOWN,
        CPU_COUNT_FILECLEAN,
        CPU_COUNT_FILEDIRTY,
-       CPU_COUNT__UNUSED1,
-       CPU_COUNT__UNUSED2,
+       CPU_COUNT_FLTUP,
+       CPU_COUNT_FLTNOUP,
        CPU_COUNT_MAX                   /* 48 */
 };
 
diff -r cc2fb35f5c66 -r 01e1f264c9c4 sys/sys/vnode.h
--- a/sys/sys/vnode.h   Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/sys/vnode.h   Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: vnode.h,v 1.293 2020/03/14 20:45:23 ad Exp $   */
+/*     $NetBSD: vnode.h,v 1.294 2020/03/22 18:32:42 ad Exp $   */
 
 /*-
  * Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -212,6 +212,7 @@
 #define        VI_WRMAP        0x00000400      /* might have PROT_WRITE u. mappings */
 #define        VI_PAGES        0x00000800      /* UVM object has >0 pages */
 #define        VI_ONWORKLST    0x00004000      /* On syncer work-list */
+#define        VI_DEADCHECK    0x00008000      /* UVM: need to call vdead_check() */
 
 /*
  * The third set are locked by the underlying file system.
@@ -220,7 +221,7 @@
 
 #define        VNODE_FLAGBITS \
     "\20\1ROOT\2SYSTEM\3ISTTY\4MAPPED\5MPSAFE\6LOCKSWORK\11TEXT\12EXECMAP" \
-    "\13WRMAP\14PAGES\17ONWORKLST\31DIROP"
+    "\13WRMAP\14PAGES\17ONWORKLST\18DEADCHECK\31DIROP"
 
 #define        VSIZENOTSET     ((voff_t)-1)
 
diff -r cc2fb35f5c66 -r 01e1f264c9c4 sys/uvm/uvm_anon.c
--- a/sys/uvm/uvm_anon.c        Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/uvm/uvm_anon.c        Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_anon.c,v 1.76 2020/03/20 19:08:54 ad Exp $ */
+/*     $NetBSD: uvm_anon.c,v 1.77 2020/03/22 18:32:42 ad Exp $ */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.76 2020/03/20 19:08:54 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.77 2020/03/22 18:32:42 ad Exp $");
 
 #include "opt_uvmhist.h"
 
@@ -296,6 +296,8 @@
                 * anon was freed.
                 */
                return false;
+       case ENOLCK:
+               panic("uvm_anon_pagein");
        default:
                return true;
        }
diff -r cc2fb35f5c66 -r 01e1f264c9c4 sys/uvm/uvm_aobj.c
--- a/sys/uvm/uvm_aobj.c        Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/uvm/uvm_aobj.c        Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_aobj.c,v 1.138 2020/03/17 18:31:39 ad Exp $        */
+/*     $NetBSD: uvm_aobj.c,v 1.139 2020/03/22 18:32:42 ad Exp $        */
 
 /*
  * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
@@ -38,7 +38,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.138 2020/03/17 18:31:39 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.139 2020/03/22 18:32:42 ad Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_uvmhist.h"
@@ -807,6 +807,16 @@
                    (uintptr_t)uobj, offset, flags,0);
 
        /*
+        * the object must be locked.  it can only be a read lock when
+        * processing a read fault with PGO_LOCKED | PGO_NOBUSY.
+        */
+
+       KASSERT(rw_lock_held(uobj->vmobjlock));
+       KASSERT(rw_write_held(uobj->vmobjlock) ||
+          ((~flags & (PGO_LOCKED | PGO_NOBUSY)) == 0 &&
+          (access_type & VM_PROT_WRITE) == 0));
+
+       /*
         * get number of pages
         */
 
@@ -835,10 +845,12 @@
 
                        /*
                         * if page is new, attempt to allocate the page,
-                        * zero-fill'd.
+                        * zero-fill'd.  we can only do this if busying
+                        * pages, as otherwise the object is read locked.
                         */
 
-                       if (ptmp == NULL && uao_find_swslot(uobj,
+                       if ((flags & PGO_NOBUSY) == 0 && ptmp == NULL &&
+                           uao_find_swslot(uobj,
                            current_offset >> PAGE_SHIFT) == 0) {
                                ptmp = uao_pagealloc(uobj, current_offset,
                                    UVM_FLAG_COLORMATCH|UVM_PGA_ZERO);
@@ -870,9 +882,11 @@
                        KASSERT(uvm_pagegetdirty(ptmp) !=
                            UVM_PAGE_STATUS_CLEAN);
 
-                       /* caller must un-busy this page */
-                       ptmp->flags |= PG_BUSY;
-                       UVM_PAGE_OWN(ptmp, "uao_get1");
+                       if ((flags & PGO_NOBUSY) == 0) {
+                               /* caller must un-busy this page */
+                               ptmp->flags |= PG_BUSY;
+                               UVM_PAGE_OWN(ptmp, "uao_get1");
+                       }
 gotpage:
                        pps[lcv] = ptmp;
                        gotpages++;
diff -r cc2fb35f5c66 -r 01e1f264c9c4 sys/uvm/uvm_extern.h
--- a/sys/uvm/uvm_extern.h      Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/uvm/uvm_extern.h      Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_extern.h,v 1.221 2020/02/23 15:46:43 ad Exp $      */
+/*     $NetBSD: uvm_extern.h,v 1.222 2020/03/22 18:32:42 ad Exp $      */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -249,6 +249,7 @@
 #define UFP_NORDONLY   0x08



Home | Main Index | Thread Index | Old Index