Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src Process concurrent page faults on individual uvm_objects / v...
details: https://anonhg.NetBSD.org/src/rev/d03aebffc58c
branches: trunk
changeset: 746129:d03aebffc58c
user: ad <ad%NetBSD.org@localhost>
date: Sun Mar 22 18:32:41 2020 +0000
description:
Process concurrent page faults on individual uvm_objects / vm_amaps in
parallel, where the relevant pages are already in-core. Proposed on
tech-kern.
Temporarily disabled on MP architectures with __HAVE_UNLOCKED_PMAP until
adjustments are made to their pmaps.
diffstat:
sys/miscfs/genfs/genfs_io.c | 48 +++-
sys/nfs/nfs_bio.c | 20 +-
sys/sys/cpu_data.h | 6 +-
sys/sys/vnode.h | 5 +-
sys/uvm/uvm_anon.c | 6 +-
sys/uvm/uvm_aobj.c | 28 +-
sys/uvm/uvm_extern.h | 5 +-
sys/uvm/uvm_fault.c | 467 ++++++++++++++++++++++++++++++++++++-------
sys/uvm/uvm_loan.c | 5 +-
sys/uvm/uvm_map.c | 20 +-
sys/uvm/uvm_meter.c | 6 +-
sys/uvm/uvm_pager.h | 3 +-
sys/uvm/uvm_vnode.c | 20 +-
usr.bin/vmstat/vmstat.c | 8 +-
14 files changed, 521 insertions(+), 126 deletions(-)
diffs (truncated from 1395 to 300 lines):
diff -r 851360bd6609 -r d03aebffc58c sys/miscfs/genfs/genfs_io.c
--- a/sys/miscfs/genfs/genfs_io.c Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/miscfs/genfs/genfs_io.c Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: genfs_io.c,v 1.94 2020/03/17 18:31:38 ad Exp $ */
+/* $NetBSD: genfs_io.c,v 1.95 2020/03/22 18:32:41 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.94 2020/03/17 18:31:38 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.95 2020/03/22 18:32:41 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -43,6 +43,7 @@
#include <sys/kauth.h>
#include <sys/fstrans.h>
#include <sys/buf.h>
+#include <sys/atomic.h>
#include <miscfs/genfs/genfs.h>
#include <miscfs/genfs/genfs_node.h>
@@ -103,7 +104,7 @@
} */ * const ap = v;
off_t diskeof, memeof;
- int i, error, npages;
+ int i, error, npages, iflag;
const int flags = ap->a_flags;
struct vnode * const vp = ap->a_vp;
struct uvm_object * const uobj = &vp->v_uobj;
@@ -125,18 +126,35 @@
KASSERT(vp->v_type == VREG || vp->v_type == VDIR ||
vp->v_type == VLNK || vp->v_type == VBLK);
+ /*
+ * the object must be locked. it can only be a read lock when
+ * processing a read fault with PGO_LOCKED | PGO_NOBUSY.
+ */
+
+ KASSERT(rw_lock_held(uobj->vmobjlock));
+ KASSERT(rw_write_held(uobj->vmobjlock) ||
+ ((~flags & (PGO_LOCKED | PGO_NOBUSY)) == 0 && !memwrite));
+
#ifdef DIAGNOSTIC
if ((flags & PGO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl)
WAPBL_JLOCK_ASSERT(vp->v_mount);
#endif
- mutex_enter(vp->v_interlock);
- error = vdead_check(vp, VDEAD_NOWAIT);
- mutex_exit(vp->v_interlock);
- if (error) {
- if ((flags & PGO_LOCKED) == 0)
- rw_exit(uobj->vmobjlock);
- return error;
+ /*
+ * check for reclaimed vnode. v_interlock is not held here, but
+ * VI_DEADCHECK is set with vmobjlock held.
+ */
+
+ iflag = atomic_load_relaxed(&vp->v_iflag);
+ if (__predict_false((iflag & VI_DEADCHECK) != 0)) {
+ mutex_enter(vp->v_interlock);
+ error = vdead_check(vp, VDEAD_NOWAIT);
+ mutex_exit(vp->v_interlock);
+ if (error) {
+ if ((flags & PGO_LOCKED) == 0)
+ rw_exit(uobj->vmobjlock);
+ return error;
+ }
}
startover:
@@ -217,9 +235,11 @@
KASSERT(pg == NULL || pg == PGO_DONTCARE);
}
#endif /* defined(DEBUG) */
- nfound = uvn_findpages(uobj, origoffset, &npages,
+ nfound = uvn_findpages(uobj, origoffset, &npages,
ap->a_m, NULL,
- UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0));
+ UFP_NOWAIT | UFP_NOALLOC |
+ (memwrite ? UFP_NORDONLY : 0) |
+ ((flags & PGO_NOBUSY) != 0 ? UFP_NOBUSY : 0));
KASSERT(npages == *ap->a_count);
if (nfound == 0) {
error = EBUSY;
@@ -230,7 +250,9 @@
* the file behind us.
*/
if (!genfs_node_rdtrylock(vp)) {
- genfs_rel_pages(ap->a_m, npages);
+ if ((flags & PGO_NOBUSY) == 0) {
+ genfs_rel_pages(ap->a_m, npages);
+ }
/*
* restore the array.
diff -r 851360bd6609 -r d03aebffc58c sys/nfs/nfs_bio.c
--- a/sys/nfs/nfs_bio.c Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/nfs/nfs_bio.c Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: nfs_bio.c,v 1.194 2020/02/23 15:46:41 ad Exp $ */
+/* $NetBSD: nfs_bio.c,v 1.195 2020/03/22 18:32:42 ad Exp $ */
/*
* Copyright (c) 1989, 1993
@@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.194 2020/02/23 15:46:41 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.195 2020/03/22 18:32:42 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_nfs.h"
@@ -1260,6 +1260,19 @@
bool v3 = NFS_ISV3(vp);
bool write = (ap->a_access_type & VM_PROT_WRITE) != 0;
bool locked = (ap->a_flags & PGO_LOCKED) != 0;
+ bool nobusy = (ap->a_flags & PGO_NOBUSY);
+
+ /*
+ * XXX NFS wants to modify the pages below and that can't be done
+ * with a read lock. We can't upgrade the lock here because it
+ * would screw up UVM fault processing. Have NFS take the I/O
+ * path.
+ */
+ if (locked && rw_lock_op(uobj->vmobjlock) == RW_READER) {
+ *ap->a_count = 0;
+ ap->a_m[ap->a_centeridx] = NULL;
+ return EBUSY;
+ }
/*
* If we are not locked we are not really using opgs,
@@ -1341,7 +1354,8 @@
* available and put back original pgs array.
*/
- uvm_page_unbusy(pgs, npages);
+ if (nobusy == false)
+ uvm_page_unbusy(pgs, npages);
*ap->a_count = 0;
memcpy(pgs, opgs,
npages * sizeof(struct vm_pages *));
diff -r 851360bd6609 -r d03aebffc58c sys/sys/cpu_data.h
--- a/sys/sys/cpu_data.h Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/sys/cpu_data.h Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_data.h,v 1.49 2020/01/15 17:55:44 ad Exp $ */
+/* $NetBSD: cpu_data.h,v 1.50 2020/03/22 18:32:42 ad Exp $ */
/*-
* Copyright (c) 2004, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -93,8 +93,8 @@
CPU_COUNT_FILEUNKNOWN,
CPU_COUNT_FILECLEAN,
CPU_COUNT_FILEDIRTY,
- CPU_COUNT__UNUSED1,
- CPU_COUNT__UNUSED2,
+ CPU_COUNT_FLTUP,
+ CPU_COUNT_FLTNOUP,
CPU_COUNT_MAX /* 48 */
};
diff -r 851360bd6609 -r d03aebffc58c sys/sys/vnode.h
--- a/sys/sys/vnode.h Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/sys/vnode.h Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: vnode.h,v 1.293 2020/03/14 20:45:23 ad Exp $ */
+/* $NetBSD: vnode.h,v 1.294 2020/03/22 18:32:42 ad Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -212,6 +212,7 @@
#define VI_WRMAP 0x00000400 /* might have PROT_WRITE u. mappings */
#define VI_PAGES 0x00000800 /* UVM object has >0 pages */
#define VI_ONWORKLST 0x00004000 /* On syncer work-list */
+#define VI_DEADCHECK 0x00008000 /* UVM: need to call vdead_check() */
/*
* The third set are locked by the underlying file system.
@@ -220,7 +221,7 @@
#define VNODE_FLAGBITS \
"\20\1ROOT\2SYSTEM\3ISTTY\4MAPPED\5MPSAFE\6LOCKSWORK\11TEXT\12EXECMAP" \
- "\13WRMAP\14PAGES\17ONWORKLST\31DIROP"
+ "\13WRMAP\14PAGES\17ONWORKLST\18DEADCHECK\31DIROP"
#define VSIZENOTSET ((voff_t)-1)
diff -r 851360bd6609 -r d03aebffc58c sys/uvm/uvm_anon.c
--- a/sys/uvm/uvm_anon.c Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/uvm/uvm_anon.c Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_anon.c,v 1.76 2020/03/20 19:08:54 ad Exp $ */
+/* $NetBSD: uvm_anon.c,v 1.77 2020/03/22 18:32:42 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.76 2020/03/20 19:08:54 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.77 2020/03/22 18:32:42 ad Exp $");
#include "opt_uvmhist.h"
@@ -296,6 +296,8 @@
* anon was freed.
*/
return false;
+ case ENOLCK:
+ panic("uvm_anon_pagein");
default:
return true;
}
diff -r 851360bd6609 -r d03aebffc58c sys/uvm/uvm_aobj.c
--- a/sys/uvm/uvm_aobj.c Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/uvm/uvm_aobj.c Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_aobj.c,v 1.138 2020/03/17 18:31:39 ad Exp $ */
+/* $NetBSD: uvm_aobj.c,v 1.139 2020/03/22 18:32:42 ad Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
@@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.138 2020/03/17 18:31:39 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.139 2020/03/22 18:32:42 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_uvmhist.h"
@@ -807,6 +807,16 @@
(uintptr_t)uobj, offset, flags,0);
/*
+ * the object must be locked. it can only be a read lock when
+ * processing a read fault with PGO_LOCKED | PGO_NOBUSY.
+ */
+
+ KASSERT(rw_lock_held(uobj->vmobjlock));
+ KASSERT(rw_write_held(uobj->vmobjlock) ||
+ ((~flags & (PGO_LOCKED | PGO_NOBUSY)) == 0 &&
+ (access_type & VM_PROT_WRITE) == 0));
+
+ /*
* get number of pages
*/
@@ -835,10 +845,12 @@
/*
* if page is new, attempt to allocate the page,
- * zero-fill'd.
+ * zero-fill'd. we can only do this if busying
+ * pages, as otherwise the object is read locked.
*/
- if (ptmp == NULL && uao_find_swslot(uobj,
+ if ((flags & PGO_NOBUSY) == 0 && ptmp == NULL &&
+ uao_find_swslot(uobj,
current_offset >> PAGE_SHIFT) == 0) {
ptmp = uao_pagealloc(uobj, current_offset,
UVM_FLAG_COLORMATCH|UVM_PGA_ZERO);
@@ -870,9 +882,11 @@
KASSERT(uvm_pagegetdirty(ptmp) !=
UVM_PAGE_STATUS_CLEAN);
- /* caller must un-busy this page */
- ptmp->flags |= PG_BUSY;
- UVM_PAGE_OWN(ptmp, "uao_get1");
+ if ((flags & PGO_NOBUSY) == 0) {
+ /* caller must un-busy this page */
+ ptmp->flags |= PG_BUSY;
+ UVM_PAGE_OWN(ptmp, "uao_get1");
+ }
gotpage:
pps[lcv] = ptmp;
gotpages++;
diff -r 851360bd6609 -r d03aebffc58c sys/uvm/uvm_extern.h
--- a/sys/uvm/uvm_extern.h Sun Mar 22 17:35:29 2020 +0000
+++ b/sys/uvm/uvm_extern.h Sun Mar 22 18:32:41 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_extern.h,v 1.221 2020/02/23 15:46:43 ad Exp $ */
+/* $NetBSD: uvm_extern.h,v 1.222 2020/03/22 18:32:42 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -249,6 +249,7 @@
#define UFP_NORDONLY 0x08
Home |
Main Index |
Thread Index |
Old Index