Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys Make page waits (WANTED vs BUSY) interlocked by pg->inte...
details: https://anonhg.NetBSD.org/src/rev/4d7a7b1f058c
branches: trunk
changeset: 745868:4d7a7b1f058c
user: ad <ad%NetBSD.org@localhost>
date: Sat Mar 14 20:23:51 2020 +0000
description:
Make page waits (WANTED vs BUSY) interlocked by pg->interlock. Gets RW
locks out of the equation for sleep/wakeup, and allows observing+waiting
for busy pages when holding only a read lock. Proposed on tech-kern.
diffstat:
sys/kern/kern_synch.c | 44 +--------------------
sys/miscfs/genfs/genfs_io.c | 13 ++---
sys/rump/librump/rumpkern/ltsleep.c | 19 +--------
sys/rump/librump/rumpkern/vm.c | 48 +++++++++++++++++++---
sys/sys/proc.h | 3 +-
sys/ufs/lfs/lfs_pages.c | 17 +++----
sys/ufs/lfs/lfs_vfsops.c | 5 +-
sys/uvm/uvm.h | 9 +---
sys/uvm/uvm_amap.c | 13 ++---
sys/uvm/uvm_anon.c | 11 +---
sys/uvm/uvm_aobj.c | 25 +++--------
sys/uvm/uvm_bio.c | 10 +---
sys/uvm/uvm_fault.c | 64 +++++++++++-------------------
sys/uvm/uvm_km.c | 10 +--
sys/uvm/uvm_loan.c | 41 ++++++-------------
sys/uvm/uvm_page.c | 77 ++++++++++++++++++++++++++++++------
sys/uvm/uvm_page.h | 36 +++++++++++-----
sys/uvm/uvm_vnode.c | 8 +--
18 files changed, 210 insertions(+), 243 deletions(-)
diffs (truncated from 1136 to 300 lines):
diff -r 4f22f9a0d235 -r 4d7a7b1f058c sys/kern/kern_synch.c
--- a/sys/kern/kern_synch.c Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/kern/kern_synch.c Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_synch.c,v 1.343 2020/03/14 18:08:39 ad Exp $ */
+/* $NetBSD: kern_synch.c,v 1.344 2020/03/14 20:23:51 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009, 2019, 2020
@@ -69,7 +69,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.343 2020/03/14 18:08:39 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.344 2020/03/14 20:23:51 ad Exp $");
#include "opt_kstack.h"
#include "opt_dtrace.h"
@@ -220,46 +220,6 @@
}
/*
- * XXXAD Temporary - for use of UVM only. PLEASE DO NOT USE ELSEWHERE.
- * Will go once there is a better solution, eg waits interlocked by
- * pg->interlock. To wake an LWP sleeping with this, you need to hold a
- * write lock.
- */
-int
-rwtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
- krwlock_t *rw)
-{
- struct lwp *l = curlwp;
- sleepq_t *sq;
- kmutex_t *mp;
- int error;
- krw_t op;
-
- KASSERT((l->l_pflag & LP_INTR) == 0);
- KASSERT(ident != &lbolt);
-
- if (sleepq_dontsleep(l)) {
- (void)sleepq_abort(NULL, (priority & PNORELOCK) != 0);
- if ((priority & PNORELOCK) != 0)
- rw_exit(rw);
- return 0;
- }
-
- l->l_kpriority = true;
- sq = sleeptab_lookup(&sleeptab, ident, &mp);
- sleepq_enter(sq, l, mp);
- sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
- op = rw_lock_op(rw);
- rw_exit(rw);
- error = sleepq_block(timo, priority & PCATCH);
-
- if ((priority & PNORELOCK) == 0)
- rw_enter(rw, op);
-
- return error;
-}
-
-/*
* General sleep call for situations where a wake-up is not expected.
*/
int
diff -r 4f22f9a0d235 -r 4d7a7b1f058c sys/miscfs/genfs/genfs_io.c
--- a/sys/miscfs/genfs/genfs_io.c Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/miscfs/genfs/genfs_io.c Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: genfs_io.c,v 1.91 2020/03/14 19:07:22 ad Exp $ */
+/* $NetBSD: genfs_io.c,v 1.92 2020/03/14 20:23:51 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.91 2020/03/14 19:07:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.92 2020/03/14 20:23:51 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -524,9 +524,6 @@
if (i < ridx || i >= ridx + orignmempages || async) {
UVMHIST_LOG(ubchist, "unbusy pg %#jx offset 0x%jx",
(uintptr_t)pg, pg->offset,0,0);
- if (pg->flags & PG_WANTED) {
- wakeup(pg);
- }
if (pg->flags & PG_FAKE) {
KASSERT(overwrite);
uvm_pagezero(pg);
@@ -537,8 +534,9 @@
}
uvm_pagelock(pg);
uvm_pageenqueue(pg);
+ uvm_pageunbusy(pg);
uvm_pageunlock(pg);
- pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
+ pg->flags &= ~PG_FAKE;
UVM_PAGE_OWN(pg, NULL);
} else if (memwrite && !overwrite &&
uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN) {
@@ -1093,8 +1091,7 @@
continue;
}
nextoff = pg->offset; /* visit this page again */
- pg->flags |= PG_WANTED;
- UVM_UNLOCK_AND_WAIT_RW(pg, slock, 0, "genput", 0);
+ uvm_pagewait(pg, slock, "genput");
/*
* as we dropped the object lock, our cached pages can
* be stale.
diff -r 4f22f9a0d235 -r 4d7a7b1f058c sys/rump/librump/rumpkern/ltsleep.c
--- a/sys/rump/librump/rumpkern/ltsleep.c Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/rump/librump/rumpkern/ltsleep.c Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ltsleep.c,v 1.35 2020/02/23 15:46:42 ad Exp $ */
+/* $NetBSD: ltsleep.c,v 1.36 2020/03/14 20:23:51 ad Exp $ */
/*
* Copyright (c) 2009, 2010 Antti Kantee. All Rights Reserved.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ltsleep.c,v 1.35 2020/02/23 15:46:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ltsleep.c,v 1.36 2020/03/14 20:23:51 ad Exp $");
#include <sys/param.h>
#include <sys/kernel.h>
@@ -151,21 +151,6 @@
return rv;
}
-int
-rwtsleep(wchan_t ident, pri_t prio, const char *wmesg, int timo, krwlock_t *lock)
-{
- krw_t op = rw_write_held(lock) ? RW_WRITER : RW_READER;
- int rv;
-
- mutex_spin_enter(&qlock);
- rw_exit(lock);
- rv = sleeper(ident, timo, true);
- if ((prio & PNORELOCK) == 0)
- rw_enter(lock, op);
-
- return rv;
-}
-
void
wakeup(wchan_t ident)
{
diff -r 4f22f9a0d235 -r 4d7a7b1f058c sys/rump/librump/rumpkern/vm.c
--- a/sys/rump/librump/rumpkern/vm.c Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/rump/librump/rumpkern/vm.c Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: vm.c,v 1.185 2020/03/14 19:54:06 ad Exp $ */
+/* $NetBSD: vm.c,v 1.186 2020/03/14 20:23:51 ad Exp $ */
/*
* Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved.
@@ -41,7 +41,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.185 2020/03/14 19:54:06 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.186 2020/03/14 20:23:51 ad Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -219,8 +219,12 @@
KASSERT(rw_write_held(uobj->vmobjlock));
- if (pg->flags & PG_WANTED)
+ mutex_enter(&pg->interlock);
+ if (pg->pqflags & PQ_WANTED) {
+ pg->pqflags &= ~PQ_WANTED;
wakeup(pg);
+ }
+ mutex_exit(&pg->interlock);
uobj->uo_npages--;
pg2 = radix_tree_remove_node(&uobj->uo_pages, pg->offset >> PAGE_SHIFT);
@@ -682,16 +686,44 @@
continue;
KASSERT(pg->flags & PG_BUSY);
- if (pg->flags & PG_WANTED)
- wakeup(pg);
- if (pg->flags & PG_RELEASED)
+ if (pg->flags & PG_RELEASED) {
uvm_pagefree(pg);
- else
- pg->flags &= ~(PG_WANTED|PG_BUSY);
+ } else {
+ uvm_pagelock(pg);
+ uvm_pageunbusy(pg);
+ uvm_pageunlock(pg);
+ }
}
}
void
+uvm_pagewait(struct vm_page *pg, krwlock_t *lock, const char *wmesg)
+{
+
+ KASSERT(rw_lock_held(lock));
+ KASSERT((pg->flags & PG_BUSY) != 0);
+
+ mutex_enter(&pg->interlock);
+ pg->pqflags |= PQ_WANTED;
+ rw_exit(lock);
+ UVM_UNLOCK_AND_WAIT(pg, &pg->interlock, false, wmesg, 0);
+}
+
+void
+uvm_pageunbusy(struct vm_page *pg)
+{
+
+ KASSERT((pg->flags & PG_BUSY) != 0);
+ KASSERT(mutex_owned(&pg->interlock));
+
+ if ((pg->pqflags & PQ_WANTED) != 0) {
+ pg->pqflags &= ~PQ_WANTED;
+ wakeup(pg);
+ }
+ pg->flags &= ~PG_BUSY;
+}
+
+void
uvm_estimatepageable(int *active, int *inactive)
{
diff -r 4f22f9a0d235 -r 4d7a7b1f058c sys/sys/proc.h
--- a/sys/sys/proc.h Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/sys/proc.h Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: proc.h,v 1.359 2020/02/23 15:46:42 ad Exp $ */
+/* $NetBSD: proc.h,v 1.360 2020/03/14 20:23:51 ad Exp $ */
/*-
* Copyright (c) 2006, 2007, 2008, 2020 The NetBSD Foundation, Inc.
@@ -513,7 +513,6 @@
int tsleep(wchan_t, pri_t, const char *, int);
int mtsleep(wchan_t, pri_t, const char *, int, kmutex_t *);
-int rwtsleep(wchan_t, pri_t, const char *, int, krwlock_t *);
void wakeup(wchan_t);
int kpause(const char *, bool, int, kmutex_t *);
void exit1(struct lwp *, int, int) __dead;
diff -r 4f22f9a0d235 -r 4d7a7b1f058c sys/ufs/lfs/lfs_pages.c
--- a/sys/ufs/lfs/lfs_pages.c Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/ufs/lfs/lfs_pages.c Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: lfs_pages.c,v 1.22 2020/02/23 15:46:42 ad Exp $ */
+/* $NetBSD: lfs_pages.c,v 1.23 2020/03/14 20:23:51 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2003, 2019 The NetBSD Foundation, Inc.
@@ -60,7 +60,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.22 2020/02/23 15:46:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.23 2020/03/14 20:23:51 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_compat_netbsd.h"
@@ -167,8 +167,7 @@
lastpg = pg;
#endif
- pg->flags |= PG_WANTED;
- UVM_UNLOCK_AND_WAIT_RW(pg, vp->v_uobj.vmobjlock, 0, "lfsput", 0);
+ uvm_pagewait(pg, vp->v_uobj.vmobjlock, "lfsput");
rw_enter(vp->v_uobj.vmobjlock, RW_WRITER);
}
@@ -349,9 +348,9 @@
pg->flags |= PG_DELWRI;
}
}
- if (pg->flags & PG_WANTED)
- wakeup(pg);
- pg->flags &= ~(PG_WANTED|PG_BUSY);
+ uvm_pagelock(pg);
+ uvm_pageunbusy(pg);
+ uvm_pageunlock(pg);
UVM_PAGE_OWN(pg, NULL);
}
@@ -495,9 +494,7 @@
pg = uvm_pagelookup(&vp->v_uobj, off);
KASSERT(pg != NULL);
while (pg->flags & PG_BUSY) {
- pg->flags |= PG_WANTED;
- UVM_UNLOCK_AND_WAIT_RW(pg, vp->v_uobj.vmobjlock, 0,
Home |
Main Index |
Thread Index |
Old Index