Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys Make page waits (WANTED vs BUSY) interlocked by pg->inte...



details:   https://anonhg.NetBSD.org/src/rev/739ddc7b7767
branches:  trunk
changeset: 1008204:739ddc7b7767
user:      ad <ad%NetBSD.org@localhost>
date:      Sat Mar 14 20:23:51 2020 +0000

description:
Make page waits (WANTED vs BUSY) interlocked by pg->interlock.  Gets RW
locks out of the equation for sleep/wakeup, and allows observing+waiting
for busy pages when holding only a read lock.  Proposed on tech-kern.

diffstat:

 sys/kern/kern_synch.c               |  44 +--------------------
 sys/miscfs/genfs/genfs_io.c         |  13 ++---
 sys/rump/librump/rumpkern/ltsleep.c |  19 +--------
 sys/rump/librump/rumpkern/vm.c      |  48 +++++++++++++++++++---
 sys/sys/proc.h                      |   3 +-
 sys/ufs/lfs/lfs_pages.c             |  17 +++----
 sys/ufs/lfs/lfs_vfsops.c            |   5 +-
 sys/uvm/uvm.h                       |   9 +---
 sys/uvm/uvm_amap.c                  |  13 ++---
 sys/uvm/uvm_anon.c                  |  11 +---
 sys/uvm/uvm_aobj.c                  |  25 +++--------
 sys/uvm/uvm_bio.c                   |  10 +---
 sys/uvm/uvm_fault.c                 |  64 +++++++++++-------------------
 sys/uvm/uvm_km.c                    |  10 +--
 sys/uvm/uvm_loan.c                  |  41 ++++++-------------
 sys/uvm/uvm_page.c                  |  77 ++++++++++++++++++++++++++++++------
 sys/uvm/uvm_page.h                  |  36 +++++++++++-----
 sys/uvm/uvm_vnode.c                 |   8 +--
 18 files changed, 210 insertions(+), 243 deletions(-)

diffs (truncated from 1136 to 300 lines):

diff -r babfc6fc7ed2 -r 739ddc7b7767 sys/kern/kern_synch.c
--- a/sys/kern/kern_synch.c     Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/kern/kern_synch.c     Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: kern_synch.c,v 1.343 2020/03/14 18:08:39 ad Exp $      */
+/*     $NetBSD: kern_synch.c,v 1.344 2020/03/14 20:23:51 ad Exp $      */
 
 /*-
  * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009, 2019, 2020
@@ -69,7 +69,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.343 2020/03/14 18:08:39 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.344 2020/03/14 20:23:51 ad Exp $");
 
 #include "opt_kstack.h"
 #include "opt_dtrace.h"
@@ -220,46 +220,6 @@
 }
 
 /*
- * XXXAD Temporary - for use of UVM only.  PLEASE DO NOT USE ELSEWHERE. 
- * Will go once there is a better solution, eg waits interlocked by
- * pg->interlock.  To wake an LWP sleeping with this, you need to hold a
- * write lock.
- */
-int
-rwtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
-        krwlock_t *rw)
-{
-       struct lwp *l = curlwp;
-       sleepq_t *sq;
-       kmutex_t *mp;
-       int error;
-       krw_t op;
-
-       KASSERT((l->l_pflag & LP_INTR) == 0);
-       KASSERT(ident != &lbolt);
-
-       if (sleepq_dontsleep(l)) {
-               (void)sleepq_abort(NULL, (priority & PNORELOCK) != 0);
-               if ((priority & PNORELOCK) != 0)
-                       rw_exit(rw);
-               return 0;
-       }
-
-       l->l_kpriority = true;
-       sq = sleeptab_lookup(&sleeptab, ident, &mp);
-       sleepq_enter(sq, l, mp);
-       sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
-       op = rw_lock_op(rw);
-       rw_exit(rw);
-       error = sleepq_block(timo, priority & PCATCH);
-
-       if ((priority & PNORELOCK) == 0)
-               rw_enter(rw, op);
-
-       return error;
-}
-
-/*
  * General sleep call for situations where a wake-up is not expected.
  */
 int
diff -r babfc6fc7ed2 -r 739ddc7b7767 sys/miscfs/genfs/genfs_io.c
--- a/sys/miscfs/genfs/genfs_io.c       Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/miscfs/genfs/genfs_io.c       Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: genfs_io.c,v 1.91 2020/03/14 19:07:22 ad Exp $ */
+/*     $NetBSD: genfs_io.c,v 1.92 2020/03/14 20:23:51 ad Exp $ */
 
 /*
  * Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.91 2020/03/14 19:07:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.92 2020/03/14 20:23:51 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -524,9 +524,6 @@
                if (i < ridx || i >= ridx + orignmempages || async) {
                        UVMHIST_LOG(ubchist, "unbusy pg %#jx offset 0x%jx",
                            (uintptr_t)pg, pg->offset,0,0);
-                       if (pg->flags & PG_WANTED) {
-                               wakeup(pg);
-                       }
                        if (pg->flags & PG_FAKE) {
                                KASSERT(overwrite);
                                uvm_pagezero(pg);
@@ -537,8 +534,9 @@
                        }
                        uvm_pagelock(pg);
                        uvm_pageenqueue(pg);
+                       uvm_pageunbusy(pg);
                        uvm_pageunlock(pg);
-                       pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
+                       pg->flags &= ~PG_FAKE;
                        UVM_PAGE_OWN(pg, NULL);
                } else if (memwrite && !overwrite &&
                    uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN) {
@@ -1093,8 +1091,7 @@
                                continue;
                        }
                        nextoff = pg->offset; /* visit this page again */
-                       pg->flags |= PG_WANTED;
-                       UVM_UNLOCK_AND_WAIT_RW(pg, slock, 0, "genput", 0);
+                       uvm_pagewait(pg, slock, "genput");
                        /*
                         * as we dropped the object lock, our cached pages can
                         * be stale.
diff -r babfc6fc7ed2 -r 739ddc7b7767 sys/rump/librump/rumpkern/ltsleep.c
--- a/sys/rump/librump/rumpkern/ltsleep.c       Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/rump/librump/rumpkern/ltsleep.c       Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ltsleep.c,v 1.35 2020/02/23 15:46:42 ad Exp $  */
+/*     $NetBSD: ltsleep.c,v 1.36 2020/03/14 20:23:51 ad Exp $  */
 
 /*
  * Copyright (c) 2009, 2010 Antti Kantee.  All Rights Reserved.
@@ -34,7 +34,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ltsleep.c,v 1.35 2020/02/23 15:46:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ltsleep.c,v 1.36 2020/03/14 20:23:51 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/kernel.h>
@@ -151,21 +151,6 @@
        return rv;
 }
 
-int
-rwtsleep(wchan_t ident, pri_t prio, const char *wmesg, int timo, krwlock_t *lock)
-{
-       krw_t op = rw_write_held(lock) ? RW_WRITER : RW_READER;
-       int rv;
-
-       mutex_spin_enter(&qlock);
-       rw_exit(lock);
-       rv = sleeper(ident, timo, true);
-       if ((prio & PNORELOCK) == 0)
-               rw_enter(lock, op);
-
-       return rv;
-}
-
 void
 wakeup(wchan_t ident)
 {
diff -r babfc6fc7ed2 -r 739ddc7b7767 sys/rump/librump/rumpkern/vm.c
--- a/sys/rump/librump/rumpkern/vm.c    Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/rump/librump/rumpkern/vm.c    Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: vm.c,v 1.185 2020/03/14 19:54:06 ad Exp $      */
+/*     $NetBSD: vm.c,v 1.186 2020/03/14 20:23:51 ad Exp $      */
 
 /*
  * Copyright (c) 2007-2011 Antti Kantee.  All Rights Reserved.
@@ -41,7 +41,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.185 2020/03/14 19:54:06 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.186 2020/03/14 20:23:51 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -219,8 +219,12 @@
 
        KASSERT(rw_write_held(uobj->vmobjlock));
 
-       if (pg->flags & PG_WANTED)
+       mutex_enter(&pg->interlock);
+       if (pg->pqflags & PQ_WANTED) {
+               pg->pqflags &= ~PQ_WANTED;
                wakeup(pg);
+       }
+       mutex_exit(&pg->interlock);
 
        uobj->uo_npages--;
        pg2 = radix_tree_remove_node(&uobj->uo_pages, pg->offset >> PAGE_SHIFT);
@@ -682,16 +686,44 @@
                        continue;
 
                KASSERT(pg->flags & PG_BUSY);
-               if (pg->flags & PG_WANTED)
-                       wakeup(pg);
-               if (pg->flags & PG_RELEASED)
+               if (pg->flags & PG_RELEASED) {
                        uvm_pagefree(pg);
-               else
-                       pg->flags &= ~(PG_WANTED|PG_BUSY);
+               } else {
+                       uvm_pagelock(pg);
+                       uvm_pageunbusy(pg);
+                       uvm_pageunlock(pg);
+               }
        }
 }
 
 void
+uvm_pagewait(struct vm_page *pg, krwlock_t *lock, const char *wmesg)
+{
+
+       KASSERT(rw_lock_held(lock));
+       KASSERT((pg->flags & PG_BUSY) != 0);
+
+       mutex_enter(&pg->interlock);
+       pg->pqflags |= PQ_WANTED;
+       rw_exit(lock);
+       UVM_UNLOCK_AND_WAIT(pg, &pg->interlock, false, wmesg, 0);
+}
+
+void
+uvm_pageunbusy(struct vm_page *pg)
+{
+
+       KASSERT((pg->flags & PG_BUSY) != 0);
+       KASSERT(mutex_owned(&pg->interlock));
+
+       if ((pg->pqflags & PQ_WANTED) != 0) {
+               pg->pqflags &= ~PQ_WANTED;
+               wakeup(pg);
+       }
+       pg->flags &= ~PG_BUSY;
+}
+
+void
 uvm_estimatepageable(int *active, int *inactive)
 {
 
diff -r babfc6fc7ed2 -r 739ddc7b7767 sys/sys/proc.h
--- a/sys/sys/proc.h    Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/sys/proc.h    Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: proc.h,v 1.359 2020/02/23 15:46:42 ad Exp $    */
+/*     $NetBSD: proc.h,v 1.360 2020/03/14 20:23:51 ad Exp $    */
 
 /*-
  * Copyright (c) 2006, 2007, 2008, 2020 The NetBSD Foundation, Inc.
@@ -513,7 +513,6 @@
 
 int    tsleep(wchan_t, pri_t, const char *, int);
 int    mtsleep(wchan_t, pri_t, const char *, int, kmutex_t *);
-int    rwtsleep(wchan_t, pri_t, const char *, int, krwlock_t *);
 void   wakeup(wchan_t);
 int    kpause(const char *, bool, int, kmutex_t *);
 void   exit1(struct lwp *, int, int) __dead;
diff -r babfc6fc7ed2 -r 739ddc7b7767 sys/ufs/lfs/lfs_pages.c
--- a/sys/ufs/lfs/lfs_pages.c   Sat Mar 14 19:54:06 2020 +0000
+++ b/sys/ufs/lfs/lfs_pages.c   Sat Mar 14 20:23:51 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: lfs_pages.c,v 1.22 2020/02/23 15:46:42 ad Exp $        */
+/*     $NetBSD: lfs_pages.c,v 1.23 2020/03/14 20:23:51 ad Exp $        */
 
 /*-
  * Copyright (c) 1999, 2000, 2001, 2002, 2003, 2019 The NetBSD Foundation, Inc.
@@ -60,7 +60,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.22 2020/02/23 15:46:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.23 2020/03/14 20:23:51 ad Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_compat_netbsd.h"
@@ -167,8 +167,7 @@
        lastpg = pg;
 #endif
 
-       pg->flags |= PG_WANTED;
-       UVM_UNLOCK_AND_WAIT_RW(pg, vp->v_uobj.vmobjlock, 0, "lfsput", 0);
+       uvm_pagewait(pg, vp->v_uobj.vmobjlock, "lfsput");
        rw_enter(vp->v_uobj.vmobjlock, RW_WRITER);
 }
 
@@ -349,9 +348,9 @@
                                        pg->flags |= PG_DELWRI;
                                }
                        }
-                       if (pg->flags & PG_WANTED)
-                               wakeup(pg);
-                       pg->flags &= ~(PG_WANTED|PG_BUSY);
+                       uvm_pagelock(pg);
+                       uvm_pageunbusy(pg);
+                       uvm_pageunlock(pg);
                        UVM_PAGE_OWN(pg, NULL);
                }
 
@@ -495,9 +494,7 @@
                        pg = uvm_pagelookup(&vp->v_uobj, off);
                        KASSERT(pg != NULL);
                        while (pg->flags & PG_BUSY) {
-                               pg->flags |= PG_WANTED;
-                               UVM_UNLOCK_AND_WAIT_RW(pg, vp->v_uobj.vmobjlock, 0,



Home | Main Index | Thread Index | Old Index