Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys - Add and use wrapper functions that take and acquire pa...
details: https://anonhg.NetBSD.org/src/rev/edeecf00bd9c
branches: trunk
changeset: 466712:edeecf00bd9c
user: ad <ad%NetBSD.org@localhost>
date: Tue Dec 31 22:42:50 2019 +0000
description:
- Add and use wrapper functions that take and acquire page interlocks, and pairs
of page interlocks. Require that the page interlock be held over calls to
uvm_pageactivate(), uvm_pagewire() and similar.
- Solve the concurrency problem with page replacement state. Rather than
updating the global state synchronously, set an intended state on
individual pages (active, inactive, enqueued, dequeued) while holding the
page interlock. After the interlock is released put the pages on a 128
entry per-CPU queue for their state changes to be made real in batch.
This results in in a ~400 fold decrease in contention on my test system.
Proposed on tech-kern but modified to use the page interlock rather than
atomics to synchronise as it's much easier to maintain that way, and
cheaper.
diffstat:
sys/kern/kern_idle.c | 8 +-
sys/miscfs/genfs/genfs_io.c | 12 +-
sys/ufs/lfs/lfs_pages.c | 8 +-
sys/ufs/lfs/lfs_vfsops.c | 8 +-
sys/ufs/lfs/ulfs_inode.c | 6 +-
sys/ufs/ufs/ufs_inode.c | 6 +-
sys/uvm/uvm.h | 8 +-
sys/uvm/uvm_anon.c | 6 +-
sys/uvm/uvm_aobj.c | 8 +-
sys/uvm/uvm_bio.c | 10 +-
sys/uvm/uvm_extern.h | 3 +-
sys/uvm/uvm_fault.c | 27 ++-
sys/uvm/uvm_glue.c | 24 ++-
sys/uvm/uvm_loan.c | 52 +++--
sys/uvm/uvm_map.c | 6 +-
sys/uvm/uvm_object.c | 8 +-
sys/uvm/uvm_page.c | 137 +++++++++++++--
sys/uvm/uvm_page.h | 24 ++-
sys/uvm/uvm_pager.c | 8 +-
sys/uvm/uvm_pdaemon.c | 12 +-
sys/uvm/uvm_pdpolicy.h | 18 +-
sys/uvm/uvm_pdpolicy_clock.c | 341 ++++++++++++++++++++++++++++++++++-----
sys/uvm/uvm_pdpolicy_clockpro.c | 131 +++++++++++---
23 files changed, 699 insertions(+), 172 deletions(-)
diffs (truncated from 2029 to 300 lines):
diff -r d28e6d2f93e0 -r edeecf00bd9c sys/kern/kern_idle.c
--- a/sys/kern/kern_idle.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/kern/kern_idle.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_idle.c,v 1.28 2019/12/06 21:36:10 ad Exp $ */
+/* $NetBSD: kern_idle.c,v 1.29 2019/12/31 22:42:51 ad Exp $ */
/*-
* Copyright (c)2002, 2006, 2007 YAMAMOTO Takashi,
@@ -28,7 +28,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_idle.c,v 1.28 2019/12/06 21:36:10 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_idle.c,v 1.29 2019/12/31 22:42:51 ad Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@@ -39,7 +39,7 @@
#include <sys/proc.h>
#include <sys/atomic.h>
-#include <uvm/uvm.h> /* uvm_pageidlezero */
+#include <uvm/uvm.h> /* uvm_idle */
#include <uvm/uvm_extern.h>
void
@@ -81,7 +81,7 @@
sched_idle();
if (!sched_curcpu_runnable_p()) {
if ((spc->spc_flags & SPCF_OFFLINE) == 0) {
- uvm_pageidlezero();
+ uvm_idle();
}
if (!sched_curcpu_runnable_p()) {
cpu_idle();
diff -r d28e6d2f93e0 -r edeecf00bd9c sys/miscfs/genfs/genfs_io.c
--- a/sys/miscfs/genfs/genfs_io.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/miscfs/genfs/genfs_io.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: genfs_io.c,v 1.82 2019/12/31 12:40:27 ad Exp $ */
+/* $NetBSD: genfs_io.c,v 1.83 2019/12/31 22:42:50 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.82 2019/12/31 12:40:27 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.83 2019/12/31 22:42:50 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -491,7 +491,9 @@
uvm_pagefree(pg);
continue;
}
+ uvm_pagelock(pg);
uvm_pageenqueue(pg);
+ uvm_pageunlock(pg);
pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
}
@@ -1164,14 +1166,18 @@
if (tpg->offset < startoff || tpg->offset >= endoff)
continue;
if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) {
+ uvm_pagelock(tpg);
uvm_pagedeactivate(tpg);
+ uvm_pageunlock(tpg);
} else if (flags & PGO_FREE) {
pmap_page_protect(tpg, VM_PROT_NONE);
if (tpg->flags & PG_BUSY) {
tpg->flags |= freeflag;
if (pagedaemon) {
uvm_pageout_start(1);
+ uvm_pagelock(tpg);
uvm_pagedequeue(tpg);
+ uvm_pageunlock(tpg);
}
} else {
@@ -1603,7 +1609,9 @@
pg->flags |= PG_RELEASED;
} else {
pmap_clear_modify(pg);
+ uvm_pagelock(pg);
uvm_pageactivate(pg);
+ uvm_pageunlock(pg);
}
}
if (error) {
diff -r d28e6d2f93e0 -r edeecf00bd9c sys/ufs/lfs/lfs_pages.c
--- a/sys/ufs/lfs/lfs_pages.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/ufs/lfs/lfs_pages.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: lfs_pages.c,v 1.18 2019/12/20 20:54:48 ad Exp $ */
+/* $NetBSD: lfs_pages.c,v 1.19 2019/12/31 22:42:51 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2003, 2019 The NetBSD Foundation, Inc.
@@ -60,7 +60,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.18 2019/12/20 20:54:48 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.19 2019/12/31 22:42:51 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_compat_netbsd.h"
@@ -338,7 +338,9 @@
* Wire the page so that
* pdaemon doesn't see it again.
*/
+ uvm_pagelock(pg);
uvm_pagewire(pg);
+ uvm_pageunlock(pg);
/* Suspended write flag */
pg->flags |= PG_DELWRI;
@@ -495,7 +497,9 @@
"lfsput2", 0);
mutex_enter(vp->v_interlock);
}
+ uvm_pagelock(pg);
uvm_pageactivate(pg);
+ uvm_pageunlock(pg);
}
ap->a_offlo = blkeof;
if (ap->a_offhi > 0 && ap->a_offhi <= ap->a_offlo) {
diff -r d28e6d2f93e0 -r edeecf00bd9c sys/ufs/lfs/lfs_vfsops.c
--- a/sys/ufs/lfs/lfs_vfsops.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/ufs/lfs/lfs_vfsops.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: lfs_vfsops.c,v 1.366 2019/12/13 20:10:22 ad Exp $ */
+/* $NetBSD: lfs_vfsops.c,v 1.367 2019/12/31 22:42:51 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2003, 2007, 2007
@@ -61,7 +61,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lfs_vfsops.c,v 1.366 2019/12/13 20:10:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lfs_vfsops.c,v 1.367 2019/12/31 22:42:51 ad Exp $");
#if defined(_KERNEL_OPT)
#include "opt_lfs.h"
@@ -2054,7 +2054,9 @@
pgs[i]->flags |= PG_PAGEOUT;
uvm_pageout_start(1);
mutex_enter(vp->v_interlock);
+ uvm_pagelock(pgs[i]);
uvm_pageunwire(pgs[i]);
+ uvm_pageunlock(pgs[i]);
mutex_exit(vp->v_interlock);
}
}
@@ -2241,10 +2243,12 @@
if (pg->flags & PG_PAGEOUT)
uvm_pageout_done(1);
+ uvm_pagelock(pg);
if (pg->flags & PG_DELWRI) {
uvm_pageunwire(pg);
}
uvm_pageactivate(pg);
+ uvm_pageunlock(pg);
pg->flags &= ~(PG_CLEAN|PG_DELWRI|PG_PAGEOUT|PG_RELEASED);
DLOG((DLOG_PAGE, "pg[%d] = %p (vp %p off %" PRIx64 ")\n", i, pg,
vp, pg->offset));
diff -r d28e6d2f93e0 -r edeecf00bd9c sys/ufs/lfs/ulfs_inode.c
--- a/sys/ufs/lfs/ulfs_inode.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/ufs/lfs/ulfs_inode.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ulfs_inode.c,v 1.22 2019/12/13 20:10:22 ad Exp $ */
+/* $NetBSD: ulfs_inode.c,v 1.23 2019/12/31 22:42:51 ad Exp $ */
/* from NetBSD: ufs_inode.c,v 1.95 2015/06/13 14:56:45 hannken Exp */
/*
@@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ulfs_inode.c,v 1.22 2019/12/13 20:10:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ulfs_inode.c,v 1.23 2019/12/31 22:42:51 ad Exp $");
#if defined(_KERNEL_OPT)
#include "opt_lfs.h"
@@ -243,7 +243,9 @@
}
pgs[i]->flags &= ~PG_CLEAN;
}
+ uvm_pagelock(pgs[i]);
uvm_pageactivate(pgs[i]);
+ uvm_pageunlock(pgs[i]);
}
uvm_page_unbusy(pgs, npages);
mutex_exit(uobj->vmobjlock);
diff -r d28e6d2f93e0 -r edeecf00bd9c sys/ufs/ufs/ufs_inode.c
--- a/sys/ufs/ufs/ufs_inode.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/ufs/ufs/ufs_inode.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ufs_inode.c,v 1.106 2019/12/13 20:10:22 ad Exp $ */
+/* $NetBSD: ufs_inode.c,v 1.107 2019/12/31 22:42:51 ad Exp $ */
/*
* Copyright (c) 1991, 1993
@@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ufs_inode.c,v 1.106 2019/12/13 20:10:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ufs_inode.c,v 1.107 2019/12/31 22:42:51 ad Exp $");
#if defined(_KERNEL_OPT)
#include "opt_ffs.h"
@@ -279,7 +279,9 @@
}
pgs[i]->flags &= ~PG_CLEAN;
}
+ uvm_pagelock(pgs[i]);
uvm_pageactivate(pgs[i]);
+ uvm_pageunlock(pgs[i]);
}
uvm_page_unbusy(pgs, npages);
mutex_exit(uobj->vmobjlock);
diff -r d28e6d2f93e0 -r edeecf00bd9c sys/uvm/uvm.h
--- a/sys/uvm/uvm.h Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/uvm/uvm.h Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm.h,v 1.72 2019/12/27 13:19:24 ad Exp $ */
+/* $NetBSD: uvm.h,v 1.73 2019/12/31 22:42:51 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -87,6 +87,12 @@
/* entropy */
krndsource_t rs; /* entropy source */
+
+ /* uvmpdpol: queue of intended page status changes. */
+ struct vm_page **pdq; /* queue entries */
+ u_int pdqhead; /* current queue head */
+ u_int pdqtail; /* maximum number entries */
+ int pdqtime; /* last time queue cleared */
};
/*
diff -r d28e6d2f93e0 -r edeecf00bd9c sys/uvm/uvm_anon.c
--- a/sys/uvm/uvm_anon.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/uvm/uvm_anon.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_anon.c,v 1.69 2019/12/13 20:10:22 ad Exp $ */
+/* $NetBSD: uvm_anon.c,v 1.70 2019/12/31 22:42:51 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.69 2019/12/13 20:10:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.70 2019/12/31 22:42:51 ad Exp $");
#include "opt_uvmhist.h"
@@ -352,7 +352,9 @@
* Deactivate the page (to put it on a page queue).
*/
+ uvm_pagelock(pg);
uvm_pagedeactivate(pg);
+ uvm_pageunlock(pg);
if (pg->flags & PG_WANTED) {
pg->flags &= ~PG_WANTED;
wakeup(pg);
diff -r d28e6d2f93e0 -r edeecf00bd9c sys/uvm/uvm_aobj.c
--- a/sys/uvm/uvm_aobj.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/uvm/uvm_aobj.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_aobj.c,v 1.132 2019/12/15 21:11:35 ad Exp $ */
+/* $NetBSD: uvm_aobj.c,v 1.133 2019/12/31 22:42:51 ad Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
@@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.132 2019/12/15 21:11:35 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.133 2019/12/31 22:42:51 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_uvmhist.h"
@@ -738,7 +738,9 @@
case PGO_CLEANIT|PGO_DEACTIVATE:
case PGO_DEACTIVATE:
deactivate_it:
Home |
Main Index |
Thread Index |
Old Index