Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/uebayasi-xip]: src/sys Resurrect PGO_ZERO support.
details: https://anonhg.NetBSD.org/src/rev/855dbfc1d3bf
branches: uebayasi-xip
changeset: 751896:855dbfc1d3bf
user: uebayasi <uebayasi%NetBSD.org@localhost>
date: Sun Nov 21 12:42:59 2010 +0000
description:
Resurrect PGO_ZERO support.
When vnode pager encounters hole pages in XIP'ed vnodes, it fills
page slots with PGO_ZERO and returns them back to the caller (fault
handler). Fault handlers are responsible to check page slots and
redirect PGO_ZERO to the single "zero page" allocated by calling
uvm_page_zeropage_alloc(9).
The zero page is wired, read-only (PG_RDONLY) page. It's shared
by multiple vnodes, it has no single owner.
XIP'ed vnodes are supposed to be "stable" during I/O (unlocked).
Because XIP'ed mounts are always read-only. There's no chance to
change mappings of XIP'ed vnodes and their XIP'ed pages. Thus the
cached uobj is reused after pgo_get() for PGO_ZERO.
(Do we need a new concept of "read-only UVM object"?)
diffstat:
sys/miscfs/genfs/genfs_io.c | 27 +++++++++++++++------
sys/uvm/uvm_bio.c | 44 ++++++++++++++++++++++++++++-------
sys/uvm/uvm_fault.c | 55 ++++++++++++++++++++++++++++++++++----------
sys/uvm/uvm_pager.h | 4 ++-
4 files changed, 99 insertions(+), 31 deletions(-)
diffs (truncated from 426 to 300 lines):
diff -r 40face4afe43 -r 855dbfc1d3bf sys/miscfs/genfs/genfs_io.c
--- a/sys/miscfs/genfs/genfs_io.c Sun Nov 21 12:14:15 2010 +0000
+++ b/sys/miscfs/genfs/genfs_io.c Sun Nov 21 12:42:59 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: genfs_io.c,v 1.36.2.59 2010/11/21 07:41:49 uebayasi Exp $ */
+/* $NetBSD: genfs_io.c,v 1.36.2.60 2010/11/21 12:42:59 uebayasi Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.36.2.59 2010/11/21 07:41:49 uebayasi Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.36.2.60 2010/11/21 12:42:59 uebayasi Exp $");
#include "opt_xip.h"
@@ -652,13 +652,11 @@
*/
if (blkno == (daddr_t)-1) {
- if (!xip) {
int holepages = (round_page(offset + iobytes) -
trunc_page(offset)) >> PAGE_SHIFT;
UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0);
- KASSERT(!xip);
-
+ if (!xip) {
sawhole = true;
memset((char *)kva + (offset - startoffset), 0,
iobytes);
@@ -673,7 +671,11 @@
}
}
} else {
- panic("XIP hole page is not supported yet");
+ for (i = 0; i < holepages; i++) {
+ pgs[ridx + pidx + i] = PGO_ZERO;
+ }
+ UVMHIST_LOG(ubchist, "xip HOLE pgs %d .. %d",
+ pidx, pidx + holepages - 1, 0, 0);
}
continue;
}
@@ -899,6 +901,7 @@
for (i = ridx; i < ridx + npages; i++) {
struct vm_page *pg = pgs[i];
+ if (pg != PGO_ZERO) {
KASSERT(pg != NULL);
KASSERT((pg->flags & PG_RDONLY) != 0);
KASSERT((pg->flags & PG_BUSY) != 0);
@@ -908,10 +911,18 @@
/*
* XXXUEBS
- * Actually this is not necessary, because device pages are
- * "stateless", and they have no owner.
+ * Actually this is not necessary, because device
+ * pages are "stateless", and they have no owner.
*/
pg->uobject = &vp->v_uobj;
+ } else {
+ /*
+ * XIP hole pages are passed as a magic pointer
+ * back to fault handlers. Fault handlers are
+ * respoinsible to check it and redirect the VA to
+ * a single "zero page".
+ */
+ }
}
} /* xip */
mutex_exit(&uobj->vmobjlock);
diff -r 40face4afe43 -r 855dbfc1d3bf sys/uvm/uvm_bio.c
--- a/sys/uvm/uvm_bio.c Sun Nov 21 12:14:15 2010 +0000
+++ b/sys/uvm/uvm_bio.c Sun Nov 21 12:42:59 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_bio.c,v 1.68.2.12 2010/11/04 08:47:38 uebayasi Exp $ */
+/* $NetBSD: uvm_bio.c,v 1.68.2.13 2010/11/21 12:42:59 uebayasi Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.68.2.12 2010/11/04 08:47:38 uebayasi Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.68.2.13 2010/11/21 12:42:59 uebayasi Exp $");
#include "opt_uvmhist.h"
#include "opt_ubc.h"
@@ -224,16 +224,20 @@
static inline int
ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap,
- struct vm_page *pg, vm_prot_t prot, vm_prot_t access_type, vaddr_t va)
+ struct uvm_object *uobj, struct vm_page *pg, vm_prot_t prot,
+ vm_prot_t access_type, vaddr_t va)
{
- struct uvm_object *uobj;
vm_prot_t mask;
int error;
bool rdonly;
- uobj = pg->uobject;
+ UVMHIST_FUNC("ubc_fault_page"); UVMHIST_CALLED(ubchist);
+
+ KASSERT(pg != NULL);
+ KASSERT(pg == PGO_ZERO || uobj == pg->uobject);
KASSERT(mutex_owned(&uobj->vmobjlock));
+ if (__predict_true(pg != PGO_ZERO)) {
if (pg->flags & PG_WANTED) {
wakeup(pg);
}
@@ -264,17 +268,29 @@
pg = newpg;
}
}
+ }
/*
* Note that a page whose backing store is partially allocated
* is marked as PG_RDONLY.
*/
- KASSERT((pg->flags & PG_RDONLY) == 0 ||
+ KASSERT(pg == PGO_ZERO ||
+ (pg->flags & PG_RDONLY) == 0 ||
(access_type & VM_PROT_WRITE) == 0 ||
pg->offset < umap->writeoff ||
pg->offset + PAGE_SIZE > umap->writeoff + umap->writelen);
+ if (__predict_false(pg == PGO_ZERO)) {
+ UVMHIST_LOG(ubchist, "replacing PGO_ZERO with zeropage",0,0,0,0);
+ pg = uvm_page_zeropage_alloc();
+ UVMHIST_LOG(ubchist,
+ "PGO_ZERO replaced with pg %p (phys_addr=0x%lx)",
+ pg, VM_PAGE_TO_PHYS(pg), 0, 0);
+ KASSERT(pg != NULL);
+ KASSERT((pg->flags & PG_RDONLY) != 0);
+ }
+
rdonly = ((access_type & VM_PROT_WRITE) == 0 &&
(pg->flags & PG_RDONLY) != 0) ||
UVM_OBJ_NEEDS_WRITEFAULT(uobj);
@@ -283,6 +299,7 @@
error = pmap_enter(ufi->orig_map->pmap, va, VM_PAGE_TO_PHYS(pg),
prot & mask, PMAP_CANFAIL | (access_type & mask));
+ if (__predict_true(pg != uvm_page_zeropage)) {
if (__predict_true((pg->flags & PG_DEVICE) == 0)) {
mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
@@ -290,6 +307,7 @@
}
pg->flags &= ~(PG_BUSY|PG_WANTED);
UVM_PAGE_OWN(pg, NULL);
+ }
return error;
}
@@ -302,7 +320,7 @@
ubc_fault(struct uvm_faultinfo *ufi, vaddr_t ign1, struct vm_page **ign2,
int ign3, int ign4, vm_prot_t access_type, int flags)
{
- struct uvm_object *uobj;
+ struct uvm_object *uobj, *ouobj;
struct ubc_map *umap;
vaddr_t va, eva, ubc_offset, slot_offset;
struct vm_page *pgs[ubc_winsize >> PAGE_SHIFT];
@@ -353,7 +371,7 @@
#endif
/* no umap locking needed since we have a ref on the umap */
- uobj = umap->uobj;
+ ouobj = uobj = umap->uobj;
if ((access_type & VM_PROT_WRITE) == 0) {
npages = (ubc_winsize - slot_offset) >> PAGE_SHIFT;
@@ -422,6 +440,7 @@
if (pg == NULL || pg == PGO_DONTCARE) {
continue;
}
+ if (__predict_true(pg != PGO_ZERO)) {
if (__predict_false(pg->uobject != uobj)) {
/* Check for the first iteration and error cases. */
if (uobj != NULL) {
@@ -432,7 +451,14 @@
uobj = pg->uobject;
mutex_enter(&uobj->vmobjlock);
}
- error = ubc_fault_page(ufi, umap, pg, prot, access_type, va);
+ } else {
+ if (__predict_false(uobj != ouobj)) {
+ uobj = ouobj;
+ mutex_enter(&uobj->vmobjlock);
+ }
+ }
+ KASSERT(pg == PGO_ZERO || uobj == pg->uobject);
+ error = ubc_fault_page(ufi, umap, uobj, pg, prot, access_type, va);
if (error) {
/*
* Flush (there might be pages entered), drop the lock,
diff -r 40face4afe43 -r 855dbfc1d3bf sys/uvm/uvm_fault.c
--- a/sys/uvm/uvm_fault.c Sun Nov 21 12:14:15 2010 +0000
+++ b/sys/uvm/uvm_fault.c Sun Nov 21 12:42:59 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_fault.c,v 1.166.2.24 2010/11/19 01:44:47 uebayasi Exp $ */
+/* $NetBSD: uvm_fault.c,v 1.166.2.25 2010/11/21 12:42:59 uebayasi Exp $ */
/*
*
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.166.2.24 2010/11/19 01:44:47 uebayasi Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.166.2.25 2010/11/21 12:42:59 uebayasi Exp $");
#include "opt_uvmhist.h"
#include "opt_xip.h"
@@ -564,7 +564,7 @@
opg = oanon->an_page;
KASSERT(opg != NULL);
KASSERT(opg->uobject == NULL || opg->loan_count > 0);
- } else if (uobjpage != PGO_DONTCARE) {
+ } else if (uobjpage != PGO_DONTCARE && uobjpage != PGO_ZERO) {
/* object-backed COW */
opg = uobjpage;
} else {
@@ -579,7 +579,7 @@
KASSERT(amap != NULL);
KASSERT(uobjpage != NULL);
- KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0);
+ KASSERT(uobjpage == PGO_DONTCARE || uobjpage == PGO_ZERO || (uobjpage->flags & PG_BUSY) != 0);
KASSERT(mutex_owned(&amap->am_l));
KASSERT(oanon == NULL || mutex_owned(&oanon->an_lock));
KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
@@ -1654,7 +1654,7 @@
*/
KASSERT(amap == NULL || mutex_owned(&amap->am_l));
KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
- KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
+ KASSERT(uobj == NULL || uobjpage == PGO_ZERO || (uobjpage->flags & PG_BUSY) != 0);
/*
* notes:
@@ -1664,8 +1664,8 @@
* - at this point uobjpage could be PG_WANTED (handle later)
*/
- KASSERT(uobj == NULL || uobj == uobjpage->uobject);
- KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
+ KASSERT(uobj == NULL || uobjpage == PGO_ZERO || (uobjpage->flags & PG_DEVICE) != 0 || uobj == uobjpage->uobject);
+ KASSERT(uobj == NULL || uobjpage == PGO_ZERO || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
(uobjpage->flags & PG_CLEAN) != 0);
if (flt->promote == false) {
@@ -1865,13 +1865,17 @@
/* locked: pg */
+ KASSERT(pg != 0);
+
+ if (pg != PGO_ZERO) {
KASSERT((pg->flags & PG_BUSY) != 0);
- if ((pg->flags & PG_DEVICE) == 0) {
+ if (pg != PGO_ZERO && (pg->flags & PG_DEVICE) == 0) {
mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
}
+ }
/*
* re-verify the state of the world by first trying to relock
@@ -1882,8 +1886,12 @@
if (locked && amap)
amap_lock(amap);
+ if (pg != PGO_ZERO) {
/* might be changed */
uobj = pg->uobject;
+ } else {
+ /* XIP hole page is shared, and it has no single owner */
+ }
mutex_enter(&uobj->vmobjlock);
@@ -1896,7 +1904,7 @@
* we unlock and clean up.
*/
- if ((pg->flags & PG_RELEASED) != 0 ||
+ if ((pg != PGO_ZERO && (pg->flags & PG_RELEASED) != 0) ||
Home |
Main Index |
Thread Index |
Old Index