Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys Changes necessary to implement pre-zero'ing of pages in ...
details: https://anonhg.NetBSD.org/src/rev/278ae71766e2
branches: trunk
changeset: 485314:278ae71766e2
user: thorpej <thorpej%NetBSD.org@localhost>
date: Mon Apr 24 17:12:00 2000 +0000
description:
Changes necessary to implement pre-zero'ing of pages in the idle loop:
- Make page free lists have two actual queues: known-zero pages and
pages with unknown contents.
- Implement uvm_pageidlezero(). This function attempts to zero up to
the target number of pages until the target has been reached (currently
target is `all free pages') or until whichqs becomes non-zero (indicating
that a process is ready to run).
- Define a new hook for the pmap module for pre-zero'ing pages. This is
used to zero the pages using uncached access. This allows us to zero
as many pages as we want without polluting the cache.
In order to use this feature, each platform must add the appropropriate
glue in their idle loop.
diffstat:
sys/uvm/uvm.h | 6 +-
sys/uvm/uvm_extern.h | 7 +-
sys/uvm/uvm_map.c | 9 +-
sys/uvm/uvm_page.c | 164 ++++++++++++++++++++++++++++++++++++++++++++++----
sys/uvm/uvm_page.h | 11 +++-
sys/uvm/uvm_pglist.c | 39 +++++++----
sys/vm/pglist.h | 54 ++++++++++++++++-
sys/vm/vm_page.h | 7 +-
8 files changed, 257 insertions(+), 40 deletions(-)
diffs (truncated from 585 to 300 lines):
diff -r ceb49ae8e204 -r 278ae71766e2 sys/uvm/uvm.h
--- a/sys/uvm/uvm.h Mon Apr 24 17:09:56 2000 +0000
+++ b/sys/uvm/uvm.h Mon Apr 24 17:12:00 2000 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm.h,v 1.20 2000/04/10 02:20:06 chs Exp $ */
+/* $NetBSD: uvm.h,v 1.21 2000/04/24 17:12:00 thorpej Exp $ */
/*
*
@@ -77,13 +77,15 @@
struct uvm {
/* vm_page related parameters */
/* vm_page queues */
- struct pglist page_free[VM_NFREELIST]; /* unallocated pages */
+ struct pgfreelist page_free[VM_NFREELIST]; /* unallocated pages */
struct pglist page_active; /* allocated pages, in use */
struct pglist page_inactive_swp;/* pages inactive (reclaim or free) */
struct pglist page_inactive_obj;/* pages inactive (reclaim or free) */
simple_lock_data_t pageqlock; /* lock for active/inactive page q */
simple_lock_data_t fpageqlock; /* lock for free page q */
boolean_t page_init_done; /* TRUE if uvm_page_init() finished */
+ boolean_t page_idle_zero; /* TRUE if we should try to zero
+ pages in the idle loop */
/* page daemon trigger */
int pagedaemon; /* daemon sleeps on this */
struct proc *pagedaemon_proc; /* daemon's pid */
diff -r ceb49ae8e204 -r 278ae71766e2 sys/uvm/uvm_extern.h
--- a/sys/uvm/uvm_extern.h Mon Apr 24 17:09:56 2000 +0000
+++ b/sys/uvm/uvm_extern.h Mon Apr 24 17:12:00 2000 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_extern.h,v 1.39 2000/04/10 00:28:05 thorpej Exp $ */
+/* $NetBSD: uvm_extern.h,v 1.40 2000/04/24 17:12:00 thorpej Exp $ */
/*
*
@@ -176,6 +176,7 @@
int inactive; /* number of pages that we free'd but may want back */
int paging; /* number of pages in the process of being paged out */
int wired; /* number of wired pages */
+ int zeropages; /* number of zero'd pages */
int reserve_pagedaemon; /* number of pages reserved for pagedaemon */
int reserve_kernel; /* number of pages reserved for kernel */
@@ -212,6 +213,10 @@
int forks; /* forks */
int forks_ppwait; /* forks where parent waits */
int forks_sharevm; /* forks where vmspace is shared */
+ int pga_zerohit; /* pagealloc where zero wanted and zero
+ was available */
+ int pga_zeromiss; /* pagealloc where zero wanted and zero
+ not available */
/* fault subcounters */
int fltnoram; /* number of times fault was out of ram */
diff -r ceb49ae8e204 -r 278ae71766e2 sys/uvm/uvm_map.c
--- a/sys/uvm/uvm_map.c Mon Apr 24 17:09:56 2000 +0000
+++ b/sys/uvm/uvm_map.c Mon Apr 24 17:12:00 2000 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_map.c,v 1.72 2000/04/16 20:52:29 chs Exp $ */
+/* $NetBSD: uvm_map.c,v 1.73 2000/04/24 17:12:00 thorpej Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -3367,8 +3367,11 @@
}
/* cross-verify page queue */
- if (pg->pqflags & PQ_FREE)
- pgl = &uvm.page_free[uvm_page_lookup_freelist(pg)];
+ if (pg->pqflags & PQ_FREE) {
+ int fl = uvm_page_lookup_freelist(pg);
+ pgl = &uvm.page_free[fl].pgfl_queues[((pg)->flags & PG_ZERO) ?
+ PGFL_ZEROS : PGFL_UNKNOWN];
+ }
else if (pg->pqflags & PQ_INACTIVE)
pgl = (pg->pqflags & PQ_SWAPBACKED) ?
&uvm.page_inactive_swp : &uvm.page_inactive_obj;
diff -r ceb49ae8e204 -r 278ae71766e2 sys/uvm/uvm_page.c
--- a/sys/uvm/uvm_page.c Mon Apr 24 17:09:56 2000 +0000
+++ b/sys/uvm/uvm_page.c Mon Apr 24 17:12:00 2000 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_page.c,v 1.33 2000/04/10 00:28:05 thorpej Exp $ */
+/* $NetBSD: uvm_page.c,v 1.34 2000/04/24 17:12:01 thorpej Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -94,6 +94,12 @@
int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */
/*
+ * for testing the idle page zero loop.
+ */
+
+boolean_t vm_page_zero_enable = TRUE;
+
+/*
* local variables
*/
@@ -217,8 +223,10 @@
/*
* step 1: init the page queues and page queue locks
*/
- for (lcv = 0; lcv < VM_NFREELIST; lcv++)
- TAILQ_INIT(&uvm.page_free[lcv]);
+ for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
+ for (i = 0; i < PGFL_NQUEUES; i++)
+ TAILQ_INIT(&uvm.page_free[lcv].pgfl_queues[i]);
+ }
TAILQ_INIT(&uvm.page_active);
TAILQ_INIT(&uvm.page_inactive_swp);
TAILQ_INIT(&uvm.page_inactive_obj);
@@ -334,6 +342,16 @@
uvmexp.reserve_kernel = 5;
/*
+ * step 8: determine if we should zero pages in the idle
+ * loop.
+ *
+ * XXXJRT - might consider zero'ing up to the target *now*,
+ * but that could take an awfully long time if you
+ * have a lot of memory.
+ */
+ uvm.page_idle_zero = vm_page_zero_enable;
+
+ /*
* done!
*/
@@ -848,6 +866,12 @@
* => only one of obj or anon can be non-null
* => caller must activate/deactivate page if it is not wired.
* => free_list is ignored if strat == UVM_PGA_STRAT_NORMAL.
+ * => policy decision: it is more important to pull a page off of the
+ * appropriate priority free list than it is to get a zero'd or
+ * unknown contents page. This is because we live with the
+ * consequences of a bad free list decision for the entire
+ * lifetime of the page, e.g. if the page comes from memory that
+ * is slower to access.
*/
struct vm_page *
@@ -858,9 +882,10 @@
struct vm_anon *anon;
int strat, free_list;
{
- int lcv, s;
+ int lcv, try1, try2, s, zeroit = 0;
struct vm_page *pg;
struct pglist *freeq;
+ struct pgfreelist *pgfl;
boolean_t use_reserve;
#ifdef DIAGNOSTIC
@@ -896,13 +921,32 @@
!(use_reserve && curproc == uvm.pagedaemon_proc)))
goto fail;
+#if PGFL_NQUEUES != 2
+#error uvm_pagealloc_strat needs to be updated
+#endif
+
+ /*
+ * If we want a zero'd page, try the ZEROS queue first, otherwise
+ * we try the UNKNOWN queue first.
+ */
+ if (flags & UVM_PGA_ZERO) {
+ try1 = PGFL_ZEROS;
+ try2 = PGFL_UNKNOWN;
+ } else {
+ try1 = PGFL_UNKNOWN;
+ try2 = PGFL_ZEROS;
+ }
+
again:
switch (strat) {
case UVM_PGA_STRAT_NORMAL:
/* Check all freelists in descending priority order. */
for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
- freeq = &uvm.page_free[lcv];
- if ((pg = freeq->tqh_first) != NULL)
+ pgfl = &uvm.page_free[lcv];
+ if ((pg = TAILQ_FIRST((freeq =
+ &pgfl->pgfl_queues[try1]))) != NULL ||
+ (pg = TAILQ_FIRST((freeq =
+ &pgfl->pgfl_queues[try2]))) != NULL)
goto gotit;
}
@@ -917,8 +961,11 @@
panic("uvm_pagealloc_strat: bad free list %d",
free_list);
#endif
- freeq = &uvm.page_free[free_list];
- if ((pg = freeq->tqh_first) != NULL)
+ pgfl = &uvm.page_free[free_list];
+ if ((pg = TAILQ_FIRST((freeq =
+ &pgfl->pgfl_queues[try1]))) != NULL ||
+ (pg = TAILQ_FIRST((freeq =
+ &pgfl->pgfl_queues[try2]))) != NULL)
goto gotit;
/* Fall back, if possible. */
@@ -939,6 +986,24 @@
TAILQ_REMOVE(freeq, pg, pageq);
uvmexp.free--;
+ /* update zero'd page count */
+ if (pg->flags & PG_ZERO)
+ uvmexp.zeropages--;
+
+ /*
+ * update allocation statistics and remember if we have to
+ * zero the page
+ */
+ if (flags & UVM_PGA_ZERO) {
+ if (pg->flags & PG_ZERO) {
+ uvmexp.pga_zerohit++;
+ zeroit = 0;
+ } else {
+ uvmexp.pga_zeromiss++;
+ zeroit = 1;
+ }
+ }
+
uvm_unlock_fpageq(s); /* unlock free page queue */
pg->offset = off;
@@ -963,11 +1028,12 @@
if (flags & UVM_PGA_ZERO) {
/*
- * This is an inline of uvm_pagezero(); this will change
- * when we have pre-zero'd pages.
+ * A zero'd page is not clean. If we got a page not already
+ * zero'd, then we have to zero it ourselves.
*/
pg->flags &= ~PG_CLEAN;
- pmap_zero_page(VM_PAGE_TO_PHYS(pg));
+ if (zeroit)
+ pmap_zero_page(VM_PAGE_TO_PHYS(pg));
}
return(pg);
@@ -1105,8 +1171,7 @@
/*
* if the page was wired, unwire it now.
*/
- if (pg->wire_count)
- {
+ if (pg->wire_count) {
pg->wire_count = 0;
uvmexp.wired--;
}
@@ -1115,9 +1180,11 @@
* and put on free queue
*/
+ pg->flags &= ~PG_ZERO;
+
s = uvm_lock_fpageq();
- TAILQ_INSERT_TAIL(&uvm.page_free[uvm_page_lookup_freelist(pg)],
- pg, pageq);
+ TAILQ_INSERT_TAIL(&uvm.page_free[
+ uvm_page_lookup_freelist(pg)].pgfl_queues[PGFL_UNKNOWN], pg, pageq);
pg->pqflags = PQ_FREE;
#ifdef DEBUG
pg->uobject = (void *)0xdeadbeef;
@@ -1125,6 +1192,10 @@
pg->uanon = (void *)0xdeadbeef;
#endif
uvmexp.free++;
+
+ if (uvmexp.zeropages < UVM_PAGEZERO_TARGET)
+ uvm.page_idle_zero = vm_page_zero_enable;
+
uvm_unlock_fpageq(s);
}
@@ -1166,3 +1237,66 @@
return;
}
#endif
+
+/*
+ * uvm_pageidlezero: zero free pages while the system is idle.
+ *
+ * => we do at least one iteration per call, if we are below the target.
+ * => we loop until we either reach the target or whichqs indicates that
+ * there is a process ready to run.
+ */
+void
+uvm_pageidlezero()
+{
+ struct vm_page *pg;
+ struct pgfreelist *pgfl;
+ int free_list, s;
+
+ do {
+ s = uvm_lock_fpageq();
Home |
Main Index |
Thread Index |
Old Index