Subject: multiple free list support for UVM, take 2
To: None <tech-kern@NetBSD.ORG>
From: Jason Thorpe <thorpej@nas.nasa.gov>
List: tech-kern
Date: 06/27/1998 23:49:07
This addresses the concern of adding more stuff to the vm_page_t; there
is now a (inline) function to lookup the free list for a given page.
I don't suspect this will be much overhead.
Jason R. Thorpe thorpej@nas.nasa.gov
NASA Ames Research Center Home: +1 408 866 1912
NAS: M/S 258-5 Work: +1 650 604 0935
Moffett Field, CA 94035 Pager: +1 650 428 6939
Index: uvm/uvm.h
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm.h,v
retrieving revision 1.8
diff -c -r1.8 uvm.h
*** uvm.h 1998/05/20 01:32:29 1.8
--- uvm.h 1998/06/28 07:01:00
***************
*** 67,72 ****
--- 67,77 ----
#include <uvm/uvm_swap.h>
/*
+ * pull in VM_NFREELIST
+ */
+ #include <machine/vmparam.h>
+
+ /*
* uvm structure (vm global state: collected in one structure for ease
* of reference...)
*/
***************
*** 74,80 ****
struct uvm {
/* vm_page related parameters */
/* vm_page queues */
! struct pglist page_free; /* unallocated pages */
struct pglist page_active; /* allocated pages, in use */
struct pglist page_inactive_swp;/* pages inactive (reclaim or free) */
struct pglist page_inactive_obj;/* pages inactive (reclaim or free) */
--- 79,85 ----
struct uvm {
/* vm_page related parameters */
/* vm_page queues */
! struct pglist page_free[VM_NFREELIST]; /* unallocated pages */
struct pglist page_active; /* allocated pages, in use */
struct pglist page_inactive_swp;/* pages inactive (reclaim or free) */
struct pglist page_inactive_obj;/* pages inactive (reclaim or free) */
Index: uvm/uvm_extern.h
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_extern.h,v
retrieving revision 1.13
diff -c -r1.13 uvm_extern.h
*** uvm_extern.h 1998/05/09 15:04:40 1.13
--- uvm_extern.h 1998/06/28 07:01:00
***************
*** 128,133 ****
--- 128,140 ----
#define UVM_KMF_TRYLOCK UVM_FLAG_TRYLOCK /* try locking only */
/*
+ * the following defines the strategies for uvm_pagealloc_strat()
+ */
+ #define UVM_PGA_STRAT_NORMAL 0 /* high -> low free list walk */
+ #define UVM_PGA_STRAT_ONLY 1 /* only specified free list */
+ #define UVM_PGA_STRAT_FALLBACK 2 /* ONLY falls back on NORMAL */
+
+ /*
* structures
*/
***************
*** 341,352 ****
caddr_t, vm_offset_t));
/* uvm_page.c */
! struct vm_page *uvm_pagealloc __P((struct uvm_object *, vm_offset_t,
! struct vm_anon *));
void uvm_pagerealloc __P((struct vm_page *,
struct uvm_object *, vm_offset_t));
void uvm_page_physload __P((vm_offset_t, vm_offset_t,
! vm_offset_t, vm_offset_t));
void uvm_setpagesize __P((void));
/* uvm_pdaemon.c */
--- 348,361 ----
caddr_t, vm_offset_t));
/* uvm_page.c */
! struct vm_page *uvm_pagealloc_strat __P((struct uvm_object *,
! vm_offset_t, struct vm_anon *, int, int));
! #define uvm_pagealloc(obj, off, anon) \
! uvm_pagealloc_strat((obj), (off), (anon), UVM_PGA_STRAT_NORMAL, 0)
void uvm_pagerealloc __P((struct vm_page *,
struct uvm_object *, vm_offset_t));
void uvm_page_physload __P((vm_offset_t, vm_offset_t,
! vm_offset_t, vm_offset_t, int));
void uvm_setpagesize __P((void));
/* uvm_pdaemon.c */
Index: uvm/uvm_map.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_map.c,v
retrieving revision 1.20
diff -c -r1.20 uvm_map.c
*** uvm_map.c 1998/05/22 02:01:54 1.20
--- uvm_map.c 1998/06/28 07:01:01
***************
*** 3251,3257 ****
/* cross-verify page queue */
if (pg->pqflags & PQ_FREE)
! pgl = &uvm.page_free;
else if (pg->pqflags & PQ_INACTIVE)
pgl = (pg->pqflags & PQ_SWAPBACKED) ?
&uvm.page_inactive_swp : &uvm.page_inactive_obj;
--- 3251,3257 ----
/* cross-verify page queue */
if (pg->pqflags & PQ_FREE)
! pgl = &uvm.page_free[uvm_page_lookup_freelist(pg)];
else if (pg->pqflags & PQ_INACTIVE)
pgl = (pg->pqflags & PQ_SWAPBACKED) ?
&uvm.page_inactive_swp : &uvm.page_inactive_obj;
Index: uvm/uvm_page.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_page.c,v
retrieving revision 1.11
diff -c -r1.11 uvm_page.c
*** uvm_page.c 1998/05/28 15:31:31 1.11
--- uvm_page.c 1998/06/28 07:01:01
***************
*** 224,230 ****
* step 1: init the page queues and page queue locks
*/
! TAILQ_INIT(&uvm.page_free);
TAILQ_INIT(&uvm.page_active);
TAILQ_INIT(&uvm.page_inactive_swp);
TAILQ_INIT(&uvm.page_inactive_obj);
--- 224,231 ----
* step 1: init the page queues and page queue locks
*/
! for (lcv = 0; lcv < VM_NFREELIST; lcv++)
! TAILQ_INIT(&uvm.page_free[lcv]);
TAILQ_INIT(&uvm.page_active);
TAILQ_INIT(&uvm.page_inactive_swp);
TAILQ_INIT(&uvm.page_inactive_obj);
***************
*** 548,555 ****
*/
void
! uvm_page_physload(start, end, avail_start, avail_end)
vm_offset_t start, end, avail_start, avail_end;
{
int preload, lcv, npages;
struct vm_page *pgs;
--- 549,557 ----
*/
void
! uvm_page_physload(start, end, avail_start, avail_end, free_list)
vm_offset_t start, end, avail_start, avail_end;
+ int free_list;
{
int preload, lcv, npages;
struct vm_page *pgs;
***************
*** 597,607 ****
printf("\tignoring 0x%lx -> 0x%lx\n", start, end);
return;
}
! /* zero data, init phys_addr, and free pages */
bzero(pgs, sizeof(struct vm_page) * npages);
for (lcv = 0, paddr = ptoa(start) ;
lcv < npages ; lcv++, paddr += PAGE_SIZE) {
pgs[lcv].phys_addr = paddr;
if (atop(paddr) >= avail_start &&
atop(paddr) <= avail_end)
uvm_pagefree(&pgs[lcv]);
--- 599,610 ----
printf("\tignoring 0x%lx -> 0x%lx\n", start, end);
return;
}
! /* zero data, init phys_addr and free_list, and free pages */
bzero(pgs, sizeof(struct vm_page) * npages);
for (lcv = 0, paddr = ptoa(start) ;
lcv < npages ; lcv++, paddr += PAGE_SIZE) {
pgs[lcv].phys_addr = paddr;
+ pgs[lcv].free_list = free_list;
if (atop(paddr) >= avail_start &&
atop(paddr) <= avail_end)
uvm_pagefree(&pgs[lcv]);
***************
*** 673,678 ****
--- 676,682 ----
ps->pgs = pgs;
ps->lastpg = pgs + npages - 1;
}
+ ps->free_list = free_list;
vm_nphysseg++;
/*
***************
*** 794,800 ****
#endif
/*
! * uvm_pagealloc: allocate vm_page.
*
* => return null if no pages free
* => wake up pagedaemon if number of free pages drops below low water mark
--- 798,804 ----
#endif
/*
! * uvm_pagealloc_strat: allocate vm_page from a particular free list.
*
* => return null if no pages free
* => wake up pagedaemon if number of free pages drops below low water mark
***************
*** 802,817 ****
* => if anon != NULL, anon must be locked (to put in anon)
* => only one of obj or anon can be non-null
* => caller must activate/deactivate page if it is not wired.
*/
struct vm_page *
! uvm_pagealloc(obj, off, anon)
struct uvm_object *obj;
vm_offset_t off;
struct vm_anon *anon;
{
! int s;
struct vm_page *pg;
#ifdef DIAGNOSTIC
/* sanity check */
--- 806,824 ----
* => if anon != NULL, anon must be locked (to put in anon)
* => only one of obj or anon can be non-null
* => caller must activate/deactivate page if it is not wired.
+ * => free_list is ignored if strat == UVM_PGA_STRAT_NORMAL.
*/
struct vm_page *
! uvm_pagealloc_strat(obj, off, anon, strat, free_list)
struct uvm_object *obj;
vm_offset_t off;
struct vm_anon *anon;
+ int strat, free_list;
{
! int lcv, s;
struct vm_page *pg;
+ struct pglist *freeq;
#ifdef DIAGNOSTIC
/* sanity check */
***************
*** 841,858 ****
* the requestor isn't the pagedaemon.
*/
! pg = uvm.page_free.tqh_first;
! if (pg == NULL ||
! (uvmexp.free <= uvmexp.reserve_kernel &&
!(obj && obj->uo_refs == UVM_OBJ_KERN)) ||
(uvmexp.free <= uvmexp.reserve_pagedaemon &&
! !(obj == uvmexp.kmem_object && curproc == uvm.pagedaemon_proc))) {
! uvm_unlock_fpageq();
! splx(s);
! return(NULL);
}
! TAILQ_REMOVE(&uvm.page_free, pg, pageq);
uvmexp.free--;
uvm_unlock_fpageq(); /* unlock free page queue */
--- 848,900 ----
* the requestor isn't the pagedaemon.
*/
! if ((uvmexp.free <= uvmexp.reserve_kernel &&
!(obj && obj->uo_refs == UVM_OBJ_KERN)) ||
(uvmexp.free <= uvmexp.reserve_pagedaemon &&
! !(obj == uvmexp.kmem_object && curproc == uvm.pagedaemon_proc)))
! goto fail;
!
! again:
! switch (strat) {
! case UVM_PGA_STRAT_NORMAL:
! /* Check all freelists in descending priority order. */
! for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
! freeq = &uvm.page_free[lcv];
! if ((pg = freeq->tqh_first) != NULL)
! goto gotit;
! }
!
! /* No pages free! */
! goto fail;
!
! case UVM_PGA_STRAT_ONLY:
! case UVM_PGA_STRAT_FALLBACK:
! /* Attempt to allocate from the specified free list. */
! #ifdef DIAGNOSTIC
! if (free_list >= VM_NFREELIST || free_list < 0)
! panic("uvm_pagealloc_strat: bad free list %d",
! free_list);
! #endif
! freeq = &uvm.page_free[free_list];
! if ((pg = freeq->tqh_first) != NULL)
! goto gotit;
!
! /* Fall back, if possible. */
! if (strat == UVM_PGA_STRAT_FALLBACK) {
! strat = UVM_PGA_STRAT_NORMAL;
! goto again;
! }
!
! /* No pages free! */
! goto fail;
!
! default:
! panic("uvm_pagealloc_strat: bad strat %d", strat);
! /* NOTREACHED */
}
! gotit:
! TAILQ_REMOVE(freeq, pg, pageq);
uvmexp.free--;
uvm_unlock_fpageq(); /* unlock free page queue */
***************
*** 879,884 ****
--- 921,931 ----
UVM_PAGE_OWN(pg, "new alloc");
return(pg);
+
+ fail:
+ uvm_unlock_fpageq();
+ splx(s);
+ return (NULL);
}
/*
***************
*** 1021,1027 ****
s = splimp();
uvm_lock_fpageq();
! TAILQ_INSERT_TAIL(&uvm.page_free, pg, pageq);
pg->pqflags = PQ_FREE;
#ifdef DEBUG
pg->uobject = (void *)0xdeadbeef;
--- 1068,1075 ----
s = splimp();
uvm_lock_fpageq();
! TAILQ_INSERT_TAIL(&uvm.page_free[uvm_page_lookup_freelist(pg)],
! pg, pageq);
pg->pqflags = PQ_FREE;
#ifdef DEBUG
pg->uobject = (void *)0xdeadbeef;
Index: uvm/uvm_page.h
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_page.h,v
retrieving revision 1.8
diff -c -r1.8 uvm_page.h
*** uvm_page.h 1998/05/28 15:31:31 1.8
--- uvm_page.h 1998/06/28 07:01:01
***************
*** 127,130 ****
--- 127,132 ----
PAGE_INLINE void uvm_pagewire __P((struct vm_page *));
PAGE_INLINE void uvm_pagezero __P((struct vm_page *));
+ PAGE_INLINE int uvm_page_lookup_freelist __P((struct vm_page *));
+
#endif /* _UVM_UVM_PAGE_H_ */
Index: uvm/uvm_page_i.h
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_page_i.h,v
retrieving revision 1.6
diff -c -r1.6 uvm_page_i.h
*** uvm_page_i.h 1998/03/22 21:29:30 1.6
--- uvm_page_i.h 1998/06/28 07:01:02
***************
*** 269,274 ****
--- 269,292 ----
pmap_copy_page(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst));
}
+ /*
+ * uvm_page_lookup_freelist: look up the free list for the specified page
+ */
+
+ PAGE_INLINE int
+ uvm_page_lookup_freelist(pg)
+ struct vm_page *pg;
+ {
+ int lcv;
+
+ lcv = vm_physseg_find(atop(VM_PAGE_TO_PHYS(pg)), NULL);
+ #ifdef DIAGNOSTIC
+ if (lcv == -1)
+ panic("uvm_page_lookup_freelist: unable to locate physseg");
+ #endif
+ return (vm_physmem[lcv].free_list);
+ }
+
#endif /* defined(UVM_PAGE_INLINE) || defined(UVM_PAGE) */
#endif /* _UVM_UVM_PAGE_I_H_ */
Index: uvm/uvm_pglist.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_pglist.c,v
retrieving revision 1.4
diff -c -r1.4 uvm_pglist.c
*** uvm_pglist.c 1998/05/05 20:51:07 1.4
--- uvm_pglist.c 1998/06/28 07:01:02
***************
*** 100,106 ****
vm_offset_t try, idxpa, lastidxpa;
int psi;
struct vm_page *pgs;
! int s, tryidx, idx, end, error;
vm_page_t m;
u_long pagemask;
#ifdef DEBUG
--- 100,106 ----
vm_offset_t try, idxpa, lastidxpa;
int psi;
struct vm_page *pgs;
! int s, tryidx, idx, end, error, free_list;
vm_page_t m;
u_long pagemask;
#ifdef DEBUG
***************
*** 140,146 ****
uvm_lock_fpageq(); /* lock free page queue */
/* Are there even any free pages? */
! if (uvm.page_free.tqh_first == NULL)
goto out;
for (;; try += alignment) {
--- 140,149 ----
uvm_lock_fpageq(); /* lock free page queue */
/* Are there even any free pages? */
! for (idx = 0; idx < VM_NFREELIST; idx++)
! if (uvm.page_free[idx].tqh_first != NULL)
! break;
! if (idx == VM_NFREELIST)
goto out;
for (;; try += alignment) {
***************
*** 210,225 ****
idx = tryidx;
while (idx < end) {
m = &pgs[idx];
#ifdef DEBUG
! for (tp = uvm.page_free.tqh_first; tp != NULL;
! tp = tp->pageq.tqe_next) {
if (tp == m)
break;
}
if (tp == NULL)
panic("uvm_pglistalloc: page not on freelist");
#endif
! TAILQ_REMOVE(&uvm.page_free, m, pageq);
uvmexp.free--;
m->flags = PG_CLEAN;
m->pqflags = 0;
--- 213,229 ----
idx = tryidx;
while (idx < end) {
m = &pgs[idx];
+ free_list = uvm_page_lookup_freelist(m);
#ifdef DEBUG
! for (tp = uvm.page_free[free_list].tqh_first;
! tp != NULL; tp = tp->pageq.tqe_next) {
if (tp == m)
break;
}
if (tp == NULL)
panic("uvm_pglistalloc: page not on freelist");
#endif
! TAILQ_REMOVE(&uvm.page_free[free_list], m, pageq);
uvmexp.free--;
m->flags = PG_CLEAN;
m->pqflags = 0;
***************
*** 277,283 ****
#endif
TAILQ_REMOVE(list, m, pageq);
m->pqflags = PQ_FREE;
! TAILQ_INSERT_TAIL(&uvm.page_free, m, pageq);
uvmexp.free++;
STAT_DECR(uvm_pglistalloc_npages);
}
--- 281,288 ----
#endif
TAILQ_REMOVE(list, m, pageq);
m->pqflags = PQ_FREE;
! TAILQ_INSERT_TAIL(&uvm.page_free[uvm_page_lookup_freelist(m)],
! m, pageq);
uvmexp.free++;
STAT_DECR(uvm_pglistalloc_npages);
}
Index: vm/vm_page.h
===================================================================
RCS file: /cvsroot/src/sys/vm/vm_page.h,v
retrieving revision 1.26
diff -c -r1.26 vm_page.h
*** vm_page.h 1998/03/12 06:25:53 1.26
--- vm_page.h 1998/06/28 07:01:02
***************
*** 243,248 ****
--- 243,251 ----
vm_offset_t end; /* (PF# of last page in segment) + 1 */
vm_offset_t avail_start; /* PF# of first free page in segment */
vm_offset_t avail_end; /* (PF# of last free page in segment) +1 */
+ #if defined(UVM)
+ int free_list; /* which free list they belong on */
+ #endif
struct vm_page *pgs; /* vm_page structures (from start) */
struct vm_page *lastpg; /* vm_page structure for end */
struct pmap_physseg pmseg; /* pmap specific (MD) data */