Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/uvm - Rename VM_PGCOLOR_BUCKET() to VM_PGCOLOR(). I wan...



details:   https://anonhg.NetBSD.org/src/rev/baa4cd8b0536
branches:  trunk
changeset: 967766:baa4cd8b0536
user:      ad <ad%NetBSD.org@localhost>
date:      Sat Dec 21 14:41:44 2019 +0000

description:
- Rename VM_PGCOLOR_BUCKET() to VM_PGCOLOR().  I want to reuse "bucket" for
  something else soon and TBH it matches what this macro does better.

- Add inlines to set/get locator values in the unused lower bits of
  pg->phys_addr.  Begin by using it to cache the freelist index, because
  computing it is expensive and that shows up during profiling.  Discussed
  on tech-kern.

diffstat:

 sys/uvm/uvm_page.c    |  18 +++++++++-------
 sys/uvm/uvm_page.h    |  53 ++++++++++++++++++++++++++++++++++++++++++++++----
 sys/uvm/uvm_pager.c   |   6 ++--
 sys/uvm/uvm_pglist.c  |  12 +++++-----
 sys/uvm/uvm_physseg.c |  11 +++++----
 sys/uvm/uvm_vnode.c   |  10 ++++----
 6 files changed, 78 insertions(+), 32 deletions(-)

diffs (285 lines):

diff -r 16b444376ca1 -r baa4cd8b0536 sys/uvm/uvm_page.c
--- a/sys/uvm/uvm_page.c        Sat Dec 21 14:33:18 2019 +0000
+++ b/sys/uvm/uvm_page.c        Sat Dec 21 14:41:44 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_page.c,v 1.208 2019/12/21 14:33:18 ad Exp $        */
+/*     $NetBSD: uvm_page.c,v 1.209 2019/12/21 14:41:44 ad Exp $        */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -66,7 +66,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.208 2019/12/21 14:33:18 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.209 2019/12/21 14:41:44 ad Exp $");
 
 #include "opt_ddb.h"
 #include "opt_uvm.h"
@@ -672,10 +672,10 @@
                                        LIST_REMOVE(pg, pageq.list); /* global */
                                        LIST_REMOVE(pg, listq.list); /* cpu */
                                        LIST_INSERT_HEAD(&gpgfl.pgfl_buckets[
-                                           VM_PGCOLOR_BUCKET(pg)].pgfl_queues[
+                                           VM_PGCOLOR(pg)].pgfl_queues[
                                            i], pg, pageq.list);
                                        LIST_INSERT_HEAD(&pgfl.pgfl_buckets[
-                                           VM_PGCOLOR_BUCKET(pg)].pgfl_queues[
+                                           VM_PGCOLOR(pg)].pgfl_queues[
                                            i], pg, listq.list);
                                }
                        }
@@ -1282,8 +1282,8 @@
         */
 
        iszero = (pg->flags & PG_ZERO);
-       index = uvm_page_lookup_freelist(pg);
-       color = VM_PGCOLOR_BUCKET(pg);
+       index = uvm_page_get_freelist(pg);
+       color = VM_PGCOLOR(pg);
        queue = (iszero ? PGFL_ZEROS : PGFL_UNKNOWN);
 
 #ifdef DEBUG
@@ -1805,6 +1805,8 @@
            pgbuf, pg->pqflags, pg->wire_count, (long)VM_PAGE_TO_PHYS(pg));
        (*pr)("  uobject=%p, uanon=%p, offset=0x%llx loan_count=%d\n",
            pg->uobject, pg->uanon, (long long)pg->offset, pg->loan_count);
+       (*pr)("  bucket=%d freelist=%d\n",
+           uvm_page_get_bucket(pg), uvm_page_get_freelist(pg));
 #if defined(UVM_PAGE_TRKOWN)
        if (pg->flags & PG_BUSY)
                (*pr)("  owning process = %d, tag=%s\n",
@@ -1841,8 +1843,8 @@
 
        /* cross-verify page queue */
        if (pg->flags & PG_FREE) {
-               int fl = uvm_page_lookup_freelist(pg);
-               int color = VM_PGCOLOR_BUCKET(pg);
+               int fl = uvm_page_get_freelist(pg);
+               int color = VM_PGCOLOR(pg);
                pgl = &uvm.page_free[fl].pgfl_buckets[color].pgfl_queues[
                    ((pg)->flags & PG_ZERO) ? PGFL_ZEROS : PGFL_UNKNOWN];
        } else {
diff -r 16b444376ca1 -r baa4cd8b0536 sys/uvm/uvm_page.h
--- a/sys/uvm/uvm_page.h        Sat Dec 21 14:33:18 2019 +0000
+++ b/sys/uvm/uvm_page.h        Sat Dec 21 14:41:44 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_page.h,v 1.87 2019/12/15 21:11:35 ad Exp $ */
+/*     $NetBSD: uvm_page.h,v 1.88 2019/12/21 14:41:44 ad Exp $ */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -343,16 +343,59 @@
 #endif
 
 /*
- * Compute the page color bucket for a given page.
+ * Compute the page color for a given page.
  */
-#define        VM_PGCOLOR_BUCKET(pg) \
+#define        VM_PGCOLOR(pg) \
        (atop(VM_PAGE_TO_PHYS((pg))) & uvmexp.colormask)
-
 #define        PHYS_TO_VM_PAGE(pa)     uvm_phys_to_vm_page(pa)
-
 #define VM_PAGE_IS_FREE(entry)  ((entry)->flags & PG_FREE)
 #define        VM_FREE_PAGE_TO_CPU(pg) ((struct uvm_cpu *)((uintptr_t)pg->offset))
 
+/*
+ * Use the lower 10 bits of pg->phys_addr to cache some some locators for
+ * the page.  This implies that the smallest possible page size is 1kB, and
+ * that nobody should use pg->phys_addr directly (use VM_PAGE_TO_PHYS()).
+ * 
+ * - 5 bits for the freelist index, because uvm_page_lookup_freelist()
+ *   traverses an rbtree and therefore features prominently in traces
+ *   captured during performance test.  It would probably be more useful to
+ *   cache physseg index here because freelist can be inferred from physseg,
+ *   but it requires changes to allocation for UVM_HOTPLUG, so for now we'll
+ *   go with freelist.
+ *
+ * - 5 bits for "bucket", a way for us to categorise pages further as
+ *   needed (e.g. NUMA node).
+ *
+ * None of this is set in stone; it can be adjusted as needed.
+ */
+static inline unsigned
+uvm_page_get_freelist(struct vm_page *pg)
+{
+       unsigned fl = pg->phys_addr & 0x1f;
+       KASSERT(fl == (unsigned)uvm_page_lookup_freelist(pg));
+       return fl;
+}
+
+static inline unsigned
+uvm_page_get_bucket(struct vm_page *pg)
+{
+       return (pg->phys_addr & 0x3e0) >> 5;
+}
+
+static inline void
+uvm_page_set_freelist(struct vm_page *pg, unsigned fl)
+{
+       KASSERT(fl < 32);
+       pg->phys_addr = (pg->phys_addr & ~0x1f) | fl;
+}
+
+static inline void
+uvm_page_set_bucket(struct vm_page *pg, unsigned b)
+{
+       KASSERT(b < 32);
+       pg->phys_addr = (pg->phys_addr & ~0x3e0) | (b << 5);
+}
+
 #ifdef DEBUG
 void uvm_pagezerocheck(struct vm_page *);
 #endif /* DEBUG */
diff -r 16b444376ca1 -r baa4cd8b0536 sys/uvm/uvm_pager.c
--- a/sys/uvm/uvm_pager.c       Sat Dec 21 14:33:18 2019 +0000
+++ b/sys/uvm/uvm_pager.c       Sat Dec 21 14:41:44 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_pager.c,v 1.116 2019/12/14 21:36:00 ad Exp $       */
+/*     $NetBSD: uvm_pager.c,v 1.117 2019/12/21 14:41:44 ad Exp $       */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.116 2019/12/14 21:36:00 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.117 2019/12/21 14:41:44 ad Exp $");
 
 #include "opt_uvmhist.h"
 #include "opt_readahead.h"
@@ -176,7 +176,7 @@
        struct vm_page *pp;
        vm_prot_t prot;
        const bool pdaemon = (curlwp == uvm.pagedaemon_lwp);
-       const u_int first_color = VM_PGCOLOR_BUCKET(*pps);
+       const u_int first_color = VM_PGCOLOR(*pps);
        UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist);
 
        UVMHIST_LOG(maphist,"(pps=0x%#jx, npages=%jd, first_color=%ju)",
diff -r 16b444376ca1 -r baa4cd8b0536 sys/uvm/uvm_pglist.c
--- a/sys/uvm/uvm_pglist.c      Sat Dec 21 14:33:18 2019 +0000
+++ b/sys/uvm/uvm_pglist.c      Sat Dec 21 14:41:44 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_pglist.c,v 1.75 2019/12/21 13:00:25 ad Exp $       */
+/*     $NetBSD: uvm_pglist.c,v 1.76 2019/12/21 14:41:44 ad Exp $       */
 
 /*-
  * Copyright (c) 1997 The NetBSD Foundation, Inc.
@@ -35,7 +35,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pglist.c,v 1.75 2019/12/21 13:00:25 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_pglist.c,v 1.76 2019/12/21 14:41:44 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -87,8 +87,8 @@
 #error uvm_pglistalloc needs to be updated
 #endif
 
-       free_list = uvm_page_lookup_freelist(pg);
-       color = VM_PGCOLOR_BUCKET(pg);
+       free_list = uvm_page_get_freelist(pg);
+       color = VM_PGCOLOR(pg);
        pgflidx = (pg->flags & PG_ZERO) ? PGFL_ZEROS : PGFL_UNKNOWN;
 #ifdef UVMDEBUG
        struct vm_page *tp;
@@ -582,8 +582,8 @@
                if (iszero)
                        uvm_pagezerocheck(pg);
 #endif /* DEBUG */
-               index = uvm_page_lookup_freelist(pg);
-               color = VM_PGCOLOR_BUCKET(pg);
+               index = uvm_page_get_freelist(pg);
+               color = VM_PGCOLOR(pg);
                queue = iszero ? PGFL_ZEROS : PGFL_UNKNOWN;
                pg->offset = (uintptr_t)ucpu;
                LIST_INSERT_HEAD(&uvm.page_free[index].pgfl_buckets[color].
diff -r 16b444376ca1 -r baa4cd8b0536 sys/uvm/uvm_physseg.c
--- a/sys/uvm/uvm_physseg.c     Sat Dec 21 14:33:18 2019 +0000
+++ b/sys/uvm/uvm_physseg.c     Sat Dec 21 14:41:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_physseg.c,v 1.12 2019/12/20 19:03:17 ad Exp $ */
+/* $NetBSD: uvm_physseg.c,v 1.13 2019/12/21 14:41:44 ad Exp $ */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -96,7 +96,6 @@
        struct  extent *ext;            /* extent(9) structure to manage pgs[] */
        int     free_list;              /* which free list they belong on */
        u_int   start_hint;             /* start looking for free pages here */
-                                       /* protected by uvm_fpageqlock */
 #ifdef __HAVE_PMAP_PHYSSEG
        struct  pmap_physseg pmseg;     /* pmap specific (MD) data */
 #endif
@@ -1103,15 +1102,17 @@
        /* init and free vm_pages (we've already zeroed them) */
        paddr = ctob(seg->start);
        for (i = 0 ; i < n ; i++, paddr += PAGE_SIZE) {
-               seg->pgs[i].phys_addr = paddr;
+               pg = &seg->pgs[i];
+               pg->phys_addr = paddr;
 #ifdef __HAVE_VM_PAGE_MD
-               VM_MDPAGE_INIT(&seg->pgs[i]);
+               VM_MDPAGE_INIT(pg);
 #endif
                if (atop(paddr) >= seg->avail_start &&
                    atop(paddr) < seg->avail_end) {
                        uvmexp.npages++;
                        /* add page to free pool */
-                       pg = &seg->pgs[i];
+                       uvm_page_set_freelist(pg,
+                           uvm_page_lookup_freelist(pg));
                        /* Disable LOCKDEBUG: too many and too early. */
                        mutex_init(&pg->interlock, MUTEX_NODEBUG, IPL_NONE);
                        uvm_pagefree(pg);
diff -r 16b444376ca1 -r baa4cd8b0536 sys/uvm/uvm_vnode.c
--- a/sys/uvm/uvm_vnode.c       Sat Dec 21 14:33:18 2019 +0000
+++ b/sys/uvm/uvm_vnode.c       Sat Dec 21 14:41:44 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_vnode.c,v 1.103 2017/10/28 00:37:13 pgoyette Exp $ */
+/*     $NetBSD: uvm_vnode.c,v 1.104 2019/12/21 14:41:44 ad Exp $       */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -45,7 +45,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.103 2017/10/28 00:37:13 pgoyette Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.104 2019/12/21 14:41:44 ad Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_uvmhist.h"
@@ -272,7 +272,7 @@
                                continue;
                        }
                        UVMHIST_LOG(ubchist, "alloced %#jx (color %ju)",
-                           (uintptr_t)pg, VM_PGCOLOR_BUCKET(pg), 0, 0);
+                           (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
                        break;
                } else if (flags & UFP_NOCACHE) {
                        UVMHIST_LOG(ubchist, "nocache",0,0,0,0);
@@ -287,7 +287,7 @@
                        }
                        pg->flags |= PG_WANTED;
                        UVMHIST_LOG(ubchist, "wait %#jx (color %ju)",
-                           (uintptr_t)pg, VM_PGCOLOR_BUCKET(pg), 0, 0);
+                           (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
                        UVM_UNLOCK_AND_WAIT(pg, uobj->vmobjlock, 0,
                                            "uvn_fp2", 0);
                        mutex_enter(uobj->vmobjlock);
@@ -315,7 +315,7 @@
                pg->flags |= PG_BUSY;
                UVM_PAGE_OWN(pg, "uvn_findpage");
                UVMHIST_LOG(ubchist, "found %#jx (color %ju)",
-                   (uintptr_t)pg, VM_PGCOLOR_BUCKET(pg), 0, 0);
+                   (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
                break;
        }
        *pgp = pg;



Home | Main Index | Thread Index | Old Index