[src/trunk]: src/sys Redo the page allocator to perform better, especially on...

[ad <ad%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/7d4f4ca11ac6
branches:  trunk
changeset: 466613:7d4f4ca11ac6
user:      ad <ad%NetBSD.org@localhost>
date:      Fri Dec 27 12:51:56 2019 +0000

description:
Redo the page allocator to perform better, especially on multi-core and
multi-socket systems.  Proposed on tech-kern.  While here:

- add rudimentary NUMA support - needs more work.
- remove now unused "listq" from vm_page.

diffstat:

 sys/arch/amd64/amd64/autoconf.c |    14 +-
 sys/arch/i386/i386/autoconf.c   |    14 +-
 sys/ddb/db_command.c            |    20 +-
 sys/dev/acpi/acpi_srat.c        |    28 +-
 sys/dev/acpi/acpi_srat.h        |     3 +-
 sys/kern/init_main.c            |     8 +-
 sys/uvm/files.uvm               |     3 +-
 sys/uvm/uvm.h                   |    23 +-
 sys/uvm/uvm_ddb.h               |     3 +-
 sys/uvm/uvm_extern.h            |     4 +-
 sys/uvm/uvm_glue.c              |    13 +-
 sys/uvm/uvm_init.c              |     5 +-
 sys/uvm/uvm_page.c              |  1013 +++++++++++++++++++++++---------------
 sys/uvm/uvm_page.h              |    19 +-
 sys/uvm/uvm_pgflcache.c         |   471 ++++++++++++++++++
 sys/uvm/uvm_pgflcache.h         |    43 +
 sys/uvm/uvm_pglist.c            |    88 +-
 sys/uvm/uvm_pglist.h            |    54 +-
 18 files changed, 1324 insertions(+), 502 deletions(-)

diffs (truncated from 2652 to 300 lines):

diff -r 0a8ff56a136e -r 7d4f4ca11ac6 sys/arch/amd64/amd64/autoconf.c
--- a/sys/arch/amd64/amd64/autoconf.c   Fri Dec 27 11:15:06 2019 +0000
+++ b/sys/arch/amd64/amd64/autoconf.c   Fri Dec 27 12:51:56 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: autoconf.c,v 1.28 2017/10/22 00:59:28 maya Exp $       */
+/*     $NetBSD: autoconf.c,v 1.29 2019/12/27 12:51:56 ad Exp $ */
 
 /*-
  * Copyright (c) 1990 The Regents of the University of California.
@@ -46,7 +46,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: autoconf.c,v 1.28 2017/10/22 00:59:28 maya Exp $");
+__KERNEL_RCSID(0, "$NetBSD: autoconf.c,v 1.29 2019/12/27 12:51:56 ad Exp $");
 
 #include "opt_multiprocessor.h"
 #include "opt_intrdebug.h"
@@ -60,9 +60,14 @@
 #include <machine/pte.h>
 #include <machine/cpufunc.h>
 
+#include "acpica.h"
 #include "ioapic.h"
 #include "lapic.h"
 
+#if NACPICA > 0
+#include <dev/acpi/acpi_srat.h>
+#endif
+
 #if NIOAPIC > 0
 #include <machine/i82093var.h>
 #endif
@@ -112,6 +117,11 @@
        cpu_init_idle_lwps();
 #endif
 
+#if NACPICA > 0
+       /* Load NUMA memory regions into UVM. */
+       acpisrat_load_uvm();
+#endif
+
        spl0();
        lcr8(0);
 }
diff -r 0a8ff56a136e -r 7d4f4ca11ac6 sys/arch/i386/i386/autoconf.c
--- a/sys/arch/i386/i386/autoconf.c     Fri Dec 27 11:15:06 2019 +0000
+++ b/sys/arch/i386/i386/autoconf.c     Fri Dec 27 12:51:56 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: autoconf.c,v 1.105 2017/10/22 00:59:28 maya Exp $      */
+/*     $NetBSD: autoconf.c,v 1.106 2019/12/27 12:51:56 ad Exp $        */
 
 /*-
  * Copyright (c) 1990 The Regents of the University of California.
@@ -46,7 +46,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: autoconf.c,v 1.105 2017/10/22 00:59:28 maya Exp $");
+__KERNEL_RCSID(0, "$NetBSD: autoconf.c,v 1.106 2019/12/27 12:51:56 ad Exp $");
 
 #include "opt_intrdebug.h"
 #include "opt_multiprocessor.h"
@@ -65,9 +65,14 @@
 #include <machine/cpufunc.h>
 #include <x86/fpu.h>
 
+#include "acpica.h"
 #include "ioapic.h"
 #include "lapic.h"
 
+#if NACPICA > 0
+#include <dev/acpi/acpi_srat.h>
+#endif
+
 #if NIOAPIC > 0
 #include <machine/i82093var.h>
 #endif
@@ -132,6 +137,11 @@
        cpu_init_idle_lwps();
 #endif
 
+#if NACPICA > 0
+       /* Load NUMA memory regions into UVM. */
+       acpisrat_load_uvm();
+#endif
+
        spl0();
 #if NLAPIC > 0
        lapic_write_tpri(0);
diff -r 0a8ff56a136e -r 7d4f4ca11ac6 sys/ddb/db_command.c
--- a/sys/ddb/db_command.c      Fri Dec 27 11:15:06 2019 +0000
+++ b/sys/ddb/db_command.c      Fri Dec 27 12:51:56 2019 +0000
@@ -1,7 +1,8 @@
-/*     $NetBSD: db_command.c,v 1.165 2019/12/15 20:29:08 joerg Exp $   */
+/*     $NetBSD: db_command.c,v 1.166 2019/12/27 12:51:56 ad Exp $      */
 
 /*
- * Copyright (c) 1996, 1997, 1998, 1999, 2002, 2009 The NetBSD Foundation, Inc.
+ * Copyright (c) 1996, 1997, 1998, 1999, 2002, 2009, 2019
+ *     The NetBSD Foundation, Inc.
  * All rights reserved.
  *
  * This code is derived from software contributed to The NetBSD Foundation
@@ -60,7 +61,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: db_command.c,v 1.165 2019/12/15 20:29:08 joerg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: db_command.c,v 1.166 2019/12/27 12:51:56 ad Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_aio.h"
@@ -193,6 +194,7 @@
 static void    db_lock_print_cmd(db_expr_t, bool, db_expr_t, const char *);
 static void    db_show_all_locks(db_expr_t, bool, db_expr_t, const char *);
 static void    db_show_lockstats(db_expr_t, bool, db_expr_t, const char *);
+static void    db_show_all_freelists(db_expr_t, bool, db_expr_t, const char *);
 static void    db_mount_print_cmd(db_expr_t, bool, db_expr_t, const char *);
 static void    db_show_all_mount(db_expr_t, bool, db_expr_t, const char *);
 static void    db_mbuf_print_cmd(db_expr_t, bool, db_expr_t, const char *);
@@ -234,6 +236,8 @@
            0 ,"Show all held locks", "[/t]", NULL) },
        { DDB_ADD_CMD("mount",  db_show_all_mount,      0,
            "Print all mount structures.", "[/f]", NULL) },
+       { DDB_ADD_CMD("freelists",      db_show_all_freelists,
+           0 ,"Show all freelists", NULL, NULL) },
 #ifdef AIO
        /*added from all sub cmds*/
        { DDB_ADD_CMD("aio_jobs",       db_show_aio_jobs,       0,
@@ -1285,6 +1289,16 @@
 }
 
 static void
+db_show_all_freelists(db_expr_t addr, bool have_addr,
+    db_expr_t count, const char *modif)
+{
+
+#ifdef _KERNEL /* XXX CRASH(8) */
+       uvm_page_print_freelists(db_printf);
+#endif
+}
+
+static void
 db_show_lockstats(db_expr_t addr, bool have_addr,
     db_expr_t count, const char *modif)
 {
diff -r 0a8ff56a136e -r 7d4f4ca11ac6 sys/dev/acpi/acpi_srat.c
--- a/sys/dev/acpi/acpi_srat.c  Fri Dec 27 11:15:06 2019 +0000
+++ b/sys/dev/acpi/acpi_srat.c  Fri Dec 27 12:51:56 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: acpi_srat.c,v 1.7 2019/12/22 22:18:04 ad Exp $ */
+/* $NetBSD: acpi_srat.c,v 1.8 2019/12/27 12:51:57 ad Exp $ */
 
 /*
  * Copyright (c) 2009 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: acpi_srat.c,v 1.7 2019/12/22 22:18:04 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: acpi_srat.c,v 1.8 2019/12/27 12:51:57 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/kmem.h>
@@ -39,6 +39,8 @@
 #include <dev/acpi/acpivar.h>
 #include <dev/acpi/acpi_srat.h>
 
+#include <uvm/uvm_extern.h>
+
 static ACPI_TABLE_SRAT *srat;
 
 static uint32_t nnodes; /* Number of NUMA nodes */
@@ -472,6 +474,28 @@
        }
 }
 
+void
+acpisrat_load_uvm(void)
+{
+       uint32_t i, j, nn, nm;
+       struct acpisrat_mem m;
+
+       nn = acpisrat_nodes();
+       aprint_debug("SRAT: %u NUMA nodes\n", nn);
+       for (i = 0; i < nn; i++) {
+               nm = acpisrat_node_memoryranges(i);
+               for (j = 0; j < nm; j++) {
+                       acpisrat_mem(i, j, &m);
+                       aprint_debug("SRAT: node %u memory range %u (0x%"
+                           PRIx64" - 0x%"PRIx64" flags %u)\n",
+                           m.nodeid, j, m.baseaddress,
+                           m.baseaddress + m.length, m.flags);
+                       uvm_page_numa_load(trunc_page(m.baseaddress),
+                           trunc_page(m.length), m.nodeid);
+               }
+       }
+}
+
 /*
  * Get number of NUMA nodes.
  */
diff -r 0a8ff56a136e -r 7d4f4ca11ac6 sys/dev/acpi/acpi_srat.h
--- a/sys/dev/acpi/acpi_srat.h  Fri Dec 27 11:15:06 2019 +0000
+++ b/sys/dev/acpi/acpi_srat.h  Fri Dec 27 12:51:56 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: acpi_srat.h,v 1.4 2017/12/28 08:49:28 maxv Exp $ */
+/* $NetBSD: acpi_srat.h,v 1.5 2019/12/27 12:51:57 ad Exp $ */
 
 /*
  * Copyright (c) 2009 The NetBSD Foundation, Inc.
@@ -68,6 +68,7 @@
 int acpisrat_refresh(void);
 int acpisrat_exit(void);
 void acpisrat_dump(void);
+void acpisrat_load_uvm(void);
 uint32_t acpisrat_nodes(void);
 uint32_t acpisrat_node_cpus(acpisrat_nodeid_t);
 uint32_t acpisrat_node_memoryranges(acpisrat_nodeid_t);
diff -r 0a8ff56a136e -r 7d4f4ca11ac6 sys/kern/init_main.c
--- a/sys/kern/init_main.c      Fri Dec 27 11:15:06 2019 +0000
+++ b/sys/kern/init_main.c      Fri Dec 27 12:51:56 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: init_main.c,v 1.512 2019/12/22 15:00:42 ad Exp $       */
+/*     $NetBSD: init_main.c,v 1.513 2019/12/27 12:51:57 ad Exp $       */
 
 /*-
  * Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -97,7 +97,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.512 2019/12/22 15:00:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.513 2019/12/27 12:51:57 ad Exp $");
 
 #include "opt_ddb.h"
 #include "opt_inet.h"
@@ -814,6 +814,10 @@
        for (CPU_INFO_FOREACH(cii, ci)) {
                uvm_cpu_attach(ci);
        }
+
+       /* Decide how to partition free memory. */
+       uvm_page_rebucket();
+
        mp_online = true;
 #if defined(MULTIPROCESSOR)
        cpu_boot_secondary_processors();
diff -r 0a8ff56a136e -r 7d4f4ca11ac6 sys/uvm/files.uvm
--- a/sys/uvm/files.uvm Fri Dec 27 11:15:06 2019 +0000
+++ b/sys/uvm/files.uvm Fri Dec 27 12:51:56 2019 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: files.uvm,v 1.31 2019/12/15 21:11:35 ad Exp $
+#      $NetBSD: files.uvm,v 1.32 2019/12/27 12:51:57 ad Exp $
 
 #
 # UVM options
@@ -42,6 +42,7 @@
 file   uvm/uvm_pdaemon.c               uvm
 file   uvm/uvm_pdpolicy_clock.c        !pdpolicy_clockpro
 file   uvm/uvm_pdpolicy_clockpro.c     pdpolicy_clockpro
+file   uvm/uvm_pgflcache.c             uvm
 file   uvm/uvm_pglist.c                uvm
 file   uvm/uvm_physseg.c               uvm
 file   uvm/uvm_readahead.c             uvm
diff -r 0a8ff56a136e -r 7d4f4ca11ac6 sys/uvm/uvm.h
--- a/sys/uvm/uvm.h     Fri Dec 27 11:15:06 2019 +0000
+++ b/sys/uvm/uvm.h     Fri Dec 27 12:51:56 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm.h,v 1.70 2019/12/13 20:10:22 ad Exp $      */
+/*     $NetBSD: uvm.h,v 1.71 2019/12/27 12:51:57 ad Exp $      */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -71,21 +71,19 @@
 #include <machine/vmparam.h>
 
 struct workqueue;
+struct pgflcache;
 
 /*
  * per-cpu data
  */
 
 struct uvm_cpu {
-       struct pgfreelist page_free[VM_NFREELIST]; /* unallocated pages */
-       int page_free_nextcolor;        /* next color to allocate from */
-       int page_idlezero_next;         /* which color to zero next */
-       bool page_idle_zero;            /* TRUE if we should try to zero
-                                          pages in the idle loop */
-       int pages[PGFL_NQUEUES];        /* total of pages in page_free */
-       u_int emap_gen;                 /* emap generation number */
-
-       krndsource_t rs;                /* entropy source */
+       struct pgflcache *pgflcache[VM_NFREELIST];/* cpu-local cached pages */
+       void            *pgflcachemem;          /* pointer to allocated mem */
+       size_t          pgflcachememsz;         /* size of allocated memory */
+       u_int           pgflcolor;              /* next color to allocate */
+       u_int           pgflbucket;             /* where to send our pages */