[src/trunk]: src/sys Use large pages for the kMSan shadows. This greatly impr...

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/a6d108ce1bb1
branches:  trunk
changeset: 930861:a6d108ce1bb1
user:      maxv <maxv%NetBSD.org@localhost>
date:      Wed Apr 15 16:28:28 2020 +0000

description:
Use large pages for the kMSan shadows. This greatly improves performance,
and slightly reduces memory consumption.

diffstat:

 sys/arch/amd64/include/msan.h |  41 ++++++++++++++++++++++++++++++++++++-----
 sys/kern/subr_msan.c          |  10 +++++-----
 2 files changed, 41 insertions(+), 10 deletions(-)

diffs (114 lines):

diff -r c63f2ed92c97 -r a6d108ce1bb1 sys/arch/amd64/include/msan.h
--- a/sys/arch/amd64/include/msan.h     Wed Apr 15 15:55:24 2020 +0000
+++ b/sys/arch/amd64/include/msan.h     Wed Apr 15 16:28:28 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: msan.h,v 1.1 2019/11/14 16:23:52 maxv Exp $    */
+/*     $NetBSD: msan.h,v 1.2 2020/04/15 16:28:29 maxv Exp $    */
 
 /*
  * Copyright (c) 2019 The NetBSD Foundation, Inc.
@@ -97,31 +97,62 @@
 static inline paddr_t
 __md_palloc(void)
 {
+       /* The page is zeroed. */
        return pmap_get_physpage();
 }
 
+static inline paddr_t
+__md_palloc_large(void)
+{
+       struct pglist pglist;
+       int ret;
+
+       if (!uvm.page_init_done)
+               return 0;
+
+       kmsan_init_arg(sizeof(psize_t) + 4 * sizeof(paddr_t) +
+           sizeof(struct pglist *) + 2 * sizeof(int));
+       ret = uvm_pglistalloc(NBPD_L2, 0, ~0UL, NBPD_L2, 0,
+           &pglist, 1, 0);
+       if (ret != 0)
+               return 0;
+
+       /* The page may not be zeroed. */
+       return VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist));
+}
+
 static void
 kmsan_md_shadow_map_page(vaddr_t va)
 {
+       const pt_entry_t pteflags = PTE_W | pmap_pg_nx | PTE_P;
        paddr_t pa;
 
        KASSERT(va >= __MD_SHADOW_START && va < __MD_SHADOW_END);
 
        if (!pmap_valid_entry(L4_BASE[pl4_i(va)])) {
                pa = __md_palloc();
-               L4_BASE[pl4_i(va)] = pa | PTE_W | pmap_pg_nx | PTE_P;
+               L4_BASE[pl4_i(va)] = pa | pteflags;
        }
        if (!pmap_valid_entry(L3_BASE[pl3_i(va)])) {
                pa = __md_palloc();
-               L3_BASE[pl3_i(va)] = pa | PTE_W | pmap_pg_nx | PTE_P;
+               L3_BASE[pl3_i(va)] = pa | pteflags;
        }
        if (!pmap_valid_entry(L2_BASE[pl2_i(va)])) {
+               if ((pa = __md_palloc_large()) != 0) {
+                       L2_BASE[pl2_i(va)] = pa | pteflags | PTE_PS |
+                           pmap_pg_g;
+                       __insn_barrier();
+                       __builtin_memset(va, 0, NBPD_L2);
+                       return;
+               }
                pa = __md_palloc();
-               L2_BASE[pl2_i(va)] = pa | PTE_W | pmap_pg_nx | PTE_P;
+               L2_BASE[pl2_i(va)] = pa | pteflags;
+       } else if (L2_BASE[pl2_i(va)] & PTE_PS) {
+               return;
        }
        if (!pmap_valid_entry(L1_BASE[pl1_i(va)])) {
                pa = __md_palloc();
-               L1_BASE[pl1_i(va)] = pa | PTE_W | pmap_pg_g | pmap_pg_nx | PTE_P;
+               L1_BASE[pl1_i(va)] = pa | pteflags | pmap_pg_g;
        }
 }
 
diff -r c63f2ed92c97 -r a6d108ce1bb1 sys/kern/subr_msan.c
--- a/sys/kern/subr_msan.c      Wed Apr 15 15:55:24 2020 +0000
+++ b/sys/kern/subr_msan.c      Wed Apr 15 16:28:28 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: subr_msan.c,v 1.9 2020/04/03 18:26:14 maxv Exp $       */
+/*     $NetBSD: subr_msan.c,v 1.10 2020/04/15 16:28:28 maxv Exp $      */
 
 /*
  * Copyright (c) 2019-2020 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.9 2020/04/03 18:26:14 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.10 2020/04/15 16:28:28 maxv Exp $");
 
 #include <sys/param.h>
 #include <sys/device.h>
@@ -50,6 +50,9 @@
 
 static void kmsan_printf(const char *, ...);
 
+void kmsan_init_arg(size_t);
+void kmsan_init_ret(size_t);
+
 #ifdef KMSAN_PANIC
 #define REPORT panic
 #else
@@ -356,9 +359,6 @@
        }
 }
 
-void kmsan_init_arg(size_t);
-void kmsan_init_ret(size_t);
-
 void
 kmsan_init_arg(size_t n)
 {