[src/trunk]: src/sys With DEBUG defined, it's possible to execute a TLB-vs-se...

[thorpej <thorpej%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/d7f433b0e4c6
branches:  trunk
changeset: 745766:d7f433b0e4c6
user:      thorpej <thorpej%NetBSD.org@localhost>
date:      Wed Mar 11 13:30:31 2020 +0000

description:
With DEBUG defined, it's possible to execute a TLB-vs-segmap consistency
check from a (soft) interrupt handler.  But if a platform does not otherwise
require the pmap_tlb_miss_lock, then where will be a brief window of
inconsistency that, while harmless, will still fire an assertion in the
consistency check.

Fix this with the following changes:
1- Refactor the pmap_tlb_miss_lock into MI code and rename it from
   pmap_tlb_miss_lock_{enter,exit}() to pmap_tlb_miss_lock_{enter,exit}().
   MD code can still define the "md" hooks as necessary, and if so, will
   override the common implementation.
2- Provde a pmap_bootstrap_common() function to perform common pmap bootstrap
   operations, namely initializing the pmap_tlb_miss_lock if it's needed.
   If MD code overrides the implementation, it's responsible for initializing
   its own lock.
3- Call pmap_bootstrap_common() from the mips, powerpc booke, and riscv
   pmap_bootstrap() routines.  (This required adding one for riscv.)
4- Switch powerpc booke to the common pmap_tlb_miss_lock.
5- Enable pmap_tlb_miss_lock if DEBUG is defined, even if it's not otherwise
   required.

PR port-mips/55062 (Failed assertion in pmap_md_tlb_check_entry())

diffstat:

 sys/arch/mips/mips/pmap_machdep.c     |    7 +-
 sys/arch/powerpc/booke/booke_pmap.c   |   29 +------
 sys/arch/powerpc/include/booke/pmap.h |    6 +-
 sys/arch/riscv/include/pmap.h         |    4 +-
 sys/arch/riscv/riscv/pmap_machdep.c   |   11 ++-
 sys/arch/riscv/riscv/riscv_machdep.c  |    5 +-
 sys/uvm/pmap/pmap.c                   |  117 ++++++++++++++++++++++++---------
 sys/uvm/pmap/pmap.h                   |    3 +-
 8 files changed, 113 insertions(+), 69 deletions(-)

diffs (truncated from 498 to 300 lines):

diff -r 63f52f65d600 -r d7f433b0e4c6 sys/arch/mips/mips/pmap_machdep.c
--- a/sys/arch/mips/mips/pmap_machdep.c Wed Mar 11 10:57:32 2020 +0000
+++ b/sys/arch/mips/mips/pmap_machdep.c Wed Mar 11 13:30:31 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap_machdep.c,v 1.26 2019/10/20 08:29:38 skrll Exp $  */
+/*     $NetBSD: pmap_machdep.c,v 1.27 2020/03/11 13:30:31 thorpej Exp $        */
 
 /*-
  * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
@@ -67,7 +67,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: pmap_machdep.c,v 1.26 2019/10/20 08:29:38 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap_machdep.c,v 1.27 2020/03/11 13:30:31 thorpej Exp $");
 
 /*
  *     Manages physical address maps.
@@ -358,6 +358,9 @@
        kcpuset_set(pm->pm_onproc, cpu_number());
        kcpuset_set(pm->pm_active, cpu_number());
 #endif
+
+       pmap_bootstrap_common();
+
        pmap_tlb_info_init(&pmap_tlb0_info);            /* init the lock */
 
        /*
diff -r 63f52f65d600 -r d7f433b0e4c6 sys/arch/powerpc/booke/booke_pmap.c
--- a/sys/arch/powerpc/booke/booke_pmap.c       Wed Mar 11 10:57:32 2020 +0000
+++ b/sys/arch/powerpc/booke/booke_pmap.c       Wed Mar 11 13:30:31 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: booke_pmap.c,v 1.26 2018/09/03 16:29:26 riastradh Exp $        */
+/*     $NetBSD: booke_pmap.c,v 1.27 2020/03/11 13:30:31 thorpej Exp $  */
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -38,7 +38,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: booke_pmap.c,v 1.26 2018/09/03 16:29:26 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: booke_pmap.c,v 1.27 2020/03/11 13:30:31 thorpej Exp $");
 
 #include <sys/param.h>
 #include <sys/kcore.h>
@@ -49,10 +49,6 @@
 
 #include <machine/pmap.h>
 
-#if defined(MULTIPROCESSOR)
-kmutex_t pmap_tlb_miss_lock;
-#endif
-
 PMAP_COUNTER(zeroed_pages, "pages zeroed");
 PMAP_COUNTER(copied_pages, "pages copied");
 
@@ -154,13 +150,12 @@
 
        KASSERT(endkernel == trunc_page(endkernel));
 
+       /* common initialization */
+       pmap_bootstrap_common();
+
        /* init the lock */
        pmap_tlb_info_init(&pmap_tlb0_info);
 
-#if defined(MULTIPROCESSOR)
-       mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH);
-#endif
-
        /*
         * Compute the number of pages kmem_arena will have.
         */
@@ -422,18 +417,4 @@
 {
        /* nothing */
 }
-
-void
-pmap_md_tlb_miss_lock_enter(void)
-{
-
-       mutex_spin_enter(&pmap_tlb_miss_lock);
-}
-
-void
-pmap_md_tlb_miss_lock_exit(void)
-{
-
-       mutex_spin_exit(&pmap_tlb_miss_lock);
-}
 #endif /* MULTIPROCESSOR */
diff -r 63f52f65d600 -r d7f433b0e4c6 sys/arch/powerpc/include/booke/pmap.h
--- a/sys/arch/powerpc/include/booke/pmap.h     Wed Mar 11 10:57:32 2020 +0000
+++ b/sys/arch/powerpc/include/booke/pmap.h     Wed Mar 11 13:30:31 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap.h,v 1.18 2018/04/19 21:50:07 christos Exp $       */
+/*     $NetBSD: pmap.h,v 1.19 2020/03/11 13:30:31 thorpej Exp $        */
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -94,9 +94,7 @@
 bool   pmap_md_tlb_check_entry(void *, vaddr_t, tlb_asid_t, pt_entry_t);
 
 #ifdef MULTIPROCESSOR
-#define        PMAP_MD_NEED_TLB_MISS_LOCK
-void   pmap_md_tlb_miss_lock_enter(void);
-void   pmap_md_tlb_miss_lock_exit(void);
+#define        PMAP_NEED_TLB_MISS_LOCK
 #endif /* MULTIPROCESSOR */
 
 #ifdef PMAP_MINIMALTLB
diff -r 63f52f65d600 -r d7f433b0e4c6 sys/arch/riscv/include/pmap.h
--- a/sys/arch/riscv/include/pmap.h     Wed Mar 11 10:57:32 2020 +0000
+++ b/sys/arch/riscv/include/pmap.h     Wed Mar 11 13:30:31 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.3 2019/06/16 07:42:52 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.4 2020/03/11 13:30:31 thorpej Exp $ */
 
 /*
  * Copyright (c) 2014, 2019 The NetBSD Foundation, Inc.
@@ -108,6 +108,8 @@
        pd_entry_t *md_pdetab;
 };
 
+void   pmap_bootstrap(void);
+
 struct vm_page *
         pmap_md_alloc_poolpage(int flags);
 vaddr_t pmap_md_map_poolpage(paddr_t, vsize_t);
diff -r 63f52f65d600 -r d7f433b0e4c6 sys/arch/riscv/riscv/pmap_machdep.c
--- a/sys/arch/riscv/riscv/pmap_machdep.c       Wed Mar 11 10:57:32 2020 +0000
+++ b/sys/arch/riscv/riscv/pmap_machdep.c       Wed Mar 11 13:30:31 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap_machdep.c,v 1.4 2019/06/16 07:42:52 maxv Exp $ */
+/* $NetBSD: pmap_machdep.c,v 1.5 2020/03/11 13:30:31 thorpej Exp $ */
 
 /*
  * Copyright (c) 2014, 2019 The NetBSD Foundation, Inc.
@@ -33,7 +33,7 @@
 
 #include <sys/cdefs.h>
 
-__RCSID("$NetBSD: pmap_machdep.c,v 1.4 2019/06/16 07:42:52 maxv Exp $");
+__RCSID("$NetBSD: pmap_machdep.c,v 1.5 2020/03/11 13:30:31 thorpej Exp $");
 
 #include <sys/param.h>
 
@@ -47,6 +47,13 @@
 vaddr_t pmap_direct_end __read_mostly;
 
 void
+pmap_bootstrap(void)
+{
+
+       pmap_bootstrap_common();
+}
+
+void
 pmap_zero_page(paddr_t pa)
 {
 #ifdef PMAP_DIRECT_MAP
diff -r 63f52f65d600 -r d7f433b0e4c6 sys/arch/riscv/riscv/riscv_machdep.c
--- a/sys/arch/riscv/riscv/riscv_machdep.c      Wed Mar 11 10:57:32 2020 +0000
+++ b/sys/arch/riscv/riscv/riscv_machdep.c      Wed Mar 11 13:30:31 2020 +0000
@@ -31,7 +31,7 @@
 
 #include "opt_modular.h"
 
-__RCSID("$NetBSD: riscv_machdep.c,v 1.8 2019/12/31 13:07:12 ad Exp $");
+__RCSID("$NetBSD: riscv_machdep.c,v 1.9 2020/03/11 13:30:31 thorpej Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -333,4 +333,7 @@
 void
 init_riscv(vaddr_t kernstart, vaddr_t kernend)
 {
+
+       /* Early VM bootstrap. */
+       pmap_bootstrap();
 }
diff -r 63f52f65d600 -r d7f433b0e4c6 sys/uvm/pmap/pmap.c
--- a/sys/uvm/pmap/pmap.c       Wed Mar 11 10:57:32 2020 +0000
+++ b/sys/uvm/pmap/pmap.c       Wed Mar 11 13:30:31 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap.c,v 1.45 2019/12/18 10:55:50 skrll Exp $  */
+/*     $NetBSD: pmap.c,v 1.46 2020/03/11 13:30:31 thorpej Exp $        */
 
 /*-
  * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
@@ -67,7 +67,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.45 2019/12/18 10:55:50 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.46 2020/03/11 13:30:31 thorpej Exp $");
 
 /*
  *     Manages physical address maps.
@@ -262,10 +262,49 @@
 #define        pmap_pv_alloc()         pool_get(&pmap_pv_pool, PR_NOWAIT)
 #define        pmap_pv_free(pv)        pool_put(&pmap_pv_pool, (pv))
 
-#if !defined(MULTIPROCESSOR) || !defined(PMAP_MD_NEED_TLB_MISS_LOCK)
-#define        pmap_md_tlb_miss_lock_enter()   do { } while(/*CONSTCOND*/0)
-#define        pmap_md_tlb_miss_lock_exit()    do { } while(/*CONSTCOND*/0)
-#endif /* !MULTIPROCESSOR || !PMAP_MD_NEED_TLB_MISS_LOCK */
+#ifndef PMAP_NEED_TLB_MISS_LOCK
+
+#if defined(PMAP_MD_NEED_TLB_MISS_LOCK) || defined(DEBUG)
+#define        PMAP_NEED_TLB_MISS_LOCK
+#endif /* PMAP_MD_NEED_TLB_MISS_LOCK || DEBUG */
+
+#endif /* PMAP_NEED_TLB_MISS_LOCK */
+
+#ifdef PMAP_NEED_TLB_MISS_LOCK
+
+#ifdef PMAP_MD_NEED_TLB_MISS_LOCK
+#define        pmap_tlb_miss_lock_init()       __nothing /* MD code deals with this */
+#define        pmap_tlb_miss_lock_enter()      pmap_md_tlb_miss_lock_enter()
+#define        pmap_tlb_miss_lock_exit()       pmap_md_tlb_miss_lock_exit()
+#else
+static kmutex_t pmap_tlb_miss_lock __cacheline_aligned;
+
+static void
+pmap_tlb_miss_lock_init(void)
+{
+       mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH);
+}
+
+static inline void
+pmap_tlb_miss_lock_enter(void)
+{
+       mutex_spin_enter(&pmap_tlb_miss_lock);
+}
+
+static inline void
+pmap_tlb_miss_lock_exit(void)
+{
+       mutex_spin_exit(&pmap_tlb_miss_lock);
+}
+#endif /* PMAP_MD_NEED_TLB_MISS_LOCK */
+
+#else
+
+#define        pmap_tlb_miss_lock_init()       __nothing
+#define        pmap_tlb_miss_lock_enter()      __nothing
+#define        pmap_tlb_miss_lock_exit()       __nothing
+
+#endif /* PMAP_NEED_TLB_MISS_LOCK */
 
 #ifndef MULTIPROCESSOR
 kmutex_t pmap_pvlist_mutex     __cacheline_aligned;
@@ -522,6 +561,16 @@
 }
 
 /*
+ *     Bootstrap the system enough to run with virtual memory.
+ *     (Common routine called by machine-dependent bootstrap code.)
+ */
+void
+pmap_bootstrap_common(void)
+{
+       pmap_tlb_miss_lock_init();
+}
+
+/*
  *     Initialize the pmap module.
  *     Called by vm_init, to initialize any structures that the pmap
  *     system needs to map virtual memory.
@@ -621,10 +670,10 @@
        PMAP_COUNT(destroy);
        KASSERT(pmap->pm_count == 0);
        kpreempt_disable();
-       pmap_md_tlb_miss_lock_enter();
+       pmap_tlb_miss_lock_enter();
        pmap_tlb_asid_release_all(pmap);
        pmap_segtab_destroy(pmap, NULL, 0);
-       pmap_md_tlb_miss_lock_exit();
+       pmap_tlb_miss_lock_exit();
 
 #ifdef MULTIPROCESSOR
        kcpuset_destroy(pmap->pm_active);
@@ -670,12 +719,12 @@
        PMAP_COUNT(activate);
 
        kpreempt_disable();
-       pmap_md_tlb_miss_lock_enter();
+       pmap_tlb_miss_lock_enter();
        pmap_tlb_asid_acquire(pmap, l);
        if (l == curlwp) {
                pmap_segtab_activate(pmap, l);
        }
-       pmap_md_tlb_miss_lock_exit();
+       pmap_tlb_miss_lock_exit();
        kpreempt_enable();
 
        UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid,