Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys Avoid race condition between PTE update and TLB miss walk.



details:   https://anonhg.NetBSD.org/src/rev/ab61974449ad
branches:  trunk
changeset: 805990:ab61974449ad
user:      nonaka <nonaka%NetBSD.org@localhost>
date:      Mon Jan 26 04:47:53 2015 +0000

description:
Avoid race condition between PTE update and TLB miss walk.

diffstat:

 sys/arch/powerpc/booke/booke_pmap.c   |  27 +++++++++++++++-
 sys/arch/powerpc/booke/genassym.cf    |   5 ++-
 sys/arch/powerpc/booke/trap_subr.S    |  55 +++++++++++++++++++++++++++++++++-
 sys/arch/powerpc/include/booke/pmap.h |   8 ++++-
 sys/uvm/pmap/pmap.c                   |  35 +++++++++++++++++++++-
 5 files changed, 122 insertions(+), 8 deletions(-)

diffs (truncated from 407 to 300 lines):

diff -r cfc65b63f2bc -r ab61974449ad sys/arch/powerpc/booke/booke_pmap.c
--- a/sys/arch/powerpc/booke/booke_pmap.c       Mon Jan 26 04:12:46 2015 +0000
+++ b/sys/arch/powerpc/booke/booke_pmap.c       Mon Jan 26 04:47:53 2015 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: booke_pmap.c,v 1.21 2015/01/23 06:39:41 nonaka Exp $   */
+/*     $NetBSD: booke_pmap.c,v 1.22 2015/01/26 04:47:53 nonaka Exp $   */
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -38,16 +38,21 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: booke_pmap.c,v 1.21 2015/01/23 06:39:41 nonaka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: booke_pmap.c,v 1.22 2015/01/26 04:47:53 nonaka Exp $");
 
 #include <sys/param.h>
 #include <sys/kcore.h>
 #include <sys/buf.h>
+#include <sys/mutex.h>
 
 #include <uvm/uvm.h>
 
 #include <machine/pmap.h>
 
+#if defined(MULTIPROCESSOR)
+kmutex_t pmap_tlb_miss_lock;
+#endif
+
 /*
  * Initialize the kernel pmap.
  */
@@ -166,6 +171,10 @@
        /* init the lock */
        pmap_tlb_info_init(&pmap_tlb0_info);
 
+#if defined(MULTIPROCESSOR)
+       mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH);
+#endif
+
        /*
         * Compute the number of pages kmem_arena will have.
         */
@@ -427,4 +436,18 @@
 {
        /* nothing */
 }
+
+void
+pmap_md_tlb_miss_lock_enter(void)
+{
+
+       mutex_spin_enter(&pmap_tlb_miss_lock);
+}
+
+void
+pmap_md_tlb_miss_lock_exit(void)
+{
+
+       mutex_spin_exit(&pmap_tlb_miss_lock);
+}
 #endif /* MULTIPROCESSOR */
diff -r cfc65b63f2bc -r ab61974449ad sys/arch/powerpc/booke/genassym.cf
--- a/sys/arch/powerpc/booke/genassym.cf        Mon Jan 26 04:12:46 2015 +0000
+++ b/sys/arch/powerpc/booke/genassym.cf        Mon Jan 26 04:47:53 2015 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: genassym.cf,v 1.10 2012/11/27 19:24:46 matt Exp $
+#      $NetBSD: genassym.cf,v 1.11 2015/01/26 04:47:53 nonaka Exp $
 
 #-
 # Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
@@ -111,4 +111,7 @@
 define HATCH_TBU               offsetof(struct cpu_hatch_data, hatch_tbu)
 define HATCH_TBL               offsetof(struct cpu_hatch_data, hatch_tbl)
 define HATCH_TLBIDX            offsetof(struct cpu_hatch_data, hatch_tlbidx)
+
+define __SIMPLELOCK_LOCKED     __SIMPLELOCK_LOCKED
+define __SIMPLELOCK_UNLOCKED   __SIMPLELOCK_UNLOCKED
 endif
diff -r cfc65b63f2bc -r ab61974449ad sys/arch/powerpc/booke/trap_subr.S
--- a/sys/arch/powerpc/booke/trap_subr.S        Mon Jan 26 04:12:46 2015 +0000
+++ b/sys/arch/powerpc/booke/trap_subr.S        Mon Jan 26 04:47:53 2015 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: trap_subr.S,v 1.11 2014/09/18 23:37:51 joerg Exp $     */
+/*     $NetBSD: trap_subr.S,v 1.12 2015/01/26 04:47:53 nonaka Exp $    */
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -34,7 +34,7 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-RCSID("$NetBSD: trap_subr.S,v 1.11 2014/09/18 23:37:51 joerg Exp $")
+RCSID("$NetBSD: trap_subr.S,v 1.12 2015/01/26 04:47:53 nonaka Exp $")
 
        .globl  _C_LABEL(sctrapexit), _C_LABEL(trapexit), _C_LABEL(intrcall)
 
@@ -346,6 +346,50 @@
        RESTORE_SPRG1(%r6);     \
        FRAME_INTR_XEXIT(rfci, CSRR)
 
+#if defined(MULTIPROCESSOR)
+#define        FRAME_TLBMISSLOCK                                               \
+       GET_CPUINFO(%r23);                                              \
+       ldint   %r22, CI_MTX_COUNT(%r23);                               \
+       subi    %r22, %r22, 1;                                          \
+       stint   %r22, CI_MTX_COUNT(%r23);                               \
+       isync;                                                          \
+       cmpwi   %r22, 0;                                                \
+       bne     1f;                                                     \
+       ldint   %r22, CI_CPL(%r23);                                     \
+       stint   %r22, CI_MTX_OLDSPL(%r23);                              \
+1:     lis     %r23, _C_LABEL(pmap_tlb_miss_lock)@h;                   \
+       ori     %r23, %r23, _C_LABEL(pmap_tlb_miss_lock)@l;             \
+       li      %r20, MTX_LOCK;                                         \
+2:     lwarx   %r22, %r20, %r23;                                       \
+       cmpwi   %r22, __SIMPLELOCK_UNLOCKED;                            \
+       beq+    4f;                                                     \
+3:     lwzx    %r22, %r20, %r23;                                       \
+       cmpwi   %r22, __SIMPLELOCK_UNLOCKED;                            \
+       beq+    2b;                                                     \
+       b       3b;                                                     \
+4:     li      %r21, __SIMPLELOCK_LOCKED;                              \
+       stwcx.  %r21, %r20, %r23;                                       \
+       bne-    2b;                                                     \
+       isync;                                                          \
+       msync;
+#define        FRAME_TLBMISSUNLOCK                                             \
+       sync;                                                           \
+       lis     %r23, _C_LABEL(pmap_tlb_miss_lock)@h;                   \
+       ori     %r23, %r23, _C_LABEL(pmap_tlb_miss_lock)@l;             \
+       li      %r22, __SIMPLELOCK_UNLOCKED;                            \
+       stw     %r22, MTX_LOCK(%r23);                                   \
+       isync;                                                          \
+       msync;                                                          \
+       GET_CPUINFO(%r23);                                              \
+       ldint   %r22, CI_MTX_COUNT(%r23);                               \
+       addi    %r22, %r22, 1;                                          \
+       stint   %r22, CI_MTX_COUNT(%r23);                               \
+       isync;
+#else  /* !MULTIPROCESSOR */
+#define        FRAME_TLBMISSLOCK
+#define        FRAME_TLBMISSUNLOCK
+#endif /* MULTIPROCESSOR */
+
        .text
        .p2align 4
 _C_LABEL(critical_input_vector):
@@ -535,6 +579,7 @@
 _C_LABEL(data_tlb_error_vector):
        /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */
        FRAME_TLBPROLOGUE
+       FRAME_TLBMISSLOCK
        /*
         * Registers as this point:
         *
@@ -577,6 +622,7 @@
                31-PTR_SCALESHIFT, \
                31-PTR_SCALESHIFT               /* move PSL_DS[27] to bit 29 */
        bl      pte_load
+       FRAME_TLBMISSUNLOCK
        mtlr    %r29                            /* restore LR */
        /*
         * If we returned, pte load failed so let trap deal with it but
@@ -590,6 +636,7 @@
 _C_LABEL(instruction_tlb_error_vector):
        /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */
        FRAME_TLBPROLOGUE
+       FRAME_TLBMISSLOCK
        /*
         * Attempt to update the TLB from the page table.
         */
@@ -600,6 +647,7 @@
                31-PTR_SCALESHIFT, \
                31-PTR_SCALESHIFT               /* move PSL_IS[26] to bit 29 */
        bl      pte_load
+       FRAME_TLBMISSUNLOCK
        mtlr    %r29                            /* restore LR */
        /*
         * If we returned, pte load failed so let trap deal with it but
@@ -764,6 +812,9 @@
        addic   %r31, %r31, 1
        addze   %r30, %r30
        stmw    %r30, CI_EV_TLBMISS_SOFT(%r2)
+
+       FRAME_TLBMISSUNLOCK
+
        /*
         * Cleanup and leave.  We know any higher priority exception will
         * save and restore SPRG1 and %r2 thereby preserving their values.
diff -r cfc65b63f2bc -r ab61974449ad sys/arch/powerpc/include/booke/pmap.h
--- a/sys/arch/powerpc/include/booke/pmap.h     Mon Jan 26 04:12:46 2015 +0000
+++ b/sys/arch/powerpc/include/booke/pmap.h     Mon Jan 26 04:47:53 2015 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap.h,v 1.14 2014/04/03 13:55:34 matt Exp $   */
+/*     $NetBSD: pmap.h,v 1.15 2015/01/26 04:47:53 nonaka Exp $ */
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -92,6 +92,12 @@
 
 bool   pmap_md_tlb_check_entry(void *, vaddr_t, tlb_asid_t, pt_entry_t);
 
+#ifdef MULTIPROCESSOR
+#define        PMAP_MD_NEED_TLB_MISS_LOCK
+void   pmap_md_tlb_miss_lock_enter(void);
+void   pmap_md_tlb_miss_lock_exit(void);
+#endif /* MULTIPROCESSOR */
+
 #ifdef PMAP_MINIMALTLB
 vaddr_t        pmap_kvptefill(vaddr_t, vaddr_t, pt_entry_t);
 #endif
diff -r cfc65b63f2bc -r ab61974449ad sys/uvm/pmap/pmap.c
--- a/sys/uvm/pmap/pmap.c       Mon Jan 26 04:12:46 2015 +0000
+++ b/sys/uvm/pmap/pmap.c       Mon Jan 26 04:47:53 2015 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap.c,v 1.9 2015/01/05 05:35:18 nonaka Exp $  */
+/*     $NetBSD: pmap.c,v 1.10 2015/01/26 04:47:53 nonaka Exp $ */
 
 /*-
  * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
@@ -67,7 +67,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.9 2015/01/05 05:35:18 nonaka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.10 2015/01/26 04:47:53 nonaka Exp $");
 
 /*
  *     Manages physical address maps.
@@ -263,6 +263,11 @@
 #define        pmap_pv_alloc()         pool_get(&pmap_pv_pool, PR_NOWAIT)
 #define        pmap_pv_free(pv)        pool_put(&pmap_pv_pool, (pv))
 
+#if !defined(MULTIPROCESSOR) || !defined(PMAP_MD_NEED_TLB_MISS_LOCK)
+#define        pmap_md_tlb_miss_lock_enter()   do { } while(/*CONSTCOND*/0)
+#define        pmap_md_tlb_miss_lock_exit()    do { } while(/*CONSTCOND*/0)
+#endif /* !MULTIPROCESSOR || !PMAP_MD_NEED_TLB_MISS_LOCK */
+
 /*
  * Misc. functions.
  */
@@ -541,8 +546,10 @@
        KASSERT(pmap->pm_count == 0);
        PMAP_COUNT(destroy);
        kpreempt_disable();
+       pmap_md_tlb_miss_lock_enter();
        pmap_tlb_asid_release_all(pmap);
        pmap_segtab_destroy(pmap, NULL, 0);
+       pmap_md_tlb_miss_lock_exit();
 
 #ifdef MULTIPROCESSOR
        kcpuset_destroy(pmap->pm_active);
@@ -586,10 +593,12 @@
        PMAP_COUNT(activate);
 
        kpreempt_disable();
+       pmap_md_tlb_miss_lock_enter();
        pmap_tlb_asid_acquire(pmap, l);
        if (l == curlwp) {
                pmap_segtab_activate(pmap, l);
        }
+       pmap_md_tlb_miss_lock_exit();
        kpreempt_enable();
 
        UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0);
@@ -608,8 +617,10 @@
        PMAP_COUNT(deactivate);
 
        kpreempt_disable();
+       pmap_md_tlb_miss_lock_enter();
        curcpu()->ci_pmap_user_segtab = PMAP_INVALID_SEGTAB_ADDRESS;
        pmap_tlb_asid_deactivate(pmap);
+       pmap_md_tlb_miss_lock_exit();
        kpreempt_enable();
 
        UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0);
@@ -629,6 +640,7 @@
        if (pending && pmap_tlb_shootdown_bystanders(pmap))
                PMAP_COUNT(shootdown_ipis);
 #endif
+       pmap_md_tlb_miss_lock_enter();
 #ifdef DEBUG
        pmap_tlb_check(pmap, pmap_md_tlb_check_entry);
 #endif /* DEBUG */
@@ -642,6 +654,7 @@
                pmap_tlb_asid_acquire(pmap, curlwp);
                pmap_segtab_activate(pmap, curlwp);
        }
+       pmap_md_tlb_miss_lock_exit();
        kpreempt_enable();
 
        UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0);
@@ -685,11 +698,13 @@
                        pmap_remove_pv(pmap, sva, pg,
                           pte_modified_p(pt_entry));
                }
+               pmap_md_tlb_miss_lock_enter();
                *ptep = npte;
                /*



Home | Main Index | Thread Index | Old Index