Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/arch/hppa To allow concurrent page faults on the same se...



details:   https://anonhg.NetBSD.org/src/rev/45add3d9af40
branches:  trunk
changeset: 1009204:45add3d9af40
user:      skrll <skrll%NetBSD.org@localhost>
date:      Thu Apr 16 09:51:56 2020 +0000

description:
To allow concurrent page faults on the same set of pages lock the PV
lists.  From ad@ and fixed up by me.

Remove __HAVE_UNLOCKED_PMAP

diffstat:

 sys/arch/hppa/hppa/pmap.c     |  223 ++++++++++++++++++++++++++++++++---------
 sys/arch/hppa/include/types.h |    5 +-
 2 files changed, 172 insertions(+), 56 deletions(-)

diffs (truncated from 563 to 300 lines):

diff -r 418fbd7a2878 -r 45add3d9af40 sys/arch/hppa/hppa/pmap.c
--- a/sys/arch/hppa/hppa/pmap.c Thu Apr 16 09:51:40 2020 +0000
+++ b/sys/arch/hppa/hppa/pmap.c Thu Apr 16 09:51:56 2020 +0000
@@ -1,7 +1,7 @@
-/*     $NetBSD: pmap.c,v 1.110 2020/04/16 05:22:59 skrll Exp $ */
+/*     $NetBSD: pmap.c,v 1.111 2020/04/16 09:51:56 skrll Exp $ */
 
 /*-
- * Copyright (c) 2001, 2002 The NetBSD Foundation, Inc.
+ * Copyright (c) 2001, 2002, 2020 The NetBSD Foundation, Inc.
  * All rights reserved.
  *
  * This code is derived from software contributed to The NetBSD Foundation
@@ -65,12 +65,14 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.110 2020/04/16 05:22:59 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.111 2020/04/16 09:51:56 skrll Exp $");
 
 #include "opt_cputype.h"
 
 #include <sys/param.h>
+#include <sys/atomic.h>
 #include <sys/mutex.h>
+#include <sys/pool.h>
 #include <sys/proc.h>
 #include <sys/rwlock.h>
 #include <sys/systm.h>
@@ -108,6 +110,14 @@
 
 static kmutex_t        pmaps_lock;
 
+static union pmap_pv_locks {
+       kmutex_t        lock;
+       char            padding[COHERENCY_UNIT];
+} pmap_pv_locks[64] __aligned(COHERENCY_UNIT);
+
+#define        PMAP_PV_LOCK(md) \
+    ((uintptr_t)(md) >> 7 & (__arraycount(pmap_pv_locks) - 1))
+
 u_int  hppa_prot[8];
 u_int  sid_counter;
 
@@ -140,6 +150,10 @@
 struct vm_page *pmap_pagealloc(struct uvm_object *, voff_t);
 void pmap_pagefree(struct vm_page *);
 
+static inline void pmap_lock(struct pmap *);
+static inline void pmap_unlock(struct pmap *);
+static inline bool pmap_trylock(struct pmap *);
+
 static inline void pmap_sdir_set(pa_space_t, volatile uint32_t *);
 static inline uint32_t *pmap_sdir_get(pa_space_t);
 
@@ -165,6 +179,9 @@
     vaddr_t , struct vm_page *, u_int);
 static inline struct pv_entry *pmap_pv_remove(struct vm_page *, pmap_t,
     vaddr_t);
+static inline void pmap_pv_lock(const struct vm_page_md *md);
+static inline void pmap_pv_unlock(const struct vm_page_md *md);
+static inline bool pmap_pv_locked(const struct vm_page_md *md);
 
 static inline void pmap_flush_page(struct vm_page *, bool);
 static int pmap_check_alias(struct vm_page *, vaddr_t, pt_entry_t);
@@ -195,17 +212,47 @@
 #define pmap_pvh_attrs(a) \
        (((a) & (PVF_MOD|PVF_REF)) ^ PVF_REF)
 
-#define PMAP_LOCK(pm)                                          \
-       do {                                                    \
-               if ((pm) != pmap_kernel())                      \
-                       rw_enter((pm)->pm_lock, RW_WRITER);     \
-       } while (/*CONSTCOND*/0)
+static inline void
+pmap_lock(struct pmap *pm)
+{
+
+       rw_enter(pm->pm_lock, RW_WRITER);
+}
+
+static inline void
+pmap_unlock(struct pmap *pm)
+{
+
+       rw_exit(pm->pm_lock);
+}
+
+static inline bool
+pmap_trylock(struct pmap *pm)
+{
+
+       return rw_tryenter(pm->pm_lock, RW_WRITER);
+}
 
-#define PMAP_UNLOCK(pm)                                                \
-       do {                                                    \
-               if ((pm) != pmap_kernel())                      \
-                       rw_exit((pm)->pm_lock);                 \
-       } while (/*CONSTCOND*/0)
+static inline void
+pmap_pv_lock(const struct vm_page_md *md)
+{
+
+       mutex_enter(&pmap_pv_locks[PMAP_PV_LOCK(md)].lock);
+}
+
+static inline void
+pmap_pv_unlock(const struct vm_page_md *md)
+{
+
+       mutex_exit(&pmap_pv_locks[PMAP_PV_LOCK(md)].lock);
+}
+
+static inline bool
+pmap_pv_locked(const struct vm_page_md *md)
+{
+
+       return mutex_owned(&pmap_pv_locks[PMAP_PV_LOCK(md)].lock);
+}
 
 struct vm_page *
 pmap_pagealloc(struct uvm_object *obj, voff_t off)
@@ -304,6 +351,7 @@
        UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx pdep %#jx", (uintptr_t)pm,
            va, (uintptr_t)pdep, 0);
 
+
        KASSERT(pm != pmap_kernel());
        KASSERT(rw_write_held(pm->pm_lock));
 
@@ -559,7 +607,7 @@
        UVMHIST_LOG(maphist, "...pdep %#jx flags %#jx",
            (uintptr_t)pdep, flags, 0, 0);
 
-       KASSERT(pm == pmap_kernel() || uvm_page_owner_locked_p(pg, true));
+       KASSERT(pmap_pv_locked(md));
 
        pve->pv_pmap = pm;
        pve->pv_va = va | flags;
@@ -578,7 +626,7 @@
        struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
        struct pv_entry **pve, *pv;
 
-       KASSERT(pmap == pmap_kernel() || uvm_page_owner_locked_p(pg, true));
+       KASSERT(pmap_pv_locked(md));
 
        for (pv = *(pve = &md->pvh_list);
            pv; pv = *(pve = &(*pve)->pv_next)) {
@@ -967,6 +1015,7 @@
 {
        extern void gateway_page(void);
        volatile pt_entry_t *pde;
+       int i;
 
        UVMHIST_FUNC(__func__)
        UVMHIST_CALLED(maphist);
@@ -995,6 +1044,9 @@
        pmap_pte_set(pde, SYSCALLGATE, (paddr_t)&gateway_page |
            PTE_PROT(TLB_GATE_PROT));
 
+       for (i = 0; i < __arraycount(pmap_pv_locks); i++)
+               mutex_init(&pmap_pv_locks[i].lock, MUTEX_DEFAULT, IPL_VM);
+
        pmap_initialized = true;
 
        UVMHIST_LOG(maphist, "<--- done", 0, 0, 0, 0);
@@ -1082,13 +1134,8 @@
        struct vm_page *pg;
        off_t off;
 #endif
-       int refs;
 
-       rw_enter(pmap->pm_lock, RW_WRITER);
-       refs = --pmap->pm_obj.uo_refs;
-       rw_exit(pmap->pm_lock);
-
-       if (refs > 0)
+       if (atomic_dec_uint_nv(&pmap->pm_obj.uo_refs) > 0)
                return;
 
 #ifdef DIAGNOSTIC
@@ -1101,6 +1148,7 @@
                struct vm_page *spg;
                struct pv_entry *pv, *npv;
                paddr_t pa;
+               vaddr_t va;
 
                off = pg->offset + PAGE_SIZE;
                uvm_page_array_advance(&a);
@@ -1121,6 +1169,7 @@
                                continue;
 
                        struct vm_page_md * const md = VM_PAGE_TO_MD(spg);
+                       pmap_pv_lock(md);
                        for (pv = md->pvh_list; pv != NULL; pv = npv) {
                                npv = pv->pv_next;
                                if (pv->pv_pmap != pmap)
@@ -1129,9 +1178,14 @@
                                UVMHIST_LOG(maphist, " %#jx", pv->pv_va, 0, 0,
                                    0);
 
-                               pmap_remove(pmap, pv->pv_va & PV_VAMASK,
-                                   (pv->pv_va & PV_VAMASK) + PAGE_SIZE);
+                               va = pv->pv_va & PV_VAMASK;
+                               pmap_pv_unlock(md);
+                               pmap_remove(pmap, va, va + PAGE_SIZE);
+                               pmap_pv_lock(md);
+                               /* List may have changed: restart. */
+                               npv = md->pvh_list;
                        }
+                       pmap_pv_unlock(md);
                }
        }
        rw_exit(pmap->pm_lock);
@@ -1156,9 +1210,7 @@
        UVMHIST_FUNC(__func__)
        UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pmap, 0, 0, 0);
 
-       rw_enter(pmap->pm_lock, RW_WRITER);
-       pmap->pm_obj.uo_refs++;
-       rw_exit(pmap->pm_lock);
+       atomic_inc_uint(&pmap->pm_obj.uo_refs);
 }
 
 
@@ -1207,12 +1259,12 @@
            (uintptr_t)pmap, va, pa, prot);
        UVMHIST_LOG(maphist, "...flags %#jx", flags, 0, 0, 0);
 
-       PMAP_LOCK(pmap);
+       pmap_lock(pmap);
 
        if (!(pde = pmap_pde_get(pmap->pm_pdir, va)) &&
            !(pde = pmap_pde_alloc(pmap, va, &ptp))) {
                if (flags & PMAP_CANFAIL) {
-                       PMAP_UNLOCK(pmap);
+                       pmap_unlock(pmap);
                        return (ENOMEM);
                }
 
@@ -1238,10 +1290,12 @@
                }
 
                if (pg != NULL) {
-                       pve = pmap_pv_remove(pg, pmap, va);
+                       struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 
-                       struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
+                       pmap_pv_lock(md);
+                       pve = pmap_pv_remove(pg, pmap, va);
                        md->pvh_attrs |= pmap_pvh_attrs(pte);
+                       pmap_pv_unlock(md);
                }
        } else {
                UVMHIST_LOG(maphist, "new mapping %#jx -> %#jx",
@@ -1255,9 +1309,11 @@
        }
 
        if (pmap_initialized && (pg = PHYS_TO_VM_PAGE(pa))) {
+               struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
+
                if (!pve && !(pve = pmap_pv_alloc())) {
                        if (flags & PMAP_CANFAIL) {
-                               PMAP_UNLOCK(pmap);
+                               pmap_unlock(pmap);
                                return (ENOMEM);
                        }
                        panic("%s: no pv entries available", __func__);
@@ -1265,7 +1321,9 @@
                pte |= PTE_PROT(pmap_prot(pmap, prot));
                if (pmap_check_alias(pg, va, pte))
                        pmap_page_remove(pg);
+               pmap_pv_lock(md);
                pmap_pv_enter(pg, pve, pmap, va, ptp, 0);
+               pmap_pv_unlock(md);
        } else if (pve) {
                pmap_pv_free(pve);
        }
@@ -1291,7 +1349,7 @@
                pte |= PTE_PROT(TLB_WIRED);
        pmap_pte_set(pde, va, pte);
 
-       PMAP_UNLOCK(pmap);
+       pmap_unlock(pmap);
 
        UVMHIST_LOG(maphist, "<--- done (0)", 0, 0, 0, 0);
 
@@ -1308,7 +1366,6 @@
 void
 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
 {
-
        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(maphist, "sva %#jx eva %#jx", sva, eva, 0, 0);
 
@@ -1319,7 +1376,7 @@
        vaddr_t pdemask;
        int batch;
 



Home | Main Index | Thread Index | Old Index