Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/arch/alpha When removing mappings, hang PV entries to be...



details:   https://anonhg.NetBSD.org/src/rev/bc9e5022dd21
branches:  trunk
changeset: 983621:bc9e5022dd21
user:      thorpej <thorpej%NetBSD.org@localhost>
date:      Sun May 30 14:06:37 2021 +0000

description:
When removing mappings, hang PV entries to be freed off of the
pmap_tlb_context structure, and free them back in bulk after we
release all of our locks (as we do with PT pages that are freed).

diffstat:

 sys/arch/alpha/alpha/pmap.c   |  183 ++++++++++++++++++++++++-----------------
 sys/arch/alpha/include/pmap.h |    5 +-
 2 files changed, 109 insertions(+), 79 deletions(-)

diffs (truncated from 333 to 300 lines):

diff -r 522028f8bfba -r bc9e5022dd21 sys/arch/alpha/alpha/pmap.c
--- a/sys/arch/alpha/alpha/pmap.c       Sun May 30 13:34:21 2021 +0000
+++ b/sys/arch/alpha/alpha/pmap.c       Sun May 30 14:06:37 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.288 2021/05/30 13:34:21 thorpej Exp $ */
+/* $NetBSD: pmap.c,v 1.289 2021/05/30 14:06:37 thorpej Exp $ */
 
 /*-
  * Copyright (c) 1998, 1999, 2000, 2001, 2007, 2008, 2020
@@ -135,7 +135,7 @@
 
 #include <sys/cdefs.h>                 /* RCS ID & Copyright macro defns */
 
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.288 2021/05/30 13:34:21 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.289 2021/05/30 14:06:37 thorpej Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -432,6 +432,71 @@
 #endif /* MULTIPROCESSOR */
 
 /*
+ * TLB context structure; see description in "TLB management" section
+ * below.
+ */
+#define        TLB_CTX_MAXVA           8
+#define        TLB_CTX_ALLVA           PAGE_MASK
+struct pmap_tlb_context {
+       uintptr_t               t_addrdata[TLB_CTX_MAXVA];
+       pmap_t                  t_pmap;
+       struct pmap_pagelist    t_freeptq;
+       struct pmap_pvlist      t_freepvq;
+};
+
+/*
+ * Internal routines
+ */
+static void    alpha_protection_init(void);
+static pt_entry_t pmap_remove_mapping(pmap_t, vaddr_t, pt_entry_t *, bool,
+                                     pv_entry_t *,
+                                     struct pmap_tlb_context *);
+static void    pmap_changebit(struct vm_page *, pt_entry_t, pt_entry_t,
+                              struct pmap_tlb_context *);
+
+/*
+ * PT page management functions.
+ */
+static int     pmap_ptpage_alloc(pmap_t, pt_entry_t *, int);
+static void    pmap_ptpage_free(pmap_t, pt_entry_t *,
+                                struct pmap_tlb_context *);
+static void    pmap_l3pt_delref(pmap_t, vaddr_t, pt_entry_t *,
+                    struct pmap_tlb_context *);
+static void    pmap_l2pt_delref(pmap_t, pt_entry_t *, pt_entry_t *,
+                    struct pmap_tlb_context *);
+static void    pmap_l1pt_delref(pmap_t, pt_entry_t *);
+
+static void    *pmap_l1pt_alloc(struct pool *, int);
+static void    pmap_l1pt_free(struct pool *, void *);
+
+static struct pool_allocator pmap_l1pt_allocator = {
+       pmap_l1pt_alloc, pmap_l1pt_free, 0,
+};
+
+static int     pmap_l1pt_ctor(void *, void *, int);
+
+/*
+ * PV table management functions.
+ */
+static int     pmap_pv_enter(pmap_t, struct vm_page *, vaddr_t, pt_entry_t *,
+                             bool, pv_entry_t);
+static void    pmap_pv_remove(pmap_t, struct vm_page *, vaddr_t, bool,
+                              pv_entry_t *, struct pmap_tlb_context *);
+static void    *pmap_pv_page_alloc(struct pool *, int);
+static void    pmap_pv_page_free(struct pool *, void *);
+
+static struct pool_allocator pmap_pv_page_allocator = {
+       pmap_pv_page_alloc, pmap_pv_page_free, 0,
+};
+
+#ifdef DEBUG
+void   pmap_pv_dump(paddr_t);
+#endif
+
+#define        pmap_pv_alloc()         pool_cache_get(&pmap_pv_cache, PR_NOWAIT)
+#define        pmap_pv_free(pv)        pool_cache_put(&pmap_pv_cache, (pv))
+
+/*
  * Generic routine for freeing pages on a pmap_pagelist back to
  * the system.
  */
@@ -447,6 +512,21 @@
 }
 
 /*
+ * Generic routine for freeing a list of PV entries back to the
+ * system.
+ */
+static void
+pmap_pvlist_free(struct pmap_pvlist * const list)
+{
+       pv_entry_t pv;
+
+       while ((pv = LIST_FIRST(list)) != NULL) {
+               LIST_REMOVE(pv, pv_link);
+               pmap_pv_free(pv);
+       }
+}
+
+/*
  * TLB management.
  *
  * TLB invalidations need to be performed on local and remote CPUs
@@ -518,9 +598,6 @@
  * window size (defined as 64KB on alpha in <machine/vmparam.h>).
  */
 
-#define        TLB_CTX_MAXVA           8
-#define        TLB_CTX_ALLVA           PAGE_MASK
-
 #define        TLB_CTX_F_ASM           __BIT(0)
 #define        TLB_CTX_F_IMB           __BIT(1)
 #define        TLB_CTX_F_KIMB          __BIT(2)
@@ -538,12 +615,6 @@
 #define        TLB_CTX_SETVA(ctx, i, va)                                       \
        (ctx)->t_addrdata[(i)] = (va) | ((ctx)->t_addrdata[(i)] & PAGE_MASK)
 
-struct pmap_tlb_context {
-       uintptr_t       t_addrdata[TLB_CTX_MAXVA];
-       pmap_t          t_pmap;
-       struct pmap_pagelist t_freeptq;
-};
-
 static struct {
        kmutex_t        lock;
        struct evcnt    events;
@@ -689,6 +760,7 @@
        tlbctx->t_addrdata[1] = flags;
        tlbctx->t_pmap = NULL;
        LIST_INIT(&tlbctx->t_freeptq);
+       LIST_INIT(&tlbctx->t_freepvq);
 }
 
 static void
@@ -1081,65 +1153,18 @@
 }
 #endif /* MULTIPROCESSOR */
 
-static __inline void
-pmap_tlb_ptpage_drain(struct pmap_tlb_context * const tlbctx)
+static inline void
+pmap_tlb_context_drain(struct pmap_tlb_context * const tlbctx)
 {
-       pmap_pagelist_free(&tlbctx->t_freeptq);
+       if (! LIST_EMPTY(&tlbctx->t_freeptq)) {
+               pmap_pagelist_free(&tlbctx->t_freeptq);
+       }
+       if (! LIST_EMPTY(&tlbctx->t_freepvq)) {
+               pmap_pvlist_free(&tlbctx->t_freepvq);
+       }
 }
 
 /*
- * Internal routines
- */
-static void    alpha_protection_init(void);
-static pt_entry_t pmap_remove_mapping(pmap_t, vaddr_t, pt_entry_t *, bool,
-                                     pv_entry_t *,
-                                     struct pmap_tlb_context *);
-static void    pmap_changebit(struct vm_page *, pt_entry_t, pt_entry_t,
-                              struct pmap_tlb_context *);
-
-/*
- * PT page management functions.
- */
-static int     pmap_ptpage_alloc(pmap_t, pt_entry_t *, int);
-static void    pmap_ptpage_free(pmap_t, pt_entry_t *,
-                                struct pmap_tlb_context *);
-static void    pmap_l3pt_delref(pmap_t, vaddr_t, pt_entry_t *,
-                    struct pmap_tlb_context *);
-static void    pmap_l2pt_delref(pmap_t, pt_entry_t *, pt_entry_t *,
-                    struct pmap_tlb_context *);
-static void    pmap_l1pt_delref(pmap_t, pt_entry_t *);
-
-static void    *pmap_l1pt_alloc(struct pool *, int);
-static void    pmap_l1pt_free(struct pool *, void *);
-
-static struct pool_allocator pmap_l1pt_allocator = {
-       pmap_l1pt_alloc, pmap_l1pt_free, 0,
-};
-
-static int     pmap_l1pt_ctor(void *, void *, int);
-
-/*
- * PV table management functions.
- */
-static int     pmap_pv_enter(pmap_t, struct vm_page *, vaddr_t, pt_entry_t *,
-                             bool, pv_entry_t);
-static void    pmap_pv_remove(pmap_t, struct vm_page *, vaddr_t, bool,
-                              pv_entry_t *);
-static void    *pmap_pv_page_alloc(struct pool *, int);
-static void    pmap_pv_page_free(struct pool *, void *);
-
-static struct pool_allocator pmap_pv_page_allocator = {
-       pmap_pv_page_alloc, pmap_pv_page_free, 0,
-};
-
-#ifdef DEBUG
-void   pmap_pv_dump(paddr_t);
-#endif
-
-#define        pmap_pv_alloc()         pool_cache_get(&pmap_pv_cache, PR_NOWAIT)
-#define        pmap_pv_free(pv)        pool_cache_put(&pmap_pv_cache, (pv))
-
-/*
  * ASN management functions.
  */
 static u_int   pmap_asn_alloc(pmap_t, struct cpu_info *);
@@ -1721,6 +1746,8 @@
                pmap_tlb_shootnow(tlbctx);
                /* kernel PT pages are never freed. */
                KASSERT(LIST_EMPTY(&tlbctx->t_freeptq));
+               /* ...but we might have freed PV entries. */
+               pmap_tlb_context_drain(tlbctx);
                TLB_COUNT(reason_remove_kernel);
 
                return;
@@ -1803,7 +1830,7 @@
        PMAP_MAP_TO_HEAD_UNLOCK();
        PMAP_UNLOCK(pmap);
        pmap_tlb_shootnow(tlbctx);
-       pmap_tlb_ptpage_drain(tlbctx);
+       pmap_tlb_context_drain(tlbctx);
        TLB_COUNT(reason_remove_user);
 }
 
@@ -1895,7 +1922,7 @@
        mutex_exit(lock);
        PMAP_HEAD_TO_MAP_UNLOCK();
        pmap_tlb_shootnow(&tlbctx);
-       pmap_tlb_ptpage_drain(&tlbctx);
+       pmap_tlb_context_drain(&tlbctx);
        TLB_COUNT(reason_page_protect_none);
 }
 
@@ -2007,7 +2034,7 @@
        pmap_l2pt_delref(pmap, l1pte, l2pte, &tlbctx);
        PMAP_UNLOCK(pmap);
        pmap_tlb_shootnow(&tlbctx);
-       pmap_tlb_ptpage_drain(&tlbctx);
+       pmap_tlb_context_drain(&tlbctx);
        TLB_COUNT(reason_enter_l2pt_delref);
 }
 
@@ -2035,7 +2062,7 @@
        pmap_l3pt_delref(pmap, va, pte, &tlbctx);
        PMAP_UNLOCK(pmap);
        pmap_tlb_shootnow(&tlbctx);
-       pmap_tlb_ptpage_drain(&tlbctx);
+       pmap_tlb_context_drain(&tlbctx);
        TLB_COUNT(reason_enter_l3pt_delref);
 }
 
@@ -2936,8 +2963,8 @@
 
 #ifdef DEBUG
        if (pmapdebug & (PDB_FOLLOW|PDB_REMOVE|PDB_PROTECT))
-               printf("pmap_remove_mapping(%p, %lx, %p, %d, %p)\n",
-                      pmap, va, pte, dolock, opvp);
+               printf("pmap_remove_mapping(%p, %lx, %p, %d, %p, %p)\n",
+                      pmap, va, pte, dolock, opvp, tlbctx);
 #endif
 
        /*
@@ -2988,7 +3015,7 @@
                 */
                pg = PHYS_TO_VM_PAGE(pa);
                KASSERT(pg != NULL);
-               pmap_pv_remove(pmap, pg, va, dolock, opvp);
+               pmap_pv_remove(pmap, pg, va, dolock, opvp, tlbctx);
                KASSERT(opvp == NULL || *opvp != NULL);
        }
 
@@ -3296,7 +3323,7 @@
  */
 static void
 pmap_pv_remove(pmap_t pmap, struct vm_page *pg, vaddr_t va, bool dolock,
-       pv_entry_t *opvp)
+    pv_entry_t *opvp, struct pmap_tlb_context * const tlbctx)
 {
        struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
        pv_entry_t pv, *pvp;
@@ -3333,10 +3360,12 @@
                mutex_exit(lock);
        }
 
-       if (opvp != NULL)
+       if (opvp != NULL) {
                *opvp = pv;
-       else
-               pmap_pv_free(pv);



Home | Main Index | Thread Index | Old Index