Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/uvm - uvmspace_exec(), uvmspace_free(): if pmap_remove_a...



details:   https://anonhg.NetBSD.org/src/rev/010d25398006
branches:  trunk
changeset: 970158:010d25398006
user:      ad <ad%NetBSD.org@localhost>
date:      Sat Mar 14 14:15:43 2020 +0000

description:
- uvmspace_exec(), uvmspace_free(): if pmap_remove_all() returns true the
  pmap is emptied.  Pass UVM_FLAG_VAONLY when clearing out the map and avoid
  needless extra work to tear down each mapping individually.

- uvm_map_lookup_entry(): remove the code to do a linear scan of map entries
  for small maps, in preference to using the RB tree.  It's questionable,
  and I think the code is almost never triggered because the average number
  of map entries has probably exceeded the hard-coded threshold for quite
  some time.

- vm_map_entry: get it aligned on a cacheline boundary, and cluster fields
  used during rbtree lookup at the beginning.

diffstat:

 sys/uvm/uvm_map.c |  125 +++++++++++++++--------------------------------------
 sys/uvm/uvm_map.h |   22 +++++---
 2 files changed, 48 insertions(+), 99 deletions(-)

diffs (280 lines):

diff -r 4a16152b6773 -r 010d25398006 sys/uvm/uvm_map.c
--- a/sys/uvm/uvm_map.c Sat Mar 14 14:05:42 2020 +0000
+++ b/sys/uvm/uvm_map.c Sat Mar 14 14:15:43 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_map.c,v 1.372 2020/02/23 15:46:43 ad Exp $ */
+/*     $NetBSD: uvm_map.c,v 1.373 2020/03/14 14:15:43 ad Exp $ */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -66,7 +66,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.372 2020/02/23 15:46:43 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.373 2020/03/14 14:15:43 ad Exp $");
 
 #include "opt_ddb.h"
 #include "opt_pax.h"
@@ -141,10 +141,8 @@
 UVMMAP_EVCNT_DEFINE(map_call)
 UVMMAP_EVCNT_DEFINE(mlk_call)
 UVMMAP_EVCNT_DEFINE(mlk_hint)
-UVMMAP_EVCNT_DEFINE(mlk_list)
 UVMMAP_EVCNT_DEFINE(mlk_tree)
 UVMMAP_EVCNT_DEFINE(mlk_treeloop)
-UVMMAP_EVCNT_DEFINE(mlk_listloop)
 
 const char vmmapbsy[] = "vmmapbsy";
 
@@ -823,8 +821,8 @@
 uvm_mapent_copy(struct vm_map_entry *src, struct vm_map_entry *dst)
 {
 
-       memcpy(dst, src, ((char *)&src->uvm_map_entry_stop_copy) -
-           ((char *)src));
+       memcpy(dst, src, sizeof(*dst));
+       dst->flags = 0;
 }
 
 #if defined(DEBUG)
@@ -940,7 +938,7 @@
         */
 
        pool_cache_bootstrap(&uvm_map_entry_cache, sizeof(struct vm_map_entry),
-           0, 0, 0, "vmmpepl", NULL, IPL_NONE, NULL, NULL, NULL);
+           coherency_unit, 0, 0, "vmmpepl", NULL, IPL_NONE, NULL, NULL, NULL);
        pool_cache_bootstrap(&uvm_vmspace_cache, sizeof(struct vmspace),
            0, 0, 0, "vmsppl", NULL, IPL_NONE, NULL, NULL, NULL);
 }
@@ -1679,7 +1677,6 @@
     struct vm_map_entry **entry        /* OUT */)
 {
        struct vm_map_entry *cur;
-       bool use_tree = false;
        UVMHIST_FUNC("uvm_map_lookup_entry");
        UVMHIST_CALLED(maphist);
 
@@ -1687,95 +1684,41 @@
            (uintptr_t)map, address, (uintptr_t)entry, 0);
 
        /*
-        * start looking either from the head of the
-        * list, or from the hint.
+        * make a quick check to see if we are already looking at
+        * the entry we want (which is usually the case).  note also
+        * that we don't need to save the hint here...  it is the
+        * same hint (unless we are at the header, in which case the
+        * hint didn't buy us anything anyway).
         */
 
        cur = map->hint;
-
-       if (cur == &map->header)
-               cur = cur->next;
-
        UVMMAP_EVCNT_INCR(mlk_call);
-       if (address >= cur->start) {
-
-               /*
-                * go from hint to end of list.
-                *
-                * but first, make a quick check to see if
-                * we are already looking at the entry we
-                * want (which is usually the case).
-                * note also that we don't need to save the hint
-                * here... it is the same hint (unless we are
-                * at the header, in which case the hint didn't
-                * buy us anything anyway).
-                */
-
-               if (cur != &map->header && cur->end > address) {
-                       UVMMAP_EVCNT_INCR(mlk_hint);
-                       *entry = cur;
-                       UVMHIST_LOG(maphist,"<- got it via hint (%#jx)",
-                           (uintptr_t)cur, 0, 0, 0);
-                       uvm_mapent_check(*entry);
-                       return (true);
-               }
-
-               if (map->nentries > 15)
-                       use_tree = true;
-       } else {
-
-               /*
-                * invalid hint.  use tree.
-                */
-               use_tree = true;
+       if (cur != &map->header &&
+           address >= cur->start && cur->end > address) {
+               UVMMAP_EVCNT_INCR(mlk_hint);
+               *entry = cur;
+               UVMHIST_LOG(maphist,"<- got it via hint (%#jx)",
+                   (uintptr_t)cur, 0, 0, 0);
+               uvm_mapent_check(*entry);
+               return (true);
        }
-
        uvm_map_check(map, __func__);
 
-       if (use_tree) {
-               /*
-                * Simple lookup in the tree.  Happens when the hint is
-                * invalid, or nentries reach a threshold.
-                */
-               UVMMAP_EVCNT_INCR(mlk_tree);
-               if (uvm_map_lookup_entry_bytree(map, address, entry)) {
-                       goto got;
-               } else {
-                       goto failed;
-               }
-       }
-
        /*
-        * search linearly
+        * lookup in the tree.
         */
 
-       UVMMAP_EVCNT_INCR(mlk_list);
-       while (cur != &map->header) {
-               UVMMAP_EVCNT_INCR(mlk_listloop);
-               if (cur->end > address) {
-                       if (address >= cur->start) {
-                               /*
-                                * save this lookup for future
-                                * hints, and return
-                                */
-
-                               *entry = cur;
-got:
-                               SAVE_HINT(map, map->hint, *entry);
-                               UVMHIST_LOG(maphist,"<- search got it (%#jx)",
-                                       (uintptr_t)cur, 0, 0, 0);
-                               KDASSERT((*entry)->start <= address);
-                               KDASSERT(address < (*entry)->end);
-                               uvm_mapent_check(*entry);
-                               return (true);
-                       }
-                       break;
-               }
-               cur = cur->next;
+       UVMMAP_EVCNT_INCR(mlk_tree);
+       if (__predict_true(uvm_map_lookup_entry_bytree(map, address, entry))) {
+               SAVE_HINT(map, map->hint, *entry);
+               UVMHIST_LOG(maphist,"<- search got it (%#jx)",
+                   (uintptr_t)cur, 0, 0, 0);
+               KDASSERT((*entry)->start <= address);
+               KDASSERT(address < (*entry)->end);
+               uvm_mapent_check(*entry);
+               return (true);
        }
-       *entry = cur->prev;
-failed:
-       SAVE_HINT(map, map->hint, *entry);
+
        UVMHIST_LOG(maphist,"<- failed!",0,0,0,0);
        KDASSERT((*entry) == &map->header || (*entry)->end <= address);
        KDASSERT((*entry)->next == &map->header ||
@@ -4176,6 +4119,7 @@
        struct proc *p = l->l_proc;
        struct vmspace *nvm, *ovm = p->p_vmspace;
        struct vm_map *map;
+       int flags;
 
        KASSERT(ovm != NULL);
 #ifdef __HAVE_CPU_VMSPACE_EXEC
@@ -4214,8 +4158,8 @@
                 * now unmap the old program
                 */
 
-               pmap_remove_all(map->pmap);
-               uvm_unmap(map, vm_map_min(map), vm_map_max(map));
+               flags = pmap_remove_all(map->pmap) ? UVM_FLAG_VAONLY : 0;
+               uvm_unmap1(map, vm_map_min(map), vm_map_max(map), flags);
                KASSERT(map->header.prev == &map->header);
                KASSERT(map->nentries == 0);
 
@@ -4271,6 +4215,7 @@
 {
        struct vm_map_entry *dead_entries;
        struct vm_map *map = &vm->vm_map;
+       int flags;
 
        UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist);
 
@@ -4285,7 +4230,7 @@
         */
 
        map->flags |= VM_MAP_DYING;
-       pmap_remove_all(map->pmap);
+       flags = pmap_remove_all(map->pmap) ? UVM_FLAG_VAONLY : 0;
 
        /* Get rid of any SYSV shared memory segments. */
        if (uvm_shmexit && vm->vm_shm != NULL)
@@ -4293,7 +4238,7 @@
 
        if (map->nentries) {
                uvm_unmap_remove(map, vm_map_min(map), vm_map_max(map),
-                   &dead_entries, 0);
+                   &dead_entries, flags);
                if (dead_entries != NULL)
                        uvm_unmap_detach(dead_entries, 0);
        }
diff -r 4a16152b6773 -r 010d25398006 sys/uvm/uvm_map.h
--- a/sys/uvm/uvm_map.h Sat Mar 14 14:05:42 2020 +0000
+++ b/sys/uvm/uvm_map.h Sat Mar 14 14:15:43 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_map.h,v 1.78 2020/02/23 15:46:43 ad Exp $  */
+/*     $NetBSD: uvm_map.h,v 1.79 2020/03/14 14:15:43 ad Exp $  */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -125,36 +125,40 @@
  * a VM object (or sharing map) and offset into that object,
  * and user-exported inheritance and protection information.
  * Also included is control information for virtual copy operations.
+ *
+ * At runtime this is aligned on a cacheline boundary, with fields
+ * used during fault processing to do RB tree lookup clustered at
+ * the beginning.
  */
 struct vm_map_entry {
        struct rb_node          rb_node;        /* tree information */
+       vaddr_t                 start;          /* start address */
+       vaddr_t                 end;            /* end address */
        vsize_t                 gap;            /* free space after */
        vsize_t                 maxgap;         /* space in subtree */
        struct vm_map_entry     *prev;          /* previous entry */
        struct vm_map_entry     *next;          /* next entry */
-       vaddr_t                 start;          /* start address */
-       vaddr_t                 end;            /* end address */
        union {
                struct uvm_object *uvm_obj;     /* uvm object */
                struct vm_map   *sub_map;       /* belongs to another map */
        } object;                               /* object I point to */
        voff_t                  offset;         /* offset into object */
-       int                     etype;          /* entry type */
+       uint8_t                 etype;          /* entry type */
+       uint8_t                 flags;          /* flags */
+       uint8_t                 advice;         /* madvise advice */
+       uint8_t                 unused;         /* unused */
        vm_prot_t               protection;     /* protection code */
        vm_prot_t               max_protection; /* maximum protection */
        vm_inherit_t            inheritance;    /* inheritance */
        int                     wired_count;    /* can be paged if == 0 */
        struct vm_aref          aref;           /* anonymous overlay */
-       int                     advice;         /* madvise advice */
-#define uvm_map_entry_stop_copy flags
-       u_int8_t                flags;          /* flags */
+};
 
+/* flags */
 #define        UVM_MAP_KERNEL          0x01            /* kernel map entry */
 #define        UVM_MAP_STATIC          0x04            /* special static entries */
 #define        UVM_MAP_NOMERGE         0x08            /* this entry is not mergable */
 
-};
-
 #define        VM_MAPENT_ISWIRED(entry)        ((entry)->wired_count != 0)
 
 /*



Home | Main Index | Thread Index | Old Index