Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/kern Reorganise the locking and allocation of fstrans_lw...



details:   https://anonhg.NetBSD.org/src/rev/2a30fa662c54
branches:  trunk
changeset: 972163:2a30fa662c54
user:      ad <ad%NetBSD.org@localhost>
date:      Sun May 17 19:34:07 2020 +0000

description:
Reorganise the locking and allocation of fstrans_lwp_info slightly, to
reduce contention.  "please go ahead" hannken@.

diffstat:

 sys/kern/vfs_trans.c |  153 +++++++++++++++++++++++++++++++-------------------
 1 files changed, 95 insertions(+), 58 deletions(-)

diffs (truncated from 351 to 300 lines):

diff -r 139c24714519 -r 2a30fa662c54 sys/kern/vfs_trans.c
--- a/sys/kern/vfs_trans.c      Sun May 17 18:59:02 2020 +0000
+++ b/sys/kern/vfs_trans.c      Sun May 17 19:34:07 2020 +0000
@@ -1,7 +1,7 @@
-/*     $NetBSD: vfs_trans.c,v 1.62 2020/05/13 09:21:30 hannken Exp $   */
+/*     $NetBSD: vfs_trans.c,v 1.63 2020/05/17 19:34:07 ad Exp $        */
 
 /*-
- * Copyright (c) 2007 The NetBSD Foundation, Inc.
+ * Copyright (c) 2007, 2020 The NetBSD Foundation, Inc.
  * All rights reserved.
  *
  * This code is derived from software contributed to The NetBSD Foundation
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfs_trans.c,v 1.62 2020/05/13 09:21:30 hannken Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_trans.c,v 1.63 2020/05/17 19:34:07 ad Exp $");
 
 /*
  * File system transaction operations.
@@ -50,6 +50,7 @@
 #include <sys/vnode.h>
 #include <sys/fstrans.h>
 #include <sys/proc.h>
+#include <sys/pool.h>
 
 #include <miscfs/specfs/specdev.h>
 
@@ -85,14 +86,17 @@
        struct lwp *fmi_owner;
 };
 
-static kmutex_t vfs_suspend_lock;      /* Serialize suspensions. */
-static kmutex_t fstrans_lock;          /* Fstrans big lock. */
-static kmutex_t fstrans_mount_lock;    /* Fstrans mount big lock. */
+static kmutex_t vfs_suspend_lock       /* Serialize suspensions. */
+    __cacheline_aligned;
+static kmutex_t fstrans_lock           /* Fstrans big lock. */
+    __cacheline_aligned;
 static kcondvar_t fstrans_state_cv;    /* Fstrans or cow state changed. */
 static kcondvar_t fstrans_count_cv;    /* Fstrans or cow count changed. */
 static pserialize_t fstrans_psz;       /* Pserialize state. */
 static LIST_HEAD(fstrans_lwp_head, fstrans_lwp_info) fstrans_fli_head;
                                        /* List of all fstrans_lwp_info. */
+static pool_cache_t fstrans_lwp_cache; /* Cache of fstrans_lwp_info. */
+
 static int fstrans_gone_count;         /* Number of fstrans_mount_info gone. */
 
 static void fstrans_mount_dtor(struct fstrans_mount_info *);
@@ -100,6 +104,8 @@
 static inline struct fstrans_lwp_info *
     fstrans_get_lwp_info(struct mount *, bool);
 static struct fstrans_lwp_info *fstrans_alloc_lwp_info(struct mount *);
+static int fstrans_lwp_pcc(void *, void *, int);
+static void fstrans_lwp_pcd(void *, void *);
 static inline int _fstrans_start(struct mount *, enum fstrans_lock_type, int);
 static bool grant_lock(const struct fstrans_mount_info *,
     const enum fstrans_lock_type);
@@ -125,12 +131,12 @@
 {
        struct fstrans_debug_mount *fdm, *new;
 
-       KASSERT(mutex_owned(&fstrans_mount_lock));
+       KASSERT(mutex_owned(&fstrans_lock));
 
-       mutex_exit(&fstrans_mount_lock);
+       mutex_exit(&fstrans_lock);
        new = kmem_alloc(sizeof(*new), KM_SLEEP);
        new->fdm_mount = mp;
-       mutex_enter(&fstrans_mount_lock);
+       mutex_enter(&fstrans_lock);
 
        SLIST_FOREACH(fdm, &fstrans_debug_mount_head, fdm_list)
                KASSERT(fdm->fdm_mount != mp);
@@ -142,7 +148,7 @@
 {
        struct fstrans_debug_mount *fdm;
 
-       KASSERT(mutex_owned(&fstrans_mount_lock));
+       KASSERT(mutex_owned(&fstrans_lock));
 
        SLIST_FOREACH(fdm, &fstrans_debug_mount_head, fdm_list)
                if (fdm->fdm_mount == mp)
@@ -158,7 +164,7 @@
 {
        struct fstrans_debug_mount *fdm;
 
-       KASSERT(mutex_owned(&fstrans_mount_lock));
+       KASSERT(mutex_owned(&fstrans_lock));
 
        SLIST_FOREACH(fdm, &fstrans_debug_mount_head, fdm_list)
                if (fdm->fdm_mount == mp)
@@ -183,11 +189,45 @@
 
        mutex_init(&vfs_suspend_lock, MUTEX_DEFAULT, IPL_NONE);
        mutex_init(&fstrans_lock, MUTEX_DEFAULT, IPL_NONE);
-       mutex_init(&fstrans_mount_lock, MUTEX_DEFAULT, IPL_NONE);
        cv_init(&fstrans_state_cv, "fstchg");
        cv_init(&fstrans_count_cv, "fstcnt");
        fstrans_psz = pserialize_create();
        LIST_INIT(&fstrans_fli_head);
+       fstrans_lwp_cache = pool_cache_init(sizeof(struct fstrans_lwp_info),
+           coherency_unit, 0, 0, "fstlwp", NULL, IPL_NONE,
+           fstrans_lwp_pcc, fstrans_lwp_pcd, NULL);
+       KASSERT(fstrans_lwp_cache != NULL);
+}
+
+/*
+ * pool_cache constructor for fstrans_lwp_info.  Updating the global list
+ * produces cache misses on MP.  Minimise by keeping free entries on list.
+ */
+int
+fstrans_lwp_pcc(void *arg, void *obj, int flags)
+{
+       struct fstrans_lwp_info *fli = obj;
+
+       memset(fli, 0, sizeof(*fli));
+
+       mutex_enter(&fstrans_lock);
+       LIST_INSERT_HEAD(&fstrans_fli_head, fli, fli_list);
+       mutex_exit(&fstrans_lock);
+
+       return 0;
+}
+
+/*
+ * pool_cache destructor
+ */
+void
+fstrans_lwp_pcd(void *arg, void *obj)
+{
+       struct fstrans_lwp_info *fli = obj;
+
+       mutex_enter(&fstrans_lock);
+       LIST_REMOVE(fli, fli_list);
+       mutex_exit(&fstrans_lock);
 }
 
 /*
@@ -198,6 +238,10 @@
 {
        struct fstrans_lwp_info *fli, *fli_next;
 
+       if (l->l_fstrans == NULL)
+               return;
+
+       mutex_enter(&fstrans_lock);
        for (fli = l->l_fstrans; fli; fli = fli_next) {
                KASSERT(fli->fli_trans_cnt == 0);
                KASSERT(fli->fli_cow_cnt == 0);
@@ -209,10 +253,14 @@
                fli->fli_mount = NULL;
                fli->fli_alias = NULL;
                fli->fli_mountinfo = NULL;
-               membar_sync();
                fli->fli_self = NULL;
        }
+       mutex_exit(&fstrans_lock);
 
+       for (fli = l->l_fstrans; fli; fli = fli_next) {
+               fli_next = fli->fli_succ;
+               pool_cache_put(fstrans_lwp_cache, fli);
+       }
        l->l_fstrans = NULL;
 }
 
@@ -223,12 +271,11 @@
 fstrans_mount_dtor(struct fstrans_mount_info *fmi)
 {
 
-       mutex_enter(&fstrans_mount_lock);
+       KASSERT(mutex_owned(&fstrans_lock));
 
        KASSERT(fmi != NULL);
        fmi->fmi_ref_cnt -= 1;
-       if (fmi->fmi_ref_cnt > 0) {
-               mutex_exit(&fstrans_mount_lock);
+       if (__predict_true(fmi->fmi_ref_cnt > 0)) {
                return;
        }
 
@@ -239,8 +286,6 @@
        KASSERT(fstrans_gone_count > 0);
        fstrans_gone_count -= 1;
 
-       mutex_exit(&fstrans_mount_lock);
-
        kmem_free(fmi->fmi_mount, sizeof(*fmi->fmi_mount));
        kmem_free(fmi, sizeof(*fmi));
 }
@@ -262,10 +307,10 @@
        newfmi->fmi_mount = mp;
        newfmi->fmi_owner = NULL;
 
-       mutex_enter(&fstrans_mount_lock);
+       mutex_enter(&fstrans_lock);
        mp->mnt_transinfo = newfmi;
        fstrans_debug_mount(mp);
-       mutex_exit(&fstrans_mount_lock);
+       mutex_exit(&fstrans_lock);
 
        return 0;
 }
@@ -280,14 +325,13 @@
 
        KASSERT(fmi != NULL);
 
-       mutex_enter(&fstrans_mount_lock);
+       mutex_enter(&fstrans_lock);
        fstrans_debug_unmount(mp);
        fmi->fmi_gone = true;
        mp->mnt_transinfo = NULL;
        fstrans_gone_count += 1;
-       mutex_exit(&fstrans_mount_lock);
-
        fstrans_mount_dtor(fmi);
+       mutex_exit(&fstrans_lock);
 }
 
 /*
@@ -296,11 +340,12 @@
 static void
 fstrans_clear_lwp_info(void)
 {
-       struct fstrans_lwp_info **p, *fli;
+       struct fstrans_lwp_info **p, *fli, *tofree = NULL;
 
        /*
         * Scan our list clearing entries whose mount is gone.
         */
+       mutex_enter(&fstrans_lock);
        for (p = &curlwp->l_fstrans; *p; ) {
                fli = *p;
                if (fli->fli_mount != NULL &&
@@ -317,9 +362,10 @@
                        fli->fli_mount = NULL;
                        fli->fli_alias = NULL;
                        fli->fli_mountinfo = NULL;
-                       membar_sync();
                        fli->fli_self = NULL;
                        p = &curlwp->l_fstrans;
+                       fli->fli_succ = tofree;
+                       tofree = fli;
                } else {
                        p = &(*p)->fli_succ;
                }
@@ -329,6 +375,13 @@
                if (fli->fli_alias != NULL)
                        KASSERT(fli->fli_alias->fli_self == curlwp);
 #endif /* DIAGNOSTIC */
+       mutex_exit(&fstrans_lock);
+
+       while (tofree != NULL) {
+               fli = tofree;
+               tofree = fli->fli_succ;
+               pool_cache_put(fstrans_lwp_cache, fli);
+       }
 }
 
 /*
@@ -346,40 +399,25 @@
        }
 
        /*
-        * Try to reuse a cleared entry or allocate a new one.
+        * Allocate a new entry.
         */
-       mutex_enter(&fstrans_lock);
-       LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
-               membar_sync();
-               if (fli->fli_self == NULL) {
-                       KASSERT(fli->fli_mount == NULL);
-                       KASSERT(fli->fli_trans_cnt == 0);
-                       KASSERT(fli->fli_cow_cnt == 0);
-                       KASSERT(fli->fli_alias_cnt == 0);
-                       fli->fli_self = curlwp;
-                       fli->fli_succ = curlwp->l_fstrans;
-                       curlwp->l_fstrans = fli;
-                       break;
-               }
-       }
-       mutex_exit(&fstrans_lock);
-
-       if (fli == NULL) {
-               fli = kmem_alloc(sizeof(*fli), KM_SLEEP);
-               mutex_enter(&fstrans_lock);
-               memset(fli, 0, sizeof(*fli));
-               fli->fli_self = curlwp;
-               LIST_INSERT_HEAD(&fstrans_fli_head, fli, fli_list);
-               mutex_exit(&fstrans_lock);
-               fli->fli_succ = curlwp->l_fstrans;
-               curlwp->l_fstrans = fli;
-       }
+       fli = pool_cache_get(fstrans_lwp_cache, PR_WAITOK);
+       KASSERT(fli->fli_trans_cnt == 0);
+       KASSERT(fli->fli_cow_cnt == 0);
+       KASSERT(fli->fli_alias_cnt == 0);
+       KASSERT(fli->fli_mount == NULL);



Home | Main Index | Thread Index | Old Index