[src/trunk]: src/sys/netipsec MP-ify SAD (key_sad.sahlist and sah entries)

[ozaki-r <ozaki-r%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/46e9627edb1c
branches:  trunk
changeset: 355648:46e9627edb1c
user:      ozaki-r <ozaki-r%NetBSD.org@localhost>
date:      Tue Aug 08 04:17:34 2017 +0000

description:
MP-ify SAD (key_sad.sahlist and sah entries)

localcount(9) is used to protect key_sad.sahlist and sah entries
as well as SPD (and will be used for SAD sav).

Please read the locking notes of SAD for more details.

diffstat:

 sys/netipsec/key.c   |  296 ++++++++++++++++++++++++++++++++++++++------------
 sys/netipsec/keydb.h |    5 +-
 2 files changed, 229 insertions(+), 72 deletions(-)

diffs (truncated from 691 to 300 lines):

diff -r 253d1e6d015d -r 46e9627edb1c sys/netipsec/key.c
--- a/sys/netipsec/key.c        Tue Aug 08 03:58:43 2017 +0000
+++ b/sys/netipsec/key.c        Tue Aug 08 04:17:34 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: key.c,v 1.215 2017/08/08 01:56:49 ozaki-r Exp $        */
+/*     $NetBSD: key.c,v 1.216 2017/08/08 04:17:34 ozaki-r Exp $        */
 /*     $FreeBSD: src/sys/netipsec/key.c,v 1.3.2.3 2004/02/14 22:23:23 bms Exp $        */
 /*     $KAME: key.c,v 1.191 2001/06/27 10:46:49 sakane Exp $   */
 
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.215 2017/08/08 01:56:49 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.216 2017/08/08 04:17:34 ozaki-r Exp $");
 
 /*
  * This code is referd to RFC 2367
@@ -198,12 +198,35 @@
  *     until GC by the timer
  */
 /*
+ * Locking notes on SAD:
+ * - Data structures
+ *   - SAs are managed by the list called key_sad.sahlist and sav lists of sah
+ *     entries
+ *   - A sah has sav lists for each SA state
+ *   - Multiple sahs with the same saidx can exist
+ *     - Only one entry has MATURE state and others should be DEAD
+ *     - DEAD entries are just ignored from searching
+ * - Modifications to the key_sad.sahlist must be done with holding key_sad.lock
+ *   which is a adaptive mutex
+ * - Read accesses to the key_sad.sahlist must be in pserialize(9) read sections
+ * - sah's lifetime is managed by localcount(9)
+ * - Getting an sah entry
+ *   - We get an SP from the key_spd.splist
+ *     - Must iterate the list and increment the reference count of a found sah
+ *       (by key_sah_ref) in a pserialize read section
+ *   - A gotten sah must be released after use by key_sah_unref
+ * - An sah is destroyed when its state become DEAD and no sav is
+ *   listed to the sah
+ *   - The destruction is done only in the timer (see key_timehandler_sad)
+ */
+/*
  * Locking notes on misc data:
  * - All lists of key_misc are protected by key_misc.lock
  *   - key_misc.lock must be held even for read accesses
  */
 
-static pserialize_t key_psz __read_mostly;
+static pserialize_t key_spd_psz __read_mostly;
+static pserialize_t key_sad_psz __read_mostly;
 
 /* SPD */
 static struct {
@@ -220,6 +243,7 @@
 /* SAD */
 static struct {
        kmutex_t lock;
+       kcondvar_t cv;
        struct pslist_head sahlist;
 } key_sad __cacheline_aligned;
 
@@ -614,13 +638,18 @@
 static u_int key_getspreqmsglen (const struct secpolicy *);
 static int key_spdexpire (struct secpolicy *);
 static struct secashead *key_newsah (const struct secasindex *);
-static void key_delsah (struct secashead *);
+static void key_unlink_sah(struct secashead *);
+static void key_destroy_sah(struct secashead *);
+static bool key_sah_has_sav(struct secashead *);
+static void key_sah_ref(struct secashead *);
+static void key_sah_unref(struct secashead *);
 static struct secasvar *key_newsav(struct mbuf *,
        const struct sadb_msghdr *, int *, const char*, int);
 #define        KEY_NEWSAV(m, sadb, e)                          \
        key_newsav(m, sadb, e, __func__, __LINE__)
 static void key_delsav (struct secasvar *);
 static struct secashead *key_getsah(const struct secasindex *, int);
+static struct secashead *key_getsah_ref(const struct secasindex *, int);
 static bool key_checkspidup(const struct secasindex *, u_int32_t);
 static struct secasvar *key_getsavbyspi (struct secashead *, u_int32_t);
 static int key_setsaval (struct secasvar *, struct mbuf *,
@@ -800,7 +829,7 @@
 
 #ifdef NET_MPSAFE
        KASSERT(mutex_ownable(softnet_lock));
-       pserialize_perform(key_psz);
+       pserialize_perform(key_spd_psz);
 #endif
 
        localcount_drain(&sp->localcount, &key_spd.cv, &key_spd.lock);
@@ -2954,9 +2983,13 @@
                PSLIST_INIT(&newsah->savlist[i]);
        newsah->saidx = *saidx;
 
-       /* add to saidxtree */
+       localcount_init(&newsah->localcount);
+       /* Take a reference for the caller */
+       localcount_acquire(&newsah->localcount);
+
+       /* Add to the sah list */
+       SAHLIST_ENTRY_INIT(newsah);
        newsah->state = SADB_SASTATE_MATURE;
-       SAHLIST_ENTRY_INIT(newsah);
        mutex_enter(&key_sad.lock);
        SAHLIST_WRITER_INSERT_HEAD(newsah);
        mutex_exit(&key_sad.lock);
@@ -2964,51 +2997,55 @@
        return newsah;
 }
 
-/*
- * delete SA index and all SA registerd.
- */
+static bool
+key_sah_has_sav(struct secashead *sah)
+{
+       u_int state;
+
+       KASSERT(mutex_owned(&key_sad.lock));
+
+       SASTATE_ANY_FOREACH(state) {
+               if (!SAVLIST_WRITER_EMPTY(sah, state))
+                       return true;
+       }
+
+       return false;
+}
+
 static void
-key_delsah(struct secashead *sah)
-{
-       struct secasvar *sav;
-       u_int state;
-       int s;
-       int zombie = 0;
+key_unlink_sah(struct secashead *sah)
+{
 
        KASSERT(!cpu_softintr_p());
-       KASSERT(sah != NULL);
-
-       s = splsoftnet();
-
-       /* searching all SA registerd in the secindex. */
-       SASTATE_ANY_FOREACH(state) {
-               SAVLIST_READER_FOREACH(sav, sah, state) {
-                       /* give up to delete this sa */
-                       zombie++;
-               }
-       }
-
-       /* don't delete sah only if there are savs. */
-       if (zombie) {
-               splx(s);
-               return;
-       }
+       KASSERT(mutex_owned(&key_sad.lock));
+       KASSERT(sah->state == SADB_SASTATE_DEAD);
+
+       /* Remove from the sah list */
+       SAHLIST_WRITER_REMOVE(sah);
+
+#ifdef NET_MPSAFE
+       KASSERT(mutex_ownable(softnet_lock));
+       pserialize_perform(key_sad_psz);
+#endif
+
+       localcount_drain(&sah->localcount, &key_sad.cv, &key_sad.lock);
+}
+
+static void
+key_destroy_sah(struct secashead *sah)
+{
 
        rtcache_free(&sah->sa_route);
 
-       /* remove from tree of SA index */
-       SAHLIST_WRITER_REMOVE(sah);
+       SAHLIST_ENTRY_DESTROY(sah);
+       localcount_fini(&sah->localcount);
 
        if (sah->idents != NULL)
                kmem_free(sah->idents, sah->idents_len);
        if (sah->identd != NULL)
                kmem_free(sah->identd, sah->identd_len);
 
-       SAHLIST_ENTRY_DESTROY(sah);
        kmem_free(sah, sizeof(*sah));
-
-       splx(s);
-       return;
 }
 
 /*
@@ -3136,7 +3173,32 @@
 }
 
 /*
- * search SAD.
+ * Must be called in a pserialize read section. A held sah
+ * must be released by key_sah_unref after use.
+ */
+static void
+key_sah_ref(struct secashead *sah)
+{
+
+       localcount_acquire(&sah->localcount);
+}
+
+/*
+ * Must be called without holding key_sad.lock because the lock
+ * would be held in localcount_release.
+ */
+static void
+key_sah_unref(struct secashead *sah)
+{
+
+       KDASSERT(mutex_ownable(&key_sad.lock));
+
+       localcount_release(&sah->localcount, &key_sad.cv, &key_sad.lock);
+}
+
+/*
+ * Search SAD and return sah. Must be called in a pserialize
+ * read section.
  * OUT:
  *     NULL    : not found
  *     others  : found, pointer to a SA.
@@ -3157,6 +3219,28 @@
 }
 
 /*
+ * Search SAD and return sah. If sah is returned, the caller must call
+ * key_sah_unref to releaset a reference.
+ * OUT:
+ *     NULL    : not found
+ *     others  : found, pointer to a SA.
+ */
+static struct secashead *
+key_getsah_ref(const struct secasindex *saidx, int flag)
+{
+       struct secashead *sah;
+       int s;
+
+       s = pserialize_read_enter();
+       sah = key_getsah(saidx, flag);
+       if (sah != NULL)
+               key_sah_ref(sah);
+       pserialize_read_exit(s);
+
+       return sah;
+}
+
+/*
  * check not to be duplicated SPI.
  * NOTE: this function is too slow due to searching all SAD.
  * OUT:
@@ -3168,6 +3252,7 @@
 {
        struct secashead *sah;
        struct secasvar *sav;
+       int s;
 
        /* check address family */
        if (saidx->src.sa.sa_family != saidx->dst.sa.sa_family) {
@@ -3176,15 +3261,18 @@
        }
 
        /* check all SAD */
+       s = pserialize_read_enter();
        SAHLIST_READER_FOREACH(sah) {
                if (!key_ismyaddr((struct sockaddr *)&sah->saidx.dst))
                        continue;
                sav = key_getsavbyspi(sah, spi);
                if (sav != NULL) {
+                       pserialize_read_exit(s);
                        KEY_SA_UNREF(&sav);
                        return true;
                }
        }
+       pserialize_read_exit(s);
 
        return false;
 }
@@ -4594,15 +4682,28 @@
 key_timehandler_sad(time_t now)
 {
        struct secashead *sah;
-       struct secasvar *sav;
+       int s;
 
 restart:
+       mutex_enter(&key_sad.lock);
        SAHLIST_WRITER_FOREACH(sah) {
-               /* if sah has been dead, then delete it and process next sah. */
-               if (sah->state == SADB_SASTATE_DEAD) {
-                       key_delsah(sah);
+               /* If sah has been dead and has no sav, then delete it */