[src/trunk]: src/sys/netipsec Fix a deadlock happening if !NET_MPSAFE

[ozaki-r <ozaki-r%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/35afef03f7d9
branches:  trunk
changeset: 828149:35afef03f7d9
user:      ozaki-r <ozaki-r%NetBSD.org@localhost>
date:      Thu Nov 30 02:45:12 2017 +0000

description:
Fix a deadlock happening if !NET_MPSAFE

If NET_MPSAFE isn't set, key_timehandler_work is executed with holding
softnet_lock. This means that localcount_drain can be called with holding
softnet_lock resulting in a deadlock that localcount_drain waits for packet
processing to release a reference to SP/SA while network processing is prevented
by softnet_lock.

Fix the deadlock by not taking softnet_lock in key_timehandler_work. It's okay
because IPsec is MP-safe even if !NET_MPSAFE. Note that the change also needs
to enable pserialize_perform because the IPsec code can be run in parallel now.

Reported by christos@

diffstat:

 sys/netipsec/key.c |  20 ++------------------
 1 files changed, 2 insertions(+), 18 deletions(-)

diffs (116 lines):

diff -r b0f6d3ea667c -r 35afef03f7d9 sys/netipsec/key.c
--- a/sys/netipsec/key.c        Thu Nov 30 02:43:49 2017 +0000
+++ b/sys/netipsec/key.c        Thu Nov 30 02:45:12 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: key.c,v 1.244 2017/11/30 02:43:49 ozaki-r Exp $        */
+/*     $NetBSD: key.c,v 1.245 2017/11/30 02:45:12 ozaki-r Exp $        */
 /*     $FreeBSD: src/sys/netipsec/key.c,v 1.3.2.3 2004/02/14 22:23:23 bms Exp $        */
 /*     $KAME: key.c,v 1.191 2001/06/27 10:46:49 sakane Exp $   */
 
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.244 2017/11/30 02:43:49 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.245 2017/11/30 02:45:12 ozaki-r Exp $");
 
 /*
  * This code is referred to RFC 2367
@@ -800,7 +800,6 @@
        return 0;
 }
 
-#ifdef NET_MPSAFE
 static void
 key_spd_pserialize_perform(void)
 {
@@ -818,7 +817,6 @@
        key_spd.psz_performing = false;
        cv_broadcast(&key_spd.cv_psz);
 }
-#endif
 
 /*
  * Remove the sp from the key_spd.splist and wait for references to the sp
@@ -836,10 +834,8 @@
        /* Invalidate all cached SPD pointers in the PCBs. */
        ipsec_invalpcbcacheall();
 
-#ifdef NET_MPSAFE
        KDASSERT(mutex_ownable(softnet_lock));
        key_spd_pserialize_perform();
-#endif
 
        localcount_drain(&sp->localcount, &key_spd.cv_lc, &key_spd.lock);
 }
@@ -1493,7 +1489,6 @@
 }
 #endif
 
-#ifdef NET_MPSAFE
 static void
 key_sad_pserialize_perform(void)
 {
@@ -1511,7 +1506,6 @@
        key_sad.psz_performing = false;
        cv_broadcast(&key_sad.cv_psz);
 }
-#endif
 
 /*
  * Remove the sav from the savlist of its sah and wait for references to the sav
@@ -1525,10 +1519,8 @@
 
        SAVLIST_WRITER_REMOVE(sav);
 
-#ifdef NET_MPSAFE
        KDASSERT(mutex_ownable(softnet_lock));
        key_sad_pserialize_perform();
-#endif
 
        localcount_drain(&sav->localcount, &key_sad.cv_lc, &key_sad.lock);
 }
@@ -1567,10 +1559,8 @@
        KEY_SA_UNREF(&sav);
 
        mutex_enter(&key_sad.lock);
-#ifdef NET_MPSAFE
        KDASSERT(mutex_ownable(softnet_lock));
        key_sad_pserialize_perform();
-#endif
        localcount_drain(&sav->localcount, &key_sad.cv_lc, &key_sad.lock);
        mutex_exit(&key_sad.lock);
 
@@ -3048,10 +3038,8 @@
        /* Remove from the sah list */
        SAHLIST_WRITER_REMOVE(sah);
 
-#ifdef NET_MPSAFE
        KDASSERT(mutex_ownable(softnet_lock));
        key_sad_pserialize_perform();
-#endif
 
        localcount_drain(&sah->localcount, &key_sad.cv_lc, &key_sad.lock);
 }
@@ -4862,13 +4850,10 @@
 key_timehandler_work(struct work *wk, void *arg)
 {
        time_t now = time_uptime;
-       IPSEC_DECLARE_LOCK_VARIABLE;
 
        /* We can allow enqueuing another work at this point */
        atomic_swap_uint(&key_timehandler_work_enqueued, 0);
 
-       IPSEC_ACQUIRE_GLOBAL_LOCKS();
-
        key_timehandler_spd(now);
        key_timehandler_sad(now);
        key_timehandler_acq(now);
@@ -4879,7 +4864,6 @@
        /* do exchange to tick time !! */
        callout_reset(&key_timehandler_ch, hz, key_timehandler, NULL);
 
-       IPSEC_RELEASE_GLOBAL_LOCKS();
        return;
 }