[src/trunk]: src/sys Let PF_KEY socks hold their own lock instead of softnet_...

[ozaki-r <ozaki-r%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/138b71466c18
branches:  trunk
changeset: 825663:138b71466c18
user:      ozaki-r <ozaki-r%NetBSD.org@localhost>
date:      Thu Jul 27 09:53:57 2017 +0000

description:
Let PF_KEY socks hold their own lock instead of softnet_lock

Operations on SAD and SPD are executed via PF_KEY socks. The operations
include deletions of SAs and SPs that will use synchronization mechanisms
such as pserialize_perform to wait for references to SAs and SPs to be
released. It is known that using such mechanisms with holding softnet_lock
causes a dead lock. We should avoid the situation.

diffstat:

 sys/net/raw_cb.c       |   9 ++++++---
 sys/netipsec/key.c     |   6 ++++--
 sys/netipsec/keysock.c |  19 +++++++++++++++++--
 sys/netipsec/keysock.h |   4 +++-
 4 files changed, 30 insertions(+), 8 deletions(-)

diffs (129 lines):

diff -r 4eac7c7a0b84 -r 138b71466c18 sys/net/raw_cb.c
--- a/sys/net/raw_cb.c  Thu Jul 27 09:27:11 2017 +0000
+++ b/sys/net/raw_cb.c  Thu Jul 27 09:53:57 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: raw_cb.c,v 1.22 2014/05/21 20:43:56 rmind Exp $        */
+/*     $NetBSD: raw_cb.c,v 1.23 2017/07/27 09:53:57 ozaki-r Exp $      */
 
 /*
  * Copyright (c) 1980, 1986, 1993
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: raw_cb.c,v 1.22 2014/05/21 20:43:56 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: raw_cb.c,v 1.23 2017/07/27 09:53:57 ozaki-r Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -104,7 +104,6 @@
 
        KASSERT(rp != NULL);
        KASSERT(solocked(so));
-       KASSERT(so->so_lock == softnet_lock);   /* XXX */
 
        /* Remove the last reference. */
        LIST_REMOVE(rp, rcb_list);
@@ -113,6 +112,10 @@
        /* Note: sofree() drops the socket's lock. */
        sofree(so);
        kmem_free(rp, rcb_len);
+       if (so->so_lock != softnet_lock) {
+               so->so_lock = softnet_lock;
+               mutex_obj_hold(softnet_lock);
+       }
        mutex_enter(softnet_lock);
 }
 
diff -r 4eac7c7a0b84 -r 138b71466c18 sys/netipsec/key.c
--- a/sys/netipsec/key.c        Thu Jul 27 09:27:11 2017 +0000
+++ b/sys/netipsec/key.c        Thu Jul 27 09:53:57 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: key.c,v 1.195 2017/07/27 06:59:28 ozaki-r Exp $        */
+/*     $NetBSD: key.c,v 1.196 2017/07/27 09:53:57 ozaki-r Exp $        */
 /*     $FreeBSD: src/sys/netipsec/key.c,v 1.3.2.3 2004/02/14 22:23:23 bms Exp $        */
 /*     $KAME: key.c,v 1.191 2001/06/27 10:46:49 sakane Exp $   */
 
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.195 2017/07/27 06:59:28 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.196 2017/07/27 09:53:57 ozaki-r Exp $");
 
 /*
  * This code is referd to RFC 2367
@@ -7591,6 +7591,8 @@
        sysctl_net_key_compat_setup(NULL);
 
        RUN_ONCE(&key_init_once, key_do_init);
+
+       key_init_so();
 }
 
 /*
diff -r 4eac7c7a0b84 -r 138b71466c18 sys/netipsec/keysock.c
--- a/sys/netipsec/keysock.c    Thu Jul 27 09:27:11 2017 +0000
+++ b/sys/netipsec/keysock.c    Thu Jul 27 09:53:57 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: keysock.c,v 1.58 2017/05/25 04:45:59 ozaki-r Exp $     */
+/*     $NetBSD: keysock.c,v 1.59 2017/07/27 09:53:57 ozaki-r Exp $     */
 /*     $FreeBSD: src/sys/netipsec/keysock.c,v 1.3.2.1 2003/01/24 05:11:36 sam Exp $    */
 /*     $KAME: keysock.c,v 1.25 2001/08/13 20:07:41 itojun Exp $        */
 
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: keysock.c,v 1.58 2017/05/25 04:45:59 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: keysock.c,v 1.59 2017/07/27 09:53:57 ozaki-r Exp $");
 
 /* This code has derived from sys/net/rtsock.c on FreeBSD2.2.5 */
 
@@ -83,6 +83,15 @@
 
 int key_registered_sb_max = (2048 * MHLEN); /* XXX arbitrary */
 
+static kmutex_t *key_so_mtx;
+
+void
+key_init_so(void)
+{
+
+       key_so_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
+}
+
 /*
  * key_output()
  */
@@ -436,6 +445,12 @@
        so->so_pcb = kp;
 
        s = splsoftnet();
+
+       KASSERT(so->so_lock == NULL);
+       mutex_obj_hold(key_so_mtx);
+       so->so_lock = key_so_mtx;
+       solock(so);
+
        error = raw_attach(so, proto);
        if (error) {
                PFKEY_STATINC(PFKEY_STAT_SOCKERR);
diff -r 4eac7c7a0b84 -r 138b71466c18 sys/netipsec/keysock.h
--- a/sys/netipsec/keysock.h    Thu Jul 27 09:27:11 2017 +0000
+++ b/sys/netipsec/keysock.h    Thu Jul 27 09:53:57 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: keysock.h,v 1.9 2017/06/02 03:32:51 ozaki-r Exp $      */
+/*     $NetBSD: keysock.h,v 1.10 2017/07/27 09:53:57 ozaki-r Exp $     */
 /*     $FreeBSD: src/sys/netipsec/keysock.h,v 1.1.4.1 2003/01/24 05:11:36 sam Exp $    */
 /*     $KAME: keysock.h,v 1.8 2000/03/27 05:11:06 sumikawa Exp $       */
 
@@ -72,6 +72,8 @@
        int kp_registered;      /* registered socket */
 };
 
+void key_init_so(void);
+
 int key_sendup (struct socket *, struct sadb_msg *, u_int, int);
 int key_sendup_mbuf (struct socket *, struct mbuf *, int);
 #endif /* _KERNEL */