[src/trunk]: src Defer bpf_mtap in Rx interrupt context to softint

[ozaki-r <ozaki-r%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/62c877c7895d
branches:  trunk
changeset: 821087:62c877c7895d
user:      ozaki-r <ozaki-r%NetBSD.org@localhost>
date:      Tue Jan 24 09:05:27 2017 +0000

description:
Defer bpf_mtap in Rx interrupt context to softint

bpf_mtap of some drivers is still called in hardware interrupt context.
We want to run them in softint as well as bpf_mtap of most drivers
(see if_percpuq_softint and if_input).

To this end, bpf_mtap_softint mechanism is implemented; it defers
bpf_mtap processing to a dedicated softint for a target driver.
By using the machanism, we can move bpf_mtap processing to softint
without changing target drivers much while it adds some overhead
on CPU and memory. Once target drivers are changed to softint-based,
we should return to normal bpf_mtap.

Proposed on tech-kern and tech-net

diffstat:

 doc/TODO.smpnet       |   28 +++++++++++
 sys/dev/ic/hd64570.c  |    7 +-
 sys/dev/ic/midway.c   |    7 +-
 sys/dev/pci/if_lmc.c  |    8 +-
 sys/dev/pci/if_lmc.h  |   10 ++-
 sys/net/bpf.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++-
 sys/net/bpf.h         |    5 +-
 sys/net/bpfdesc.h     |    5 +-
 sys/netisdn/i4b_ipr.c |    7 +-
 9 files changed, 184 insertions(+), 20 deletions(-)

diffs (truncated from 427 to 300 lines):

diff -r f528e0b36106 -r 62c877c7895d doc/TODO.smpnet
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/doc/TODO.smpnet   Tue Jan 24 09:05:27 2017 +0000
@@ -0,0 +1,28 @@
+$NetBSD: TODO.smpnet,v 1.1 2017/01/24 09:05:27 ozaki-r Exp $
+
+Non MP-safe components
+======================
+
+ - bpf
+ - To be listed more...
+
+bpf
+===
+
+MP-ification of bpf requires all of bpf_mtap* are called in normal LWP context
+or softint context, i.e., not in hardware interrupt context.  For Tx, all
+bpf_mtap satisfy the requrement.  For Rx, most of bpf_mtap are called in softint.
+Unfortunately some bpf_mtap on Rx are still called in hardware interrupt context.
+
+This is the list of the functions that have such bpf_mtap:
+
+ - sca_frame_process() @ sys/dev/ic/hd64570.c
+ - en_intr() @ sys/dev/ic/midway.c
+ - rxintr_cleanup() and txintr_cleanup() @ sys/dev/pci/if_lmc.c
+ - ipr_rx_data_rdy() @ sys/netisdn/i4b_ipr.c
+
+Ideally we should make the functions run in softint somehow, but we don't have
+actual devices, no time (or interest/love) to work on the task, so instead we
+provide a deferred bpf_mtap mechanism that forcibly runs bpf_mtap in softint
+context.  It's a workaround and once the functions run in softint, we should use
+the original bpf_mtap again.
diff -r f528e0b36106 -r 62c877c7895d sys/dev/ic/hd64570.c
--- a/sys/dev/ic/hd64570.c      Tue Jan 24 07:58:58 2017 +0000
+++ b/sys/dev/ic/hd64570.c      Tue Jan 24 09:05:27 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: hd64570.c,v 1.51 2016/12/15 09:35:24 ozaki-r Exp $     */
+/*     $NetBSD: hd64570.c,v 1.52 2017/01/24 09:05:28 ozaki-r Exp $     */
 
 /*
  * Copyright (c) 1999 Christian E. Hopps
@@ -65,7 +65,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: hd64570.c,v 1.51 2016/12/15 09:35:24 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: hd64570.c,v 1.52 2017/01/24 09:05:28 ozaki-r Exp $");
 
 #include "opt_inet.h"
 
@@ -457,6 +457,7 @@
        if_attach(ifp);
        if_alloc_sadl(ifp);
        bpf_attach(ifp, DLT_HDLC, HDLC_HDRLEN);
+       bpf_mtap_softint_init(ifp);
 
        if (sc->sc_parent == NULL)
                printf("%s: port %d\n", ifp->if_xname, port);
@@ -1574,7 +1575,7 @@
                return;
        }
 
-       bpf_mtap(&scp->sp_if, m); /* XXX not in softint */
+       bpf_mtap_softint(&scp->sp_if, m);
 
        scp->sp_if.if_ipackets++;
 
diff -r f528e0b36106 -r 62c877c7895d sys/dev/ic/midway.c
--- a/sys/dev/ic/midway.c       Tue Jan 24 07:58:58 2017 +0000
+++ b/sys/dev/ic/midway.c       Tue Jan 24 09:05:27 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: midway.c,v 1.98 2016/12/15 09:35:24 ozaki-r Exp $      */
+/*     $NetBSD: midway.c,v 1.99 2017/01/24 09:05:28 ozaki-r Exp $      */
 /*     (sync'd to midway.c 1.68)       */
 
 /*
@@ -61,7 +61,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: midway.c,v 1.98 2016/12/15 09:35:24 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: midway.c,v 1.99 2017/01/24 09:05:28 ozaki-r Exp $");
 
 #include "opt_natm.h"
 
@@ -2766,7 +2766,7 @@
          ifp->if_ipackets++;
 #endif
 
-         bpf_mtap(ifp, m); /* XXX not in softint */
+         bpf_mtap_softint(ifp, m);
 
          atm_input(ifp, &ah, m, sc->rxslot[slot].rxhand);
        }
@@ -3623,6 +3623,7 @@
        LIST_INSERT_HEAD(&sc->sif_list, (struct pvcsif *)pvc_ifp, sif_links);
        if_attach(pvc_ifp);
        atm_ifattach(pvc_ifp);
+       bpf_mtap_softint_init(pvc_ifp);
 
 #ifdef ATM_PVCEXT
        rrp_add(sc, pvc_ifp);
diff -r f528e0b36106 -r 62c877c7895d sys/dev/pci/if_lmc.c
--- a/sys/dev/pci/if_lmc.c      Tue Jan 24 07:58:58 2017 +0000
+++ b/sys/dev/pci/if_lmc.c      Tue Jan 24 09:05:27 2017 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: if_lmc.c,v 1.62 2016/12/15 09:35:24 ozaki-r Exp $ */
+/* $NetBSD: if_lmc.c,v 1.63 2017/01/24 09:05:28 ozaki-r Exp $ */
 
 /*-
  * Copyright (c) 2002-2006 David Boggs. <boggs%boggs.palo-alto.ca.us@localhost>
@@ -74,7 +74,7 @@
  */
 
 # include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_lmc.c,v 1.62 2016/12/15 09:35:24 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_lmc.c,v 1.63 2017/01/24 09:05:28 ozaki-r Exp $");
 # include <sys/param.h>        /* OS version */
 # include "opt_inet.h" /* INET6, INET */
 # include "opt_altq_enabled.h" /* ALTQ */
@@ -4294,7 +4294,7 @@
     sc->status.cntrs.ipackets++;
 
     /* Berkeley Packet Filter */
-    LMC_BPF_MTAP(sc, first_mbuf); /* XXX not in softint */
+    LMC_BPF_MTAP(sc, first_mbuf);
 
     /* Give this good packet to the network stacks. */
     sc->quota--;
@@ -4446,7 +4446,7 @@
         sc->status.cntrs.opackets++;
 
         /* Berkeley Packet Filter */
-        LMC_BPF_MTAP(sc, m); /* XXX not in softint */
+        LMC_BPF_MTAP(sc, m);
        }
 
       m_freem(m);
diff -r f528e0b36106 -r 62c877c7895d sys/dev/pci/if_lmc.h
--- a/sys/dev/pci/if_lmc.h      Tue Jan 24 07:58:58 2017 +0000
+++ b/sys/dev/pci/if_lmc.h      Tue Jan 24 09:05:27 2017 +0000
@@ -1,5 +1,5 @@
 /*-
- * $NetBSD: if_lmc.h,v 1.23 2016/04/28 00:16:56 ozaki-r Exp $
+ * $NetBSD: if_lmc.h,v 1.24 2017/01/24 09:05:28 ozaki-r Exp $
  *
  * Copyright (c) 2002-2006 David Boggs. (boggs%boggs.palo-alto.ca.us@localhost)
  * All rights reserved.
@@ -984,8 +984,12 @@
 # define SLEEP(usecs)          tsleep(sc, PZERO, DEVICE_NAME, 1+(usecs/tick))
 # define DMA_SYNC(map, size, flags) bus_dmamap_sync(ring->tag, map, 0, size, flags)
 # define DMA_LOAD(map, addr, size)  bus_dmamap_load(ring->tag, map, addr, size, 0, BUS_DMA_NOWAIT)
-#  define LMC_BPF_MTAP(sc, mbuf)       bpf_mtap((sc)->ifp, mbuf)
-#  define LMC_BPF_ATTACH(sc, dlt, len) bpf_attach((sc)->ifp, dlt, len)
+#  define LMC_BPF_MTAP(sc, mbuf)       bpf_mtap_softint((sc)->ifp, mbuf)
+#  define LMC_BPF_ATTACH(sc, dlt, len)                 \
+       do {                                            \
+               bpf_attach((sc)->ifp, dlt, len);        \
+               bpf_mtap_softint_init((sc)->ifp);       \
+       } while (0)
 #  define LMC_BPF_DETACH(sc)           bpf_detach((sc)->ifp)
 
 static int driver_announced = 0;       /* print driver info once only */
diff -r f528e0b36106 -r 62c877c7895d sys/net/bpf.c
--- a/sys/net/bpf.c     Tue Jan 24 07:58:58 2017 +0000
+++ b/sys/net/bpf.c     Tue Jan 24 09:05:27 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: bpf.c,v 1.204 2017/01/23 10:17:36 ozaki-r Exp $        */
+/*     $NetBSD: bpf.c,v 1.205 2017/01/24 09:05:28 ozaki-r Exp $        */
 
 /*
  * Copyright (c) 1990, 1991, 1993
@@ -39,12 +39,13 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.204 2017/01/23 10:17:36 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.205 2017/01/24 09:05:28 ozaki-r Exp $");
 
 #if defined(_KERNEL_OPT)
 #include "opt_bpf.h"
 #include "sl.h"
 #include "strip.h"
+#include "opt_net_mpsafe.h"
 #endif
 
 #include <sys/param.h>
@@ -60,6 +61,7 @@
 #include <sys/stat.h>
 #include <sys/module.h>
 #include <sys/atomic.h>
+#include <sys/cpu.h>
 
 #include <sys/file.h>
 #include <sys/filedesc.h>
@@ -73,6 +75,7 @@
 #include <sys/poll.h>
 #include <sys/sysctl.h>
 #include <sys/kauth.h>
+#include <sys/syslog.h>
 
 #include <net/if.h>
 #include <net/slip.h>
@@ -1594,6 +1597,92 @@
        m_freem(m);
 }
 
+static struct mbuf *
+bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m)
+{
+       struct mbuf *dup;
+
+       dup = m_dup(m, 0, M_COPYALL, M_NOWAIT);
+       if (dup == NULL)
+               return NULL;
+
+       if (bp->bif_mbuf_tail != NULL) {
+               bp->bif_mbuf_tail->m_nextpkt = dup;
+       } else {
+               bp->bif_mbuf_head = dup;
+       }
+       bp->bif_mbuf_tail = dup;
+#ifdef BPF_MTAP_SOFTINT_DEBUG
+       log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n",
+           __func__, dup, bp->bif_ifp->if_xname);
+#endif
+
+       return dup;
+}
+
+static struct mbuf *
+bpf_mbuf_dequeue(struct bpf_if *bp)
+{
+       struct mbuf *m;
+       int s;
+
+       s = splnet();
+       m = bp->bif_mbuf_head;
+       if (m != NULL) {
+               bp->bif_mbuf_head = m->m_nextpkt;
+               m->m_nextpkt = NULL;
+
+               if (bp->bif_mbuf_head == NULL)
+                       bp->bif_mbuf_tail = NULL;
+#ifdef BPF_MTAP_SOFTINT_DEBUG
+               log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n",
+                   __func__, m, bp->bif_ifp->if_xname);
+#endif
+       }
+       splx(s);
+
+       return m;
+}
+
+static void
+bpf_mtap_si(void *arg)
+{
+       struct bpf_if *bp = arg;
+       struct mbuf *m;
+
+       while ((m = bpf_mbuf_dequeue(bp)) != NULL) {
+#ifdef BPF_MTAP_SOFTINT_DEBUG
+               log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n",
+                   __func__, m, bp->bif_ifp->if_xname);
+#endif
+#ifndef NET_MPSAFE
+               KERNEL_LOCK(1, NULL);
+#endif
+               bpf_ops->bpf_mtap(bp, m);
+#ifndef NET_MPSAFE
+               KERNEL_UNLOCK_ONE(NULL);
+#endif
+               m_freem(m);
+       }
+}
+
+void
+bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m)
+{
+       struct bpf_if *bp = ifp->if_bpf;
+       struct mbuf *dup;
+
+       KASSERT(cpu_intr_p());
+
+       if (bp == NULL || bp->bif_dlist == NULL)
+               return;
+       KASSERT(bp->bif_si != NULL);
+
+       dup = bpf_mbuf_enqueue(bp, m);
+       if (dup != NULL)
+               softint_schedule(bp->bif_si);
+}
+
 static int
 bpf_hdrlen(struct bpf_d *d)
 {
@@ -1790,6 +1879,7 @@
        bp->bif_driverp = driverp;
        bp->bif_ifp = ifp;
        bp->bif_dlt = dlt;