Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/net Divide Tx context of l2tp(4) to improve performance.
details: https://anonhg.NetBSD.org/src/rev/0ccd2e9c82ce
branches: trunk
changeset: 459634:0ccd2e9c82ce
user: knakahara <knakahara%NetBSD.org@localhost>
date: Thu Sep 19 04:59:42 2019 +0000
description:
Divide Tx context of l2tp(4) to improve performance.
It seems l2tp(4) call path is too long for instruction cache. So, dividing
l2tp(4) Tx context improves CPU use efficiency.
After this commit, l2tp(4) throughput gains 10% on my machine(Atom C3000).
diffstat:
sys/net/if_l2tp.c | 213 +++++++++++++++++++++++++++++++++++------------------
sys/net/if_l2tp.h | 5 +-
2 files changed, 145 insertions(+), 73 deletions(-)
diffs (truncated from 362 to 300 lines):
diff -r 3947546e69e5 -r 0ccd2e9c82ce sys/net/if_l2tp.c
--- a/sys/net/if_l2tp.c Thu Sep 19 04:46:29 2019 +0000
+++ b/sys/net/if_l2tp.c Thu Sep 19 04:59:42 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: if_l2tp.c,v 1.36 2019/08/19 03:24:05 ozaki-r Exp $ */
+/* $NetBSD: if_l2tp.c,v 1.37 2019/09/19 04:59:42 knakahara Exp $ */
/*
* Copyright (c) 2017 Internet Initiative Japan Inc.
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_l2tp.c,v 1.36 2019/08/19 03:24:05 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_l2tp.c,v 1.37 2019/09/19 04:59:42 knakahara Exp $");
#ifdef _KERNEL_OPT
#include "opt_inet.h"
@@ -118,15 +118,20 @@
static void l2tp_ro_init_pc(void *, void *, struct cpu_info *);
static void l2tp_ro_fini_pc(void *, void *, struct cpu_info *);
+static void l2tp_ifq_init_pc(void *, void *, struct cpu_info *);
+
static int l2tp_clone_create(struct if_clone *, int);
static int l2tp_clone_destroy(struct ifnet *);
struct if_clone l2tp_cloner =
IF_CLONE_INITIALIZER("l2tp", l2tp_clone_create, l2tp_clone_destroy);
+static int l2tp_tx_enqueue(struct l2tp_variant *, struct mbuf *);
static int l2tp_output(struct ifnet *, struct mbuf *,
const struct sockaddr *, const struct rtentry *);
+static void l2tp_sendit(struct l2tp_variant *, struct mbuf *);
static void l2tpintr(struct l2tp_variant *);
+static void l2tpintr_softint(void *);
static void l2tp_hash_init(void);
static int l2tp_hash_fini(void);
@@ -225,7 +230,10 @@
struct l2tp_softc *sc;
struct l2tp_variant *var;
int rv;
-
+ u_int si_flags = SOFTINT_NET;
+#ifdef NET_MPSAFE
+ si_flags |= SOFTINT_MPSAFE;
+#endif
sc = kmem_zalloc(sizeof(struct l2tp_softc), KM_SLEEP);
if_initname(&sc->l2tp_ec.ec_if, ifc->ifc_name, unit);
rv = l2tpattach0(sc);
@@ -248,6 +256,10 @@
sc->l2tp_ro_percpu = percpu_alloc(sizeof(struct l2tp_ro));
percpu_foreach(sc->l2tp_ro_percpu, l2tp_ro_init_pc, NULL);
+ sc->l2tp_ifq_percpu = percpu_alloc(sizeof(struct ifqueue));
+ percpu_foreach(sc->l2tp_ifq_percpu, l2tp_ifq_init_pc, NULL);
+ sc->l2tp_si = softint_establish(si_flags, l2tpintr_softint, sc);
+
mutex_enter(&l2tp_softcs.lock);
LIST_INSERT_HEAD(&l2tp_softcs.list, sc, l2tp_list);
mutex_exit(&l2tp_softcs.lock);
@@ -326,6 +338,15 @@
mutex_obj_free(lro->lr_lock);
}
+void
+l2tp_ifq_init_pc(void *p, void *arg __unused, struct cpu_info *ci __unused)
+{
+ struct ifqueue *ifq = p;
+
+ memset(ifq, 0, sizeof(*ifq));
+ ifq->ifq_maxlen = IFQ_MAXLEN;
+}
+
static int
l2tp_clone_destroy(struct ifnet *ifp)
{
@@ -336,13 +357,17 @@
l2tp_clear_session(sc);
l2tp_delete_tunnel(&sc->l2tp_ec.ec_if);
/*
- * To avoid for l2tp_transmit() to access sc->l2tp_var after free it.
+ * To avoid for l2tp_transmit() and l2tpintr_softint() to access
+ * sc->l2tp_var after free it.
*/
mutex_enter(&sc->l2tp_lock);
var = sc->l2tp_var;
l2tp_variant_update(sc, NULL);
mutex_exit(&sc->l2tp_lock);
+ softint_disestablish(sc->l2tp_si);
+ percpu_free(sc->l2tp_ifq_percpu, sizeof(struct ifqueue));
+
mutex_enter(&l2tp_softcs.lock);
LIST_REMOVE(sc, l2tp_list);
mutex_exit(&l2tp_softcs.lock);
@@ -363,6 +388,37 @@
}
static int
+l2tp_tx_enqueue(struct l2tp_variant *var, struct mbuf *m)
+{
+ struct l2tp_softc *sc;
+ struct ifnet *ifp;
+ struct ifqueue *ifq;
+ int s;
+
+ KASSERT(psref_held(&var->lv_psref, lv_psref_class));
+
+ sc = var->lv_softc;
+ ifp = &sc->l2tp_ec.ec_if;
+
+ s = splsoftnet();
+ ifq = percpu_getref(sc->l2tp_ifq_percpu);
+ if (IF_QFULL(ifq)) {
+ ifp->if_oerrors++;
+ percpu_putref(sc->l2tp_ifq_percpu);
+ splx(s);
+ m_freem(m);
+ return ENOBUFS;
+ }
+
+ IF_ENQUEUE(ifq, m);
+ percpu_putref(sc->l2tp_ifq_percpu);
+ softint_schedule(sc->l2tp_si);
+ /* counter is incremented in l2tpintr() */
+ splx(s);
+ return 0;
+}
+
+static int
l2tp_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
const struct rtentry *rt)
{
@@ -404,17 +460,7 @@
}
*mtod(m, int *) = dst->sa_family;
- IFQ_ENQUEUE(&ifp->if_snd, m, error);
- if (error)
- goto end;
-
- /*
- * direct call to avoid infinite loop at l2tpintr()
- */
- l2tpintr(var);
-
- error = 0;
-
+ error = l2tp_tx_enqueue(var, m);
end:
l2tp_putref_variant(var, &psref);
if (error)
@@ -424,12 +470,54 @@
}
static void
+l2tp_sendit(struct l2tp_variant *var, struct mbuf *m)
+{
+ int len;
+ int error;
+ struct l2tp_softc *sc;
+ struct ifnet *ifp;
+
+ KASSERT(psref_held(&var->lv_psref, lv_psref_class));
+
+ sc = var->lv_softc;
+ ifp = &sc->l2tp_ec.ec_if;
+
+ len = m->m_pkthdr.len;
+ m->m_flags &= ~(M_BCAST|M_MCAST);
+ bpf_mtap(ifp, m, BPF_D_OUT);
+
+ switch (var->lv_psrc->sa_family) {
+#ifdef INET
+ case AF_INET:
+ error = in_l2tp_output(var, m);
+ break;
+#endif
+#ifdef INET6
+ case AF_INET6:
+ error = in6_l2tp_output(var, m);
+ break;
+#endif
+ default:
+ m_freem(m);
+ error = ENETDOWN;
+ break;
+ }
+ if (error) {
+ ifp->if_oerrors++;
+ } else {
+ ifp->if_opackets++;
+ ifp->if_obytes += len;
+ }
+}
+
+static void
l2tpintr(struct l2tp_variant *var)
{
struct l2tp_softc *sc;
struct ifnet *ifp;
struct mbuf *m;
- int error;
+ struct ifqueue *ifq;
+ u_int cpuid = cpu_index(curcpu());
KASSERT(psref_held(&var->lv_psref, lv_psref_class));
@@ -438,45 +526,51 @@
/* output processing */
if (var->lv_my_sess_id == 0 || var->lv_peer_sess_id == 0) {
- IFQ_PURGE(&ifp->if_snd);
+ ifq = percpu_getref(sc->l2tp_ifq_percpu);
+ IF_PURGE(ifq);
+ percpu_putref(sc->l2tp_ifq_percpu);
+ if (cpuid == 0)
+ IFQ_PURGE(&ifp->if_snd);
return;
}
+ /* Currently, l2tpintr() is always called in softint context. */
+ ifq = percpu_getref(sc->l2tp_ifq_percpu);
for (;;) {
- int len;
-
- IFQ_DEQUEUE(&ifp->if_snd, m);
- if (m == NULL)
- break;
- len = m->m_pkthdr.len;
- m->m_flags &= ~(M_BCAST|M_MCAST);
- bpf_mtap(ifp, m, BPF_D_OUT);
- switch (var->lv_psrc->sa_family) {
-#ifdef INET
- case AF_INET:
- error = in_l2tp_output(var, m);
+ IF_DEQUEUE(ifq, m);
+ if (m != NULL)
+ l2tp_sendit(var, m);
+ else
break;
-#endif
-#ifdef INET6
- case AF_INET6:
- error = in6_l2tp_output(var, m);
- break;
-#endif
- default:
- m_freem(m);
- error = ENETDOWN;
- break;
- }
+ }
+ percpu_putref(sc->l2tp_ifq_percpu);
- if (error)
- ifp->if_oerrors++;
- else {
- ifp->if_opackets++;
- ifp->if_obytes += len;
+ if (cpuid == 0) {
+ for (;;) {
+ IFQ_DEQUEUE(&ifp->if_snd, m);
+ if (m != NULL)
+ l2tp_sendit(var, m);
+ else
+ break;
}
}
}
+static void
+l2tpintr_softint(void *arg)
+{
+ struct l2tp_variant *var;
+ struct psref psref;
+ struct l2tp_softc *sc = arg;
+
+ var = l2tp_getref_variant(sc, &psref);
+ if (var == NULL)
+ return;
+
+ l2tpintr(var);
+ l2tp_putref_variant(var, &psref);
+}
+
void
l2tp_input(struct mbuf *m, struct ifnet *ifp)
{
@@ -578,7 +672,7 @@
if (var->lv_psrc == NULL || var->lv_pdst == NULL)
return;
- l2tpintr(var);
+ softint_schedule(sc->l2tp_si);
l2tp_putref_variant(var, &psref);
}
Home |
Main Index |
Thread Index |
Old Index