Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys Make ipforward_rt and ip6_forward_rt percpu



details:   https://anonhg.NetBSD.org/src/rev/45d7cf466db2
branches:  trunk
changeset: 347522:45d7cf466db2
user:      ozaki-r <ozaki-r%NetBSD.org@localhost>
date:      Wed Aug 31 09:14:47 2016 +0000

description:
Make ipforward_rt and ip6_forward_rt percpu

Sharing one rtcache between CPUs is just a bad idea.

Reviewed by knakahara@

diffstat:

 sys/netinet/ip_input.c     |  30 ++++++++++++++++++++++--------
 sys/netinet6/ip6_forward.c |  32 +++++++++++++++++++++-----------
 sys/netinet6/ip6_input.c   |  18 +++++++++++++-----
 3 files changed, 56 insertions(+), 24 deletions(-)

diffs (truncated from 320 to 300 lines):

diff -r d747a4b67efe -r 45d7cf466db2 sys/netinet/ip_input.c
--- a/sys/netinet/ip_input.c    Wed Aug 31 09:03:59 2016 +0000
+++ b/sys/netinet/ip_input.c    Wed Aug 31 09:14:47 2016 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ip_input.c,v 1.339 2016/08/01 03:15:30 ozaki-r Exp $   */
+/*     $NetBSD: ip_input.c,v 1.340 2016/08/31 09:14:47 ozaki-r Exp $   */
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.339 2016/08/01 03:15:30 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.340 2016/08/31 09:14:47 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -225,7 +225,7 @@
 ipid_state_t *         ip_ids                  __read_mostly;
 percpu_t *             ipstat_percpu           __read_mostly;
 
-static struct route    ipforward_rt            __cacheline_aligned;
+static percpu_t                *ipforward_rt_percpu    __cacheline_aligned;
 
 uint16_t ip_id;
 
@@ -345,6 +345,10 @@
 #endif /* MBUFTRACE */
 
        ipstat_percpu = percpu_alloc(sizeof(uint64_t) * IP_NSTATS);
+
+       ipforward_rt_percpu = percpu_alloc(sizeof(struct route));
+       if (ipforward_rt_percpu == NULL)
+               panic("failed to allocate ipforward_rt_percpu");
 }
 
 static struct in_ifaddr *
@@ -1163,11 +1167,14 @@
                struct sockaddr         dst;
                struct sockaddr_in      dst4;
        } u;
+       struct route *ro;
 
        sockaddr_in_init(&u.dst4, &dst, 0);
 
        SOFTNET_LOCK();
-       rt = rtcache_lookup(&ipforward_rt, &u.dst);
+       ro = percpu_getref(ipforward_rt_percpu);
+       rt = rtcache_lookup(ro, &u.dst);
+       percpu_putref(ipforward_rt_percpu);
        SOFTNET_UNLOCK();
        if (rt == NULL)
                return NULL;
@@ -1300,6 +1307,7 @@
                struct sockaddr_in      dst4;
        } u;
        uint64_t *ips;
+       struct route *ro;
 
        KASSERTMSG(cpu_softintr_p(), "ip_forward: not in the software "
            "interrupt handler; synchronization assumptions violated");
@@ -1331,7 +1339,9 @@
 
        sockaddr_in_init(&u.dst4, &ip->ip_dst, 0);
 
-       if ((rt = rtcache_lookup(&ipforward_rt, &u.dst)) == NULL) {
+       ro = percpu_getref(ipforward_rt_percpu);
+       if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) {
+               percpu_putref(ipforward_rt_percpu);
                icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, dest, 0);
                SOFTNET_UNLOCK();
                return;
@@ -1376,7 +1386,7 @@
                }
        }
 
-       error = ip_output(m, NULL, &ipforward_rt,
+       error = ip_output(m, NULL, ro,
            (IP_FORWARDING | (ip_directedbcast ? IP_ALLOWBROADCAST : 0)),
            NULL, NULL);
 
@@ -1398,17 +1408,19 @@
        if (mcopy) {
 #ifdef GATEWAY
                if (mcopy->m_flags & M_CANFASTFWD)
-                       ipflow_create(&ipforward_rt, mcopy);
+                       ipflow_create(ro, mcopy);
 #endif
                m_freem(mcopy);
        }
 
+       percpu_putref(ipforward_rt_percpu);
        SOFTNET_UNLOCK();
        return;
 
 redirect:
 error:
        if (mcopy == NULL) {
+               percpu_putref(ipforward_rt_percpu);
                SOFTNET_UNLOCK();
                return;
        }
@@ -1432,7 +1444,7 @@
                type = ICMP_UNREACH;
                code = ICMP_UNREACH_NEEDFRAG;
 
-               if ((rt = rtcache_validate(&ipforward_rt)) != NULL)
+               if ((rt = rtcache_validate(ro)) != NULL)
                        destmtu = rt->rt_ifp->if_mtu;
 #ifdef IPSEC
                if (ipsec_used)
@@ -1450,10 +1462,12 @@
                 */
                if (mcopy)
                        m_freem(mcopy);
+               percpu_putref(ipforward_rt_percpu);
                SOFTNET_UNLOCK();
                return;
        }
        icmp_error(mcopy, type, code, dest, destmtu);
+       percpu_putref(ipforward_rt_percpu);
        SOFTNET_UNLOCK();
 }
 
diff -r d747a4b67efe -r 45d7cf466db2 sys/netinet6/ip6_forward.c
--- a/sys/netinet6/ip6_forward.c        Wed Aug 31 09:03:59 2016 +0000
+++ b/sys/netinet6/ip6_forward.c        Wed Aug 31 09:14:47 2016 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ip6_forward.c,v 1.80 2016/06/28 02:02:56 ozaki-r Exp $ */
+/*     $NetBSD: ip6_forward.c,v 1.81 2016/08/31 09:14:47 ozaki-r Exp $ */
 /*     $KAME: ip6_forward.c,v 1.109 2002/09/11 08:10:17 sakane Exp $   */
 
 /*
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.80 2016/06/28 02:02:56 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.81 2016/08/31 09:14:47 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_gateway.h"
@@ -48,6 +48,7 @@
 #include <sys/time.h>
 #include <sys/kernel.h>
 #include <sys/syslog.h>
+#include <sys/percpu.h>
 
 #include <net/if.h>
 #include <net/route.h>
@@ -72,7 +73,7 @@
 
 #include <net/net_osdep.h>
 
-struct route ip6_forward_rt;
+extern percpu_t *ip6_forward_rt_percpu;
 
 extern pfil_head_t *inet6_pfil_hook;   /* XXX */
 
@@ -135,6 +136,7 @@
        struct in6_addr src_in6, dst_in6;
        struct ifnet *rcvif = NULL;
        struct psref psref;
+       struct route *ro;
 #ifdef IPSEC
        int needipsec = 0;
        struct secpolicy *sp = NULL;
@@ -203,6 +205,7 @@
        }
 #endif /* IPSEC */
 
+       ro = percpu_getref(ip6_forward_rt_percpu);
        if (srcrt) {
                union {
                        struct sockaddr         dst;
@@ -210,17 +213,18 @@
                } u;
 
                sockaddr_in6_init(&u.dst6, &ip6->ip6_dst, 0, 0, 0);
-               if ((rt = rtcache_lookup(&ip6_forward_rt, &u.dst)) == NULL) {
+               if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) {
                        IP6_STATINC(IP6_STAT_NOROUTE);
                        /* XXX in6_ifstat_inc(rt->rt_ifp, ifs6_in_noroute) */
                        if (mcopy) {
                                icmp6_error(mcopy, ICMP6_DST_UNREACH,
                                            ICMP6_DST_UNREACH_NOROUTE, 0);
                        }
+                       percpu_putref(ip6_forward_rt_percpu);
                        goto drop;
                }
-       } else if ((rt = rtcache_validate(&ip6_forward_rt)) == NULL &&
-                  (rt = rtcache_update(&ip6_forward_rt, 1)) == NULL) {
+       } else if ((rt = rtcache_validate(ro)) == NULL &&
+                  (rt = rtcache_update(ro, 1)) == NULL) {
                /*
                 * rtcache_getdst(ip6_forward_rt)->sin6_addr was equal to
                 * ip6->ip6_dst
@@ -231,9 +235,11 @@
                        icmp6_error(mcopy, ICMP6_DST_UNREACH,
                            ICMP6_DST_UNREACH_NOROUTE, 0);
                }
+               percpu_putref(ip6_forward_rt_percpu);
                goto drop;
        }
-       dst = satocsin6(rtcache_getdst(&ip6_forward_rt));
+       dst = satocsin6(rtcache_getdst(ro));
+       percpu_putref(ip6_forward_rt_percpu);
 
        /*
         * Source scope check: if a packet can't be delivered to its
@@ -318,10 +324,11 @@
         */
        if (rt->rt_ifp == rcvif && !srcrt && ip6_sendredirects &&
            (rt->rt_flags & (RTF_DYNAMIC|RTF_MODIFIED)) == 0) {
+               ro = percpu_getref(ip6_forward_rt_percpu);
                if ((rt->rt_ifp->if_flags & IFF_POINTOPOINT) &&
-                   nd6_is_addr_neighbor(
-                       satocsin6(rtcache_getdst(&ip6_forward_rt)),
-                       rt->rt_ifp)) {
+                   nd6_is_addr_neighbor(satocsin6(rtcache_getdst(ro)),
+                                        rt->rt_ifp)) {
+                       percpu_putref(ip6_forward_rt_percpu);
                        /*
                         * If the incoming interface is equal to the outgoing
                         * one, the link attached to the interface is
@@ -341,6 +348,7 @@
                                    ICMP6_DST_UNREACH_ADDR, 0);
                        goto drop;
                }
+               percpu_putref(ip6_forward_rt_percpu);
                type = ND_REDIRECT;
        }
 
@@ -409,8 +417,10 @@
                        IP6_STATINC(IP6_STAT_REDIRECTSENT);
                else {
 #ifdef GATEWAY
+                       ro = percpu_getref(ip6_forward_rt_percpu);
                        if (m->m_flags & M_CANFASTFWD)
-                               ip6flow_create(&ip6_forward_rt, m);
+                               ip6flow_create(ro, m);
+                       percpu_putref(ip6_forward_rt_percpu);
 #endif
                        if (mcopy)
                                goto freecopy;
diff -r d747a4b67efe -r 45d7cf466db2 sys/netinet6/ip6_input.c
--- a/sys/netinet6/ip6_input.c  Wed Aug 31 09:03:59 2016 +0000
+++ b/sys/netinet6/ip6_input.c  Wed Aug 31 09:14:47 2016 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ip6_input.c,v 1.166 2016/08/02 04:50:16 knakahara Exp $        */
+/*     $NetBSD: ip6_input.c,v 1.167 2016/08/31 09:14:47 ozaki-r Exp $  */
 /*     $KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $     */
 
 /*
@@ -62,7 +62,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.166 2016/08/02 04:50:16 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.167 2016/08/31 09:14:47 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_gateway.h"
@@ -87,6 +87,7 @@
 #include <sys/proc.h>
 #include <sys/sysctl.h>
 #include <sys/cprng.h>
+#include <sys/percpu.h>
 
 #include <net/if.h>
 #include <net/if_types.h>
@@ -143,6 +144,8 @@
 
 percpu_t *ip6stat_percpu;
 
+percpu_t *ip6_forward_rt_percpu __cacheline_aligned;
+
 static void ip6_init2(void);
 static void ip6intr(void *);
 static struct m_tag *ip6_setdstifaddr(struct mbuf *, const struct in6_ifaddr *);
@@ -194,6 +197,10 @@
        KASSERT(inet6_pfil_hook != NULL);
 
        ip6stat_percpu = percpu_alloc(sizeof(uint64_t) * IP6_NSTATS);
+
+       ip6_forward_rt_percpu = percpu_alloc(sizeof(struct route));
+       if (ip6_forward_rt_percpu == NULL)
+               panic("failed to alllocate ip6_forward_rt_percpu");
 }
 
 static void
@@ -239,8 +246,6 @@
        mutex_exit(softnet_lock);
 }
 
-extern struct  route ip6_forward_rt;
-
 void
 ip6_input(struct mbuf *m, struct ifnet *rcvif)



Home | Main Index | Thread Index | Old Index