Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys Avoid having a rtcache directly in a percpu storage
details: https://anonhg.NetBSD.org/src/rev/a4b1b3b07dc0
branches: trunk
changeset: 459631:a4b1b3b07dc0
user: ozaki-r <ozaki-r%NetBSD.org@localhost>
date: Thu Sep 19 04:08:29 2019 +0000
description:
Avoid having a rtcache directly in a percpu storage
percpu(9) has a certain memory storage for each CPU and provides it by the piece
to users. If the storages went short, percpu(9) enlarges them by allocating new
larger memory areas, replacing old ones with them and destroying the old ones.
A percpu storage referenced by a pointer gotten via percpu_getref can be
destroyed by the mechanism after a running thread sleeps even if percpu_putref
has not been called.
Using rtcache, i.e., packet processing, typically involves sleepable operations
such as rwlock so we must avoid dereferencing a rtcache that is directly stored
in a percpu storage during packet processing. Address this situation by having
just a pointer to a rtcache in a percpu storage instead.
Reviewed by knakahara@ and yamaguchi@
diffstat:
sys/net/route.c | 27 +++++++++++++++++++++++++--
sys/net/route.h | 21 ++++++++++++++++++++-
sys/netinet/ip_input.c | 20 ++++++++++----------
sys/netinet6/ip6_forward.c | 8 ++++----
sys/netinet6/ip6_input.c | 16 ++++++++--------
sys/netipsec/ipsec_output.c | 10 +++++-----
6 files changed, 72 insertions(+), 30 deletions(-)
diffs (truncated from 322 to 300 lines):
diff -r c886672d080f -r a4b1b3b07dc0 sys/net/route.c
--- a/sys/net/route.c Thu Sep 19 03:37:31 2019 +0000
+++ b/sys/net/route.c Thu Sep 19 04:08:29 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: route.c,v 1.219 2019/05/17 03:34:26 ozaki-r Exp $ */
+/* $NetBSD: route.c,v 1.220 2019/09/19 04:08:29 ozaki-r Exp $ */
/*-
* Copyright (c) 1998, 2008 The NetBSD Foundation, Inc.
@@ -97,7 +97,7 @@
#endif
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: route.c,v 1.219 2019/05/17 03:34:26 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: route.c,v 1.220 2019/09/19 04:08:29 ozaki-r Exp $");
#include <sys/param.h>
#ifdef RTFLUSH_DEBUG
@@ -2217,6 +2217,29 @@
return 0;
}
+static void
+rtcache_percpu_init_cpu(void *p, void *arg __unused, struct cpu_info *ci __unused)
+{
+ struct route **rop = p;
+
+ /*
+ * We can't have struct route as percpu data because it can be destroyed
+ * over a memory enlargement processing of percpu.
+ */
+ *rop = kmem_zalloc(sizeof(**rop), KM_SLEEP);
+}
+
+percpu_t *
+rtcache_percpu_alloc(void)
+{
+ percpu_t *pc;
+
+ pc = percpu_alloc(sizeof(struct route *));
+ percpu_foreach(pc, rtcache_percpu_init_cpu, NULL);
+
+ return pc;
+}
+
const struct sockaddr *
rt_settag(struct rtentry *rt, const struct sockaddr *tag)
{
diff -r c886672d080f -r a4b1b3b07dc0 sys/net/route.h
--- a/sys/net/route.h Thu Sep 19 03:37:31 2019 +0000
+++ b/sys/net/route.h Thu Sep 19 04:08:29 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: route.h,v 1.124 2019/08/22 21:14:45 roy Exp $ */
+/* $NetBSD: route.h,v 1.125 2019/09/19 04:08:29 ozaki-r Exp $ */
/*
* Copyright (c) 1980, 1986, 1993
@@ -42,6 +42,7 @@
#include <sys/rwlock.h>
#include <sys/condvar.h>
#include <sys/pserialize.h>
+#include <sys/percpu.h>
#endif
#include <sys/psref.h>
@@ -507,6 +508,24 @@
void rtcache_unref(struct rtentry *, struct route *);
+percpu_t *
+ rtcache_percpu_alloc(void);
+
+static inline struct route *
+rtcache_percpu_getref(percpu_t *pc)
+{
+
+ return *(struct route **)percpu_getref(pc);
+}
+
+static inline void
+rtcache_percpu_putref(percpu_t *pc)
+{
+
+ percpu_putref(pc);
+}
+
+
/* rtsock */
void rt_ieee80211msg(struct ifnet *, int, void *, size_t);
void rt_ifannouncemsg(struct ifnet *, int);
diff -r c886672d080f -r a4b1b3b07dc0 sys/netinet/ip_input.c
--- a/sys/netinet/ip_input.c Thu Sep 19 03:37:31 2019 +0000
+++ b/sys/netinet/ip_input.c Thu Sep 19 04:08:29 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ip_input.c,v 1.390 2019/09/15 21:00:15 bouyer Exp $ */
+/* $NetBSD: ip_input.c,v 1.391 2019/09/19 04:08:29 ozaki-r Exp $ */
/*
* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.390 2019/09/15 21:00:15 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.391 2019/09/19 04:08:29 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_inet.h"
@@ -308,7 +308,7 @@
#endif
ipstat_percpu = percpu_alloc(sizeof(uint64_t) * IP_NSTATS);
- ipforward_rt_percpu = percpu_alloc(sizeof(struct route));
+ ipforward_rt_percpu = rtcache_percpu_alloc();
ip_mtudisc_timeout_q = rt_timer_queue_create(ip_mtudisc_timeout);
}
@@ -1184,16 +1184,16 @@
sockaddr_in_init(&u.dst4, &dst, 0);
- ro = percpu_getref(ipforward_rt_percpu);
+ ro = rtcache_percpu_getref(ipforward_rt_percpu);
rt = rtcache_lookup(ro, &u.dst);
if (rt == NULL) {
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
return NULL;
}
ia4_acquire(ifatoia(rt->rt_ifa), psref);
rtcache_unref(rt, ro);
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
return ifatoia(rt->rt_ifa);
}
@@ -1372,7 +1372,7 @@
ro = percpu_getref(ipforward_rt_percpu);
rt = rtcache_lookup(ro, &u.dst);
if (rt == NULL) {
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, dest, 0);
return;
}
@@ -1444,13 +1444,13 @@
m_freem(mcopy);
}
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
return;
redirect:
error:
if (mcopy == NULL) {
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
return;
}
@@ -1493,7 +1493,7 @@
*/
if (mcopy)
m_freem(mcopy);
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
return;
}
icmp_error(mcopy, type, code, dest, destmtu);
diff -r c886672d080f -r a4b1b3b07dc0 sys/netinet6/ip6_forward.c
--- a/sys/netinet6/ip6_forward.c Thu Sep 19 03:37:31 2019 +0000
+++ b/sys/netinet6/ip6_forward.c Thu Sep 19 04:08:29 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ip6_forward.c,v 1.96 2019/05/13 07:47:59 ozaki-r Exp $ */
+/* $NetBSD: ip6_forward.c,v 1.97 2019/09/19 04:08:29 ozaki-r Exp $ */
/* $KAME: ip6_forward.c,v 1.109 2002/09/11 08:10:17 sakane Exp $ */
/*
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.96 2019/05/13 07:47:59 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.97 2019/09/19 04:08:29 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_gateway.h"
@@ -197,7 +197,7 @@
}
#endif
- ro = percpu_getref(ip6_forward_rt_percpu);
+ ro = rtcache_percpu_getref(ip6_forward_rt_percpu);
if (srcrt) {
union {
struct sockaddr dst;
@@ -470,7 +470,7 @@
#endif
rtcache_unref(rt, ro);
if (ro != NULL)
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
if (rcvif != NULL)
m_put_rcvif_psref(rcvif, &psref);
return;
diff -r c886672d080f -r a4b1b3b07dc0 sys/netinet6/ip6_input.c
--- a/sys/netinet6/ip6_input.c Thu Sep 19 03:37:31 2019 +0000
+++ b/sys/netinet6/ip6_input.c Thu Sep 19 04:08:29 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ip6_input.c,v 1.209 2019/09/15 21:00:15 bouyer Exp $ */
+/* $NetBSD: ip6_input.c,v 1.210 2019/09/19 04:08:29 ozaki-r Exp $ */
/* $KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $ */
/*
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.209 2019/09/15 21:00:15 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.210 2019/09/19 04:08:29 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_gateway.h"
@@ -194,7 +194,7 @@
KASSERT(inet6_pfil_hook != NULL);
ip6stat_percpu = percpu_alloc(sizeof(uint64_t) * IP6_NSTATS);
- ip6_forward_rt_percpu = percpu_alloc(sizeof(struct route));
+ ip6_forward_rt_percpu = rtcache_percpu_alloc();
}
static void
@@ -591,7 +591,7 @@
ICMP6_PARAMPROB_HEADER,
(char *)&ip6->ip6_plen - (char *)ip6);
rtcache_unref(rt, ro);
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
return;
}
IP6_EXTHDR_GET(hbh, struct ip6_hbh *, m, sizeof(struct ip6_hdr),
@@ -599,7 +599,7 @@
if (hbh == NULL) {
IP6_STATINC(IP6_STAT_TOOSHORT);
rtcache_unref(rt, ro);
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
return;
}
KASSERT(IP6_HDR_ALIGNED_P(hbh));
@@ -653,7 +653,7 @@
if (error != 0) {
rtcache_unref(rt, ro);
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
IP6_STATINC(IP6_STAT_CANTFORWARD);
goto bad;
}
@@ -662,7 +662,7 @@
goto bad_unref;
} else if (!ours) {
rtcache_unref(rt, ro);
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
ip6_forward(m, srcrt);
return;
}
@@ -703,7 +703,7 @@
rtcache_unref(rt, ro);
rt = NULL;
}
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
rh_present = 0;
frg_present = 0;
diff -r c886672d080f -r a4b1b3b07dc0 sys/netipsec/ipsec_output.c
--- a/sys/netipsec/ipsec_output.c Thu Sep 19 03:37:31 2019 +0000
+++ b/sys/netipsec/ipsec_output.c Thu Sep 19 04:08:29 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ipsec_output.c,v 1.82 2018/12/26 08:58:51 knakahara Exp $ */
+/* $NetBSD: ipsec_output.c,v 1.83 2019/09/19 04:08:30 ozaki-r Exp $ */
/*
* Copyright (c) 2002, 2003 Sam Leffler, Errno Consulting
@@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipsec_output.c,v 1.82 2018/12/26 08:58:51 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipsec_output.c,v 1.83 2019/09/19 04:08:30 ozaki-r Exp $");
#if defined(_KERNEL_OPT)
#include "opt_inet.h"
@@ -113,7 +113,7 @@
KASSERT(af == AF_INET || af == AF_INET6);
Home |
Main Index |
Thread Index |
Old Index