Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys Use percpu_foreach_xcall() to gather volatile per-cpu co...
details: https://anonhg.NetBSD.org/src/rev/40f6b2215fb7
branches: trunk
changeset: 744589:40f6b2215fb7
user: thorpej <thorpej%NetBSD.org@localhost>
date: Fri Feb 07 12:35:33 2020 +0000
description:
Use percpu_foreach_xcall() to gather volatile per-cpu counters. These
must be serialized against the interrupts / soft-interrupts in which
they're manipulated, as well as protected from non-atomic 64-bit memory
loads on 32-bit platforms.
diffstat:
sys/net/bpf.c | 12 +++++++++---
sys/net/if_stats.c | 8 +++++---
sys/net/net_stats.c | 8 +++++---
sys/net/npf/npf.c | 9 ++++++---
sys/net/pktqueue.c | 12 +++++++++---
sys/netinet/wqinput.c | 6 ++++--
sys/netipsec/key.c | 18 +++++++++++-------
7 files changed, 49 insertions(+), 24 deletions(-)
diffs (297 lines):
diff -r 3183beaecd04 -r 40f6b2215fb7 sys/net/bpf.c
--- a/sys/net/bpf.c Fri Feb 07 11:55:22 2020 +0000
+++ b/sys/net/bpf.c Fri Feb 07 12:35:33 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: bpf.c,v 1.234 2020/02/01 02:54:02 riastradh Exp $ */
+/* $NetBSD: bpf.c,v 1.235 2020/02/07 12:35:33 thorpej Exp $ */
/*
* Copyright (c) 1990, 1991, 1993
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.234 2020/02/01 02:54:02 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.235 2020/02/07 12:35:33 thorpej Exp $");
#if defined(_KERNEL_OPT)
#include "opt_bpf.h"
@@ -78,6 +78,7 @@
#include <sys/percpu.h>
#include <sys/pserialize.h>
#include <sys/lwp.h>
+#include <sys/xcall.h>
#include <net/if.h>
#include <net/slip.h>
@@ -2395,9 +2396,13 @@
struct bpf_stat *const stats = p;
struct bpf_stat *sum = arg;
+ int s = splnet();
+
sum->bs_recv += stats->bs_recv;
sum->bs_drop += stats->bs_drop;
sum->bs_capt += stats->bs_capt;
+
+ splx(s);
}
static int
@@ -2410,7 +2415,8 @@
memset(&sum, 0, sizeof(sum));
node = *rnode;
- percpu_foreach(bpf_gstats_percpu, bpf_stats, &sum);
+ percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ bpf_stats, &sum);
node.sysctl_data = ∑
node.sysctl_size = sizeof(sum);
diff -r 3183beaecd04 -r 40f6b2215fb7 sys/net/if_stats.c
--- a/sys/net/if_stats.c Fri Feb 07 11:55:22 2020 +0000
+++ b/sys/net/if_stats.c Fri Feb 07 12:35:33 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: if_stats.c,v 1.1 2020/01/29 03:16:28 thorpej Exp $ */
+/* $NetBSD: if_stats.c,v 1.2 2020/02/07 12:35:33 thorpej Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -30,11 +30,12 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_stats.c,v 1.1 2020/01/29 03:16:28 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_stats.c,v 1.2 2020/02/07 12:35:33 thorpej Exp $");
#include <sys/param.h>
#include <sys/mbuf.h>
#include <sys/systm.h>
+#include <sys/xcall.h>
#include <net/if.h>
@@ -123,7 +124,8 @@
};
memset(ifi, 0, sizeof(*ifi));
- percpu_foreach(ifp->if_stats, if_stats_to_if_data_cb, &ctx);
+ percpu_foreach_xcall(ifp->if_stats, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ if_stats_to_if_data_cb, &ctx);
}
#else /* ! __IF_STATS_PERCPU */
diff -r 3183beaecd04 -r 40f6b2215fb7 sys/net/net_stats.c
--- a/sys/net/net_stats.c Fri Feb 07 11:55:22 2020 +0000
+++ b/sys/net/net_stats.c Fri Feb 07 12:35:33 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: net_stats.c,v 1.5 2017/06/01 02:45:14 chs Exp $ */
+/* $NetBSD: net_stats.c,v 1.6 2020/02/07 12:35:33 thorpej Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -30,12 +30,13 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: net_stats.c,v 1.5 2017/06/01 02:45:14 chs Exp $");
+__KERNEL_RCSID(0, "$NetBSD: net_stats.c,v 1.6 2020/02/07 12:35:33 thorpej Exp $");
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/kmem.h>
+#include <sys/xcall.h>
#include <net/net_stats.h>
@@ -80,7 +81,8 @@
ctx.ctx_counters = counters;
ctx.ctx_ncounters = ncounters;
- percpu_foreach(stat, netstat_convert_to_user_cb, &ctx);
+ percpu_foreach_xcall(stat, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ netstat_convert_to_user_cb, &ctx);
node = *rnode;
node.sysctl_data = counters;
diff -r 3183beaecd04 -r 40f6b2215fb7 sys/net/npf/npf.c
--- a/sys/net/npf/npf.c Fri Feb 07 11:55:22 2020 +0000
+++ b/sys/net/npf/npf.c Fri Feb 07 12:35:33 2020 +0000
@@ -33,7 +33,7 @@
#ifdef _KERNEL
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf.c,v 1.41 2019/08/25 13:21:03 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf.c,v 1.42 2020/02/07 12:35:33 thorpej Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -41,6 +41,7 @@
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/percpu.h>
+#include <sys/xcall.h>
#endif
#include "npf_impl.h"
@@ -201,11 +202,13 @@
npfk_stats(npf_t *npf, uint64_t *buf)
{
memset(buf, 0, NPF_STATS_SIZE);
- percpu_foreach(npf->stats_percpu, npf_stats_collect, buf);
+ percpu_foreach_xcall(npf->stats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ npf_stats_collect, buf);
}
__dso_public void
npfk_stats_clear(npf_t *npf)
{
- percpu_foreach(npf->stats_percpu, npf_stats_clear_cb, NULL);
+ percpu_foreach_xcall(npf->stats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ npf_stats_clear_cb, NULL);
}
diff -r 3183beaecd04 -r 40f6b2215fb7 sys/net/pktqueue.c
--- a/sys/net/pktqueue.c Fri Feb 07 11:55:22 2020 +0000
+++ b/sys/net/pktqueue.c Fri Feb 07 12:35:33 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pktqueue.c,v 1.10 2018/08/10 07:24:09 msaitoh Exp $ */
+/* $NetBSD: pktqueue.c,v 1.11 2020/02/07 12:35:33 thorpej Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -36,7 +36,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.10 2018/08/10 07:24:09 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.11 2020/02/07 12:35:33 thorpej Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -48,6 +48,7 @@
#include <sys/mbuf.h>
#include <sys/proc.h>
#include <sys/percpu.h>
+#include <sys/xcall.h>
#include <net/pktqueue.h>
@@ -161,9 +162,13 @@
const pktq_counters_t *c = mem;
pktq_counters_t *sum = arg;
+ int s = splnet();
+
for (u_int i = 0; i < PQCNT_NCOUNTERS; i++) {
sum->count[i] += c->count[i];
}
+
+ splx(s);
}
uint64_t
@@ -173,7 +178,8 @@
if (c != PKTQ_MAXLEN) {
memset(&sum, 0, sizeof(sum));
- percpu_foreach(pq->pq_counters, pktq_collect_counts, &sum);
+ percpu_foreach_xcall(pq->pq_counters,
+ XC_HIGHPRI_IPL(IPL_SOFTNET), pktq_collect_counts, &sum);
}
switch (c) {
case PKTQ_NITEMS:
diff -r 3183beaecd04 -r 40f6b2215fb7 sys/netinet/wqinput.c
--- a/sys/netinet/wqinput.c Fri Feb 07 11:55:22 2020 +0000
+++ b/sys/netinet/wqinput.c Fri Feb 07 12:35:33 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: wqinput.c,v 1.7 2020/02/01 12:54:51 riastradh Exp $ */
+/* $NetBSD: wqinput.c,v 1.8 2020/02/07 12:35:33 thorpej Exp $ */
/*-
* Copyright (c) 2017 Internet Initiative Japan Inc.
@@ -41,6 +41,7 @@
#include <sys/queue.h>
#include <sys/percpu.h>
#include <sys/sysctl.h>
+#include <sys/xcall.h>
#include <net/if.h>
#include <netinet/wqinput.h>
@@ -98,7 +99,8 @@
node = *rnode;
wqi = node.sysctl_data;
- percpu_foreach(wqi->wqi_worklists, wqinput_drops, &sum);
+ percpu_foreach_xcall(wqi->wqi_worklists, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ wqinput_drops, &sum);
node.sysctl_data = ∑
error = sysctl_lookup(SYSCTLFN_CALL(&node));
diff -r 3183beaecd04 -r 40f6b2215fb7 sys/netipsec/key.c
--- a/sys/netipsec/key.c Fri Feb 07 11:55:22 2020 +0000
+++ b/sys/netipsec/key.c Fri Feb 07 12:35:33 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: key.c,v 1.269 2019/11/14 03:17:08 knakahara Exp $ */
+/* $NetBSD: key.c,v 1.270 2020/02/07 12:35:33 thorpej Exp $ */
/* $FreeBSD: key.c,v 1.3.2.3 2004/02/14 22:23:23 bms Exp $ */
/* $KAME: key.c,v 1.191 2001/06/27 10:46:49 sakane Exp $ */
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.269 2019/11/14 03:17:08 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.270 2020/02/07 12:35:33 thorpej Exp $");
/*
* This code is referred to RFC 2367
@@ -73,6 +73,7 @@
#include <sys/localcount.h>
#include <sys/pserialize.h>
#include <sys/hash.h>
+#include <sys/xcall.h>
#include <net/if.h>
#include <net/route.h>
@@ -3961,7 +3962,8 @@
time_mono_to_wall(lt.sadb_lifetime_addtime);
lt.sadb_lifetime_usetime =
time_mono_to_wall(lt.sadb_lifetime_usetime);
- percpu_foreach(sav->lft_c_counters_percpu,
+ percpu_foreach_xcall(sav->lft_c_counters_percpu,
+ XC_HIGHPRI_IPL(IPL_SOFTNET),
key_sum_lifetime_counters, sum);
lt.sadb_lifetime_allocations =
sum[LIFETIME_COUNTER_ALLOCATIONS];
@@ -5027,7 +5029,8 @@
uint64_t lft_c_bytes = 0;
lifetime_counters_t sum = {0};
- percpu_foreach(sav->lft_c_counters_percpu,
+ percpu_foreach_xcall(sav->lft_c_counters_percpu,
+ XC_HIGHPRI_IPL(IPL_SOFTNET),
key_sum_lifetime_counters, sum);
lft_c_bytes = sum[LIFETIME_COUNTER_BYTES];
@@ -5089,7 +5092,8 @@
uint64_t lft_c_bytes = 0;
lifetime_counters_t sum = {0};
- percpu_foreach(sav->lft_c_counters_percpu,
+ percpu_foreach_xcall(sav->lft_c_counters_percpu,
+ XC_HIGHPRI_IPL(IPL_SOFTNET),
key_sum_lifetime_counters, sum);
lft_c_bytes = sum[LIFETIME_COUNTER_BYTES];
@@ -7425,8 +7429,8 @@
lt = mtod(m, struct sadb_lifetime *);
lt->sadb_lifetime_len = PFKEY_UNIT64(sizeof(struct sadb_lifetime));
lt->sadb_lifetime_exttype = SADB_EXT_LIFETIME_CURRENT;
- percpu_foreach(sav->lft_c_counters_percpu,
- key_sum_lifetime_counters, sum);
+ percpu_foreach_xcall(sav->lft_c_counters_percpu,
+ XC_HIGHPRI_IPL(IPL_SOFTNET), key_sum_lifetime_counters, sum);
lt->sadb_lifetime_allocations = sum[LIFETIME_COUNTER_ALLOCATIONS];
lt->sadb_lifetime_bytes = sum[LIFETIME_COUNTER_BYTES];
lt->sadb_lifetime_addtime =
Home |
Main Index |
Thread Index |
Old Index