Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys - Extend the per-CPU counters matt@ did to include all o...
details: https://anonhg.NetBSD.org/src/rev/8d89a6b93a46
branches: trunk
changeset: 847285:8d89a6b93a46
user: ad <ad%NetBSD.org@localhost>
date: Mon Dec 16 22:47:54 2019 +0000
description:
- Extend the per-CPU counters matt@ did to include all of the hot counters
in UVM, excluding uvmexp.free, which needs special treatment and will be
done with a separate commit. Cuts system time for a build by 20-25% on
a 48 CPU machine w/DIAGNOSTIC.
- Avoid 64-bit integer divide on every fault (for rnd_add_uint32).
diffstat:
sys/kern/kern_cpu.c | 92 +++++++++++++++++-
sys/kern/kern_fork.c | 14 +-
sys/kern/kern_softint.c | 10 +-
sys/kern/vfs_vnode.c | 14 +-
sys/kern/vfs_vnops.c | 12 +-
sys/miscfs/procfs/procfs_linux.c | 41 ++++---
sys/rump/librump/rumpkern/MAINBUS.ioconf | 3 +
sys/rump/librump/rumpkern/Makefile.rumpkern | 3 +-
sys/rump/librump/rumpkern/emul.c | 12 +-
sys/rump/librump/rumpkern/intr.c | 11 +-
sys/rump/librump/rumpkern/scheduler.c | 14 +-
sys/sys/cpu_data.h | 100 +++++++++++++++++-
sys/uvm/uvm_extern.h | 19 ++-
sys/uvm/uvm_fault.c | 93 +++++++-----------
sys/uvm/uvm_fault_i.h | 6 +-
sys/uvm/uvm_glue.c | 6 +-
sys/uvm/uvm_meter.c | 144 +++++++++++++++++++--------
sys/uvm/uvm_page.c | 46 ++++----
sys/uvm/uvm_pdpolicy_clock.c | 25 +++-
sys/uvm/uvm_pglist.c | 8 +-
sys/uvm/uvm_stat.c | 76 ++++++++-----
21 files changed, 480 insertions(+), 269 deletions(-)
diffs (truncated from 1609 to 300 lines):
diff -r 76fab308302b -r 8d89a6b93a46 sys/kern/kern_cpu.c
--- a/sys/kern/kern_cpu.c Mon Dec 16 22:22:11 2019 +0000
+++ b/sys/kern/kern_cpu.c Mon Dec 16 22:47:54 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_cpu.c,v 1.81 2019/12/04 09:34:13 wiz Exp $ */
+/* $NetBSD: kern_cpu.c,v 1.82 2019/12/16 22:47:54 ad Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019 The NetBSD Foundation, Inc.
@@ -56,9 +56,11 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.81 2019/12/04 09:34:13 wiz Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.82 2019/12/16 22:47:54 ad Exp $");
+#ifdef _KERNEL_OPT
#include "opt_cpu_ucode.h"
+#endif
#include <sys/param.h>
#include <sys/systm.h>
@@ -120,6 +122,7 @@
int ncpuonline __read_mostly;
bool mp_online __read_mostly;
static bool cpu_topology_present __read_mostly;
+int64_t cpu_counts[CPU_COUNT_MAX];
/* An array of CPUs. There are ncpu entries. */
struct cpu_info **cpu_infos __read_mostly;
@@ -305,6 +308,7 @@
return error;
}
+#ifndef _RUMPKERNEL
struct cpu_info *
cpu_lookup(u_int idx)
{
@@ -327,6 +331,7 @@
return ci;
}
+#endif
static void
cpu_xc_offline(struct cpu_info *ci, void *unused)
@@ -830,3 +835,86 @@
return error;
}
#endif
+
+/*
+ * Adjust one count, for a counter that's NOT updated from interrupt
+ * context. Hardly worth making an inline due to preemption stuff.
+ */
+void
+cpu_count(enum cpu_count idx, int64_t delta)
+{
+ lwp_t *l = curlwp;
+ KPREEMPT_DISABLE(l);
+ l->l_cpu->ci_counts[idx] += delta;
+ KPREEMPT_ENABLE(l);
+}
+
+/*
+ * Fetch fresh sum total for all counts. Expensive - don't call often.
+ */
+void
+cpu_count_sync_all(void)
+{
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *ci;
+ int64_t sum[CPU_COUNT_MAX], *ptr;
+ enum cpu_count i;
+ int s;
+
+ KASSERT(sizeof(ci->ci_counts) == sizeof(cpu_counts));
+
+ if (__predict_true(mp_online)) {
+ memset(sum, 0, sizeof(sum));
+ /*
+ * We want this to be reasonably quick, so any value we get
+ * isn't totally out of whack, so don't let the current LWP
+ * get preempted.
+ */
+ s = splvm();
+ curcpu()->ci_counts[CPU_COUNT_SYNC_ALL]++;
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ ptr = ci->ci_counts;
+ for (i = 0; i < CPU_COUNT_MAX; i += 8) {
+ sum[i+0] += ptr[i+0];
+ sum[i+1] += ptr[i+1];
+ sum[i+2] += ptr[i+2];
+ sum[i+3] += ptr[i+3];
+ sum[i+4] += ptr[i+4];
+ sum[i+5] += ptr[i+5];
+ sum[i+6] += ptr[i+6];
+ sum[i+7] += ptr[i+7];
+ }
+ KASSERT(i == CPU_COUNT_MAX);
+ }
+ memcpy(cpu_counts, sum, sizeof(cpu_counts));
+ splx(s);
+ } else {
+ memcpy(cpu_counts, curcpu()->ci_counts, sizeof(cpu_counts));
+ }
+}
+
+/*
+ * Fetch a fresh sum total for one single count. Expensive - don't call often.
+ */
+int64_t
+cpu_count_sync(enum cpu_count count)
+{
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *ci;
+ int64_t sum;
+ int s;
+
+ if (__predict_true(mp_online)) {
+ s = splvm();
+ curcpu()->ci_counts[CPU_COUNT_SYNC_ONE]++;
+ sum = 0;
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ sum += ci->ci_counts[count];
+ }
+ splx(s);
+ } else {
+ /* XXX Early boot, iterator might not be available. */
+ sum = curcpu()->ci_counts[count];
+ }
+ return cpu_counts[count] = sum;
+}
diff -r 76fab308302b -r 8d89a6b93a46 sys/kern/kern_fork.c
--- a/sys/kern/kern_fork.c Mon Dec 16 22:22:11 2019 +0000
+++ b/sys/kern/kern_fork.c Mon Dec 16 22:47:54 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_fork.c,v 1.216 2019/11/23 19:42:52 ad Exp $ */
+/* $NetBSD: kern_fork.c,v 1.217 2019/12/16 22:47:54 ad Exp $ */
/*-
* Copyright (c) 1999, 2001, 2004, 2006, 2007, 2008, 2019
@@ -68,7 +68,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_fork.c,v 1.216 2019/11/23 19:42:52 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_fork.c,v 1.217 2019/12/16 22:47:54 ad Exp $");
#include "opt_ktrace.h"
#include "opt_dtrace.h"
@@ -96,8 +96,6 @@
#include <sys/sdt.h>
#include <sys/ptrace.h>
-#include <uvm/uvm_extern.h>
-
/*
* DTrace SDT provider definitions
*/
@@ -525,11 +523,13 @@
/*
* Update stats now that we know the fork was successful.
*/
- uvmexp.forks++;
+ KPREEMPT_DISABLE(l1);
+ CPU_COUNT(CPU_COUNT_FORKS, 1);
if (flags & FORK_PPWAIT)
- uvmexp.forks_ppwait++;
+ CPU_COUNT(CPU_COUNT_FORKS_PPWAIT, 1);
if (flags & FORK_SHAREVM)
- uvmexp.forks_sharevm++;
+ CPU_COUNT(CPU_COUNT_FORKS_SHAREVM, 1);
+ KPREEMPT_ENABLE(l1);
if (ktrpoint(KTR_EMUL))
p2->p_traceflag |= KTRFAC_TRC_EMUL;
diff -r 76fab308302b -r 8d89a6b93a46 sys/kern/kern_softint.c
--- a/sys/kern/kern_softint.c Mon Dec 16 22:22:11 2019 +0000
+++ b/sys/kern/kern_softint.c Mon Dec 16 22:47:54 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_softint.c,v 1.55 2019/12/06 21:36:10 ad Exp $ */
+/* $NetBSD: kern_softint.c,v 1.56 2019/12/16 22:47:54 ad Exp $ */
/*-
* Copyright (c) 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -170,7 +170,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.55 2019/12/06 21:36:10 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.56 2019/12/16 22:47:54 ad Exp $");
#include <sys/param.h>
#include <sys/proc.h>
@@ -607,11 +607,7 @@
KERNEL_UNLOCK_ONE(l);
}
- /*
- * Unlocked, but only for statistics.
- * Should be per-CPU to prevent cache ping-pong.
- */
- curcpu()->ci_data.cpu_nsoft++;
+ CPU_COUNT(CPU_COUNT_NSOFT, 1);
KASSERT(si->si_cpu == curcpu());
KASSERT(si->si_lwp->l_wchan == NULL);
diff -r 76fab308302b -r 8d89a6b93a46 sys/kern/vfs_vnode.c
--- a/sys/kern/vfs_vnode.c Mon Dec 16 22:22:11 2019 +0000
+++ b/sys/kern/vfs_vnode.c Mon Dec 16 22:47:54 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_vnode.c,v 1.104 2019/12/01 13:56:29 ad Exp $ */
+/* $NetBSD: vfs_vnode.c,v 1.105 2019/12/16 22:47:54 ad Exp $ */
/*-
* Copyright (c) 1997-2011, 2019 The NetBSD Foundation, Inc.
@@ -146,7 +146,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.104 2019/12/01 13:56:29 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.105 2019/12/16 22:47:54 ad Exp $");
#include <sys/param.h>
#include <sys/kernel.h>
@@ -792,10 +792,8 @@
/* Take care of space accounting. */
if ((vp->v_iflag & VI_EXECMAP) != 0 &&
vp->v_uobj.uo_npages != 0) {
- atomic_add_int(&uvmexp.execpages,
- -vp->v_uobj.uo_npages);
- atomic_add_int(&uvmexp.filepages,
- vp->v_uobj.uo_npages);
+ cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
+ cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
}
vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP);
vp->v_vflag &= ~VV_MAPPED;
@@ -1565,8 +1563,8 @@
*/
VSTATE_CHANGE(vp, VS_LOADED, VS_RECLAIMING);
if ((vp->v_iflag & VI_EXECMAP) != 0 && vp->v_uobj.uo_npages != 0) {
- atomic_add_int(&uvmexp.execpages, -vp->v_uobj.uo_npages);
- atomic_add_int(&uvmexp.filepages, vp->v_uobj.uo_npages);
+ cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
+ cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
}
vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP);
mutex_exit(vp->v_interlock);
diff -r 76fab308302b -r 8d89a6b93a46 sys/kern/vfs_vnops.c
--- a/sys/kern/vfs_vnops.c Mon Dec 16 22:22:11 2019 +0000
+++ b/sys/kern/vfs_vnops.c Mon Dec 16 22:47:54 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_vnops.c,v 1.203 2019/12/01 13:56:29 ad Exp $ */
+/* $NetBSD: vfs_vnops.c,v 1.204 2019/12/16 22:47:54 ad Exp $ */
/*-
* Copyright (c) 2009 The NetBSD Foundation, Inc.
@@ -66,7 +66,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfs_vnops.c,v 1.203 2019/12/01 13:56:29 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_vnops.c,v 1.204 2019/12/16 22:47:54 ad Exp $");
#include "veriexec.h"
@@ -341,8 +341,8 @@
mutex_enter(vp->v_interlock);
if ((vp->v_iflag & VI_EXECMAP) == 0) {
- atomic_add_int(&uvmexp.filepages, -vp->v_uobj.uo_npages);
- atomic_add_int(&uvmexp.execpages, vp->v_uobj.uo_npages);
+ cpu_count(CPU_COUNT_FILEPAGES, -vp->v_uobj.uo_npages);
+ cpu_count(CPU_COUNT_EXECPAGES, vp->v_uobj.uo_npages);
vp->v_iflag |= VI_EXECMAP;
}
mutex_exit(vp->v_interlock);
@@ -368,8 +368,8 @@
return (ETXTBSY);
}
if ((vp->v_iflag & VI_EXECMAP) == 0) {
- atomic_add_int(&uvmexp.filepages, -vp->v_uobj.uo_npages);
- atomic_add_int(&uvmexp.execpages, vp->v_uobj.uo_npages);
+ cpu_count(CPU_COUNT_FILEPAGES, -vp->v_uobj.uo_npages);
+ cpu_count(CPU_COUNT_EXECPAGES, vp->v_uobj.uo_npages);
}
vp->v_iflag |= (VI_TEXT | VI_EXECMAP);
mutex_exit(vp->v_interlock);
diff -r 76fab308302b -r 8d89a6b93a46 sys/miscfs/procfs/procfs_linux.c
--- a/sys/miscfs/procfs/procfs_linux.c Mon Dec 16 22:22:11 2019 +0000
+++ b/sys/miscfs/procfs/procfs_linux.c Mon Dec 16 22:47:54 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: procfs_linux.c,v 1.76 2019/09/07 19:08:28 chs Exp $ */
+/* $NetBSD: procfs_linux.c,v 1.77 2019/12/16 22:47:55 ad Exp $ */
/*
* Copyright (c) 2001 Wasabi Systems, Inc.
@@ -36,7 +36,7 @@
Home |
Main Index |
Thread Index |
Old Index