Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys Fix false sharing problems with cpu_info. Identified wi...
details: https://anonhg.NetBSD.org/src/rev/12ab668a21ca
branches: trunk
changeset: 465662:12ab668a21ca
user: ad <ad%NetBSD.org@localhost>
date: Sun Dec 01 15:34:44 2019 +0000
description:
Fix false sharing problems with cpu_info. Identified with tprof(8).
This was a very nice win in my tests on a 48 CPU box.
- Reorganise cpu_data slightly according to usage.
- Put cpu_onproc into struct cpu_info alongside ci_curlwp (now is ci_onproc).
- On x86, put some items in their own cache lines according to usage, like
the IPI bitmask and ci_want_resched.
diffstat:
sys/arch/alpha/alpha/ipifuncs.c | 8 +++---
sys/arch/alpha/include/cpu.h | 3 +-
sys/arch/arm/include/cpu.h | 3 +-
sys/arch/hppa/include/cpu.h | 3 +-
sys/arch/ia64/include/cpu.h | 3 +-
sys/arch/m68k/include/cpu.h | 3 +-
sys/arch/mips/include/cpu.h | 3 +-
sys/arch/mips/mips/ipifuncs.c | 6 ++--
sys/arch/mips/rmi/rmixl_cpu.c | 6 ++--
sys/arch/or1k/include/cpu.h | 3 +-
sys/arch/powerpc/booke/e500_intr.c | 6 ++--
sys/arch/powerpc/include/cpu.h | 4 +-
sys/arch/powerpc/pic/ipi.c | 6 ++--
sys/arch/riscv/include/cpu.h | 3 +-
sys/arch/sh3/include/cpu.h | 3 +-
sys/arch/sparc/include/cpu.h | 3 +-
sys/arch/sparc64/include/cpu.h | 3 +-
sys/arch/sparc64/sparc64/clock.c | 6 ++--
sys/arch/usermode/include/cpu.h | 3 +-
sys/arch/vax/include/cpu.h | 3 +-
sys/arch/vax/vax/ka6400.c | 6 ++--
sys/arch/vax/vax/ka820.c | 6 ++--
sys/arch/vax/vax/ka88.c | 6 ++--
sys/arch/x86/include/cpu.h | 43 +++++++++++++++++++++++--------------
sys/arch/x86/x86/ipi.c | 6 ++--
sys/arch/xen/x86/xen_ipi.c | 8 +++---
sys/kern/kern_clock.c | 8 +++---
sys/kern/kern_cpu.c | 8 +++---
sys/kern/kern_idle.c | 6 ++--
sys/kern/kern_runq.c | 6 ++--
sys/kern/kern_softint.c | 6 ++--
sys/kern/kern_synch.c | 12 +++++-----
sys/kern/sched_4bsd.c | 6 ++--
sys/kern/sched_m2.c | 6 ++--
sys/sys/cpu_data.h | 25 ++++++++-------------
sys/sys/lwp.h | 4 +-
36 files changed, 131 insertions(+), 112 deletions(-)
diffs (truncated from 952 to 300 lines):
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/alpha/alpha/ipifuncs.c
--- a/sys/arch/alpha/alpha/ipifuncs.c Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/alpha/alpha/ipifuncs.c Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ipifuncs.c,v 1.49 2019/11/21 19:02:43 ad Exp $ */
+/* $NetBSD: ipifuncs.c,v 1.50 2019/12/01 15:34:44 ad Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.49 2019/11/21 19:02:43 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.50 2019/12/01 15:34:44 ad Exp $");
/*
* Interprocessor interrupt handlers.
@@ -263,8 +263,8 @@
alpha_ipi_ast(struct cpu_info *ci, struct trapframe *framep)
{
- if (ci->ci_data.cpu_onproc != ci->ci_data.cpu_idlelwp)
- aston(ci->ci_data.cpu_onproc);
+ if (ci->ci_onproc != ci->ci_data.cpu_idlelwp)
+ aston(ci->ci_onproc);
}
static void
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/alpha/include/cpu.h
--- a/sys/arch/alpha/include/cpu.h Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/alpha/include/cpu.h Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.85 2019/11/24 15:40:24 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.86 2019/12/01 15:34:44 ad Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
@@ -108,6 +108,7 @@
* Public members.
*/
struct lwp *ci_curlwp; /* current owner of the processor */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct cpu_data ci_data; /* MI per-cpu data */
#if !defined(_KMEMUSER)
struct cctr_state ci_cc; /* cycle counter state */
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/arm/include/cpu.h
--- a/sys/arch/arm/include/cpu.h Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/arm/include/cpu.h Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.102 2019/11/21 19:23:59 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.103 2019/12/01 15:34:44 ad Exp $ */
/*
* Copyright (c) 1994-1996 Mark Brinicombe.
@@ -152,6 +152,7 @@
ci_softints;
lwp_t * ci_curlwp; /* current lwp */
+ lwp_t * ci_onproc; /* current user LWP / kthread */
lwp_t * ci_lastlwp; /* last lwp */
struct evcnt ci_arm700bugcount;
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/hppa/include/cpu.h
--- a/sys/arch/hppa/include/cpu.h Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/hppa/include/cpu.h Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.5 2019/04/16 12:25:17 skrll Exp $ */
+/* $NetBSD: cpu.h,v 1.6 2019/12/01 15:34:44 ad Exp $ */
/* $OpenBSD: cpu.h,v 1.55 2008/07/23 17:39:35 kettenis Exp $ */
@@ -295,6 +295,7 @@
#define CPUF_PRIMARY 0x0001 /* ... is monarch/primary */
#define CPUF_RUNNING 0x0002 /* ... is running. */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
volatile u_long ci_ipi; /* IPIs pending */
struct cpu_softc *ci_softc;
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/ia64/include/cpu.h
--- a/sys/arch/ia64/include/cpu.h Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/ia64/include/cpu.h Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.19 2019/11/24 15:45:41 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.20 2019/12/01 15:34:44 ad Exp $ */
/*-
* Copyright (c) 2006 The NetBSD Foundation, Inc.
@@ -90,6 +90,7 @@
struct cpu_data ci_data; /* MI per-cpu data */
device_t ci_dev; /* pointer to our device */
struct lwp *ci_curlwp; /* current owner of the processor */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct cctr_state ci_cc; /* cycle counter state */
struct cpu_info *ci_next; /* next cpu_info structure */
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/m68k/include/cpu.h
--- a/sys/arch/m68k/include/cpu.h Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/m68k/include/cpu.h Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.16 2018/08/22 01:05:22 msaitoh Exp $ */
+/* $NetBSD: cpu.h,v 1.17 2019/12/01 15:34:44 ad Exp $ */
/*
* Copyright (c) 1988 University of Utah.
@@ -177,6 +177,7 @@
int ci_mtx_oldspl;
volatile int ci_want_resched;
volatile int ci_idepth;
+ struct lwp *ci_onproc; /* current user LWP / kthread */
};
#endif /* _KERNEL || _KMEMUSER */
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/mips/include/cpu.h
--- a/sys/arch/mips/include/cpu.h Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/mips/include/cpu.h Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.127 2019/11/21 19:24:00 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.128 2019/12/01 15:34:44 ad Exp $ */
/*-
* Copyright (c) 1992, 1993
@@ -95,6 +95,7 @@
u_long ci_divisor_delay; /* for delay/DELAY */
u_long ci_divisor_recip; /* unused, for obsolete microtime(9) */
struct lwp *ci_curlwp; /* currently running lwp */
+ struct lwp *ci_onproc; /* current user LWP / kthread */
volatile int ci_want_resched; /* user preemption pending */
int ci_mtx_count; /* negative count of held mutexes */
int ci_mtx_oldspl; /* saved SPL value */
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/mips/mips/ipifuncs.c
--- a/sys/arch/mips/mips/ipifuncs.c Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/mips/mips/ipifuncs.c Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ipifuncs.c,v 1.12 2019/11/23 19:40:35 ad Exp $ */
+/* $NetBSD: ipifuncs.c,v 1.13 2019/12/01 15:34:44 ad Exp $ */
/*-
* Copyright (c) 2010 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
#include "opt_ddb.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.12 2019/11/23 19:40:35 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.13 2019/12/01 15:34:44 ad Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@@ -76,7 +76,7 @@
static void
ipi_ast(struct cpu_info *ci)
{
- ci->ci_data.cpu_onproc->l_md.md_astpending = 1;
+ ci->ci_onproc->l_md.md_astpending = 1;
}
static void
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/mips/rmi/rmixl_cpu.c
--- a/sys/arch/mips/rmi/rmixl_cpu.c Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/mips/rmi/rmixl_cpu.c Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: rmixl_cpu.c,v 1.9 2015/06/28 22:14:38 matt Exp $ */
+/* $NetBSD: rmixl_cpu.c,v 1.10 2019/12/01 15:34:45 ad Exp $ */
/*
* Copyright 2002 Wasabi Systems, Inc.
@@ -38,7 +38,7 @@
#include "locators.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: rmixl_cpu.c,v 1.9 2015/06/28 22:14:38 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: rmixl_cpu.c,v 1.10 2019/12/01 15:34:45 ad Exp $");
#include "opt_multiprocessor.h"
#include "opt_ddb.h"
@@ -419,7 +419,6 @@
printf("&cpu_schedstate %p\n", &dp->cpu_schedstate); /* TBD */
printf("&cpu_xcall %p\n", &dp->cpu_xcall); /* TBD */
printf("cpu_xcall_pending %d\n", dp->cpu_xcall_pending);
- printf("cpu_onproc %p\n", dp->cpu_onproc);
printf("cpu_idlelwp %p\n", dp->cpu_idlelwp);
printf("cpu_lockstat %p\n", dp->cpu_lockstat);
printf("cpu_index %d\n", dp->cpu_index);
@@ -462,6 +461,7 @@
printf("ci_divisor_delay %ld\n", ci->ci_divisor_delay);
printf("ci_divisor_recip %ld\n", ci->ci_divisor_recip);
printf("ci_curlwp %p\n", ci->ci_curlwp);
+ printf("ci_onproc %p\n", dp->ci_onproc);
printf("ci_want_resched %d\n", ci->ci_want_resched);
printf("ci_mtx_count %d\n", ci->ci_mtx_count);
printf("ci_mtx_oldspl %d\n", ci->ci_mtx_oldspl);
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/or1k/include/cpu.h
--- a/sys/arch/or1k/include/cpu.h Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/or1k/include/cpu.h Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.3 2019/11/21 19:24:01 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.4 2019/12/01 15:34:45 ad Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -52,6 +52,7 @@
device_t ci_dev;
cpuid_t ci_cpuid;
struct lwp *ci_curlwp;
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct lwp *ci_softlwps[SOFTINT_COUNT];
uint64_t ci_lastintr;
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/powerpc/booke/e500_intr.c
--- a/sys/arch/powerpc/booke/e500_intr.c Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/powerpc/booke/e500_intr.c Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: e500_intr.c,v 1.39 2019/11/23 19:40:36 ad Exp $ */
+/* $NetBSD: e500_intr.c,v 1.40 2019/12/01 15:34:45 ad Exp $ */
/*-
* Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -41,7 +41,7 @@
#define __INTR_PRIVATE
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: e500_intr.c,v 1.39 2019/11/23 19:40:36 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: e500_intr.c,v 1.40 2019/12/01 15:34:45 ad Exp $");
#include <sys/param.h>
#include <sys/proc.h>
@@ -1338,7 +1338,7 @@
static void
e500_ipi_ast(void)
{
- curcpu()->ci_data.cpu_onproc->l_md.md_astpending = 1;
+ curcpu()->ci_onproc->l_md.md_astpending = 1;
}
static const ipifunc_t e500_ipifuncs[] = {
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/powerpc/include/cpu.h
--- a/sys/arch/powerpc/include/cpu.h Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/powerpc/include/cpu.h Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.109 2019/11/23 19:40:36 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.110 2019/12/01 15:34:45 ad Exp $ */
/*
* Copyright (C) 1999 Wolfgang Solfrank.
@@ -67,7 +67,7 @@
device_t ci_dev; /* device of corresponding cpu */
struct cpu_softc *ci_softc; /* private cpu info */
struct lwp *ci_curlwp; /* current owner of the processor */
-
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct pcb *ci_curpcb;
struct pmap *ci_curpm;
struct lwp *ci_softlwps[SOFTINT_COUNT];
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/powerpc/pic/ipi.c
--- a/sys/arch/powerpc/pic/ipi.c Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/powerpc/pic/ipi.c Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ipi.c,v 1.13 2019/11/24 15:49:12 ad Exp $ */
+/* $NetBSD: ipi.c,v 1.14 2019/12/01 15:34:45 ad Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.13 2019/11/24 15:49:12 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.14 2019/12/01 15:34:45 ad Exp $");
#include "opt_multiprocessor.h"
#include "opt_pic.h"
@@ -79,7 +79,7 @@
cpu_pause(NULL);
if (ipi & IPI_AST)
- ci->ci_data.cpu_onproc->l_md.md_astpending = 1;
+ ci->ci_onproc->l_md.md_astpending = 1;
if (ipi & IPI_HALT) {
struct cpuset_info * const csi = &cpuset_info;
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/riscv/include/cpu.h
--- a/sys/arch/riscv/include/cpu.h Sun Dec 01 15:28:19 2019 +0000
+++ b/sys/arch/riscv/include/cpu.h Sun Dec 01 15:34:44 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.6 2019/11/21 19:24:01 ad Exp $ */
+/* $NetBSD: cpu.h,v 1.7 2019/12/01 15:34:45 ad Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -54,6 +54,7 @@
device_t ci_dev;
cpuid_t ci_cpuid;
struct lwp *ci_curlwp;
+ struct lwp *ci_onproc; /* current user LWP / kthread */
struct lwp *ci_softlwps[SOFTINT_COUNT];
struct trapframe *ci_ddb_regs;
diff -r 6b1b08aaa98e -r 12ab668a21ca sys/arch/sh3/include/cpu.h
--- a/sys/arch/sh3/include/cpu.h Sun Dec 01 15:28:19 2019 +0000
Home |
Main Index |
Thread Index |
Old Index