Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys - Add some more failsafes to the CPU topology stuff, and...
details: https://anonhg.NetBSD.org/src/rev/2bac1cd9832d
branches: trunk
changeset: 846904:2bac1cd9832d
user: ad <ad%NetBSD.org@localhost>
date: Tue Dec 03 22:28:41 2019 +0000
description:
- Add some more failsafes to the CPU topology stuff, and build a 3rd
circular list of peer CPUs in other packages, so we might scroll through
them in the scheduler when looking to distribute or steal jobs.
- Fold the run queue data structure into spc_schedstate. Makes kern_runq.c
a far more pleasant place to work.
- Remove the code in sched_nextlwp() that tries to steal jobs from other
CPUs. It's not needed, because we do the very same thing in the idle LWP
anyway. Outside the VM system this was one of the the main causes of L3
cache misses I saw during builds. On my machine, this change yields a
60%-70% drop in time on the "hackbench" benchmark (there's clearly a bit
more going on here, but basically being less aggressive helps).
diffstat:
sys/kern/kern_cpu.c | 176 +++++++++++++++++++++++++++++----------
sys/kern/kern_runq.c | 225 ++++++++++++++++----------------------------------
sys/sys/cpu_data.h | 21 ++--
sys/sys/sched.h | 18 +++-
4 files changed, 229 insertions(+), 211 deletions(-)
diffs (truncated from 883 to 300 lines):
diff -r dcde80983d59 -r 2bac1cd9832d sys/kern/kern_cpu.c
--- a/sys/kern/kern_cpu.c Tue Dec 03 22:22:36 2019 +0000
+++ b/sys/kern/kern_cpu.c Tue Dec 03 22:28:41 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_cpu.c,v 1.79 2019/12/02 23:22:43 ad Exp $ */
+/* $NetBSD: kern_cpu.c,v 1.80 2019/12/03 22:28:41 ad Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019 The NetBSD Foundation, Inc.
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.79 2019/12/02 23:22:43 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.80 2019/12/03 22:28:41 ad Exp $");
#include "opt_cpu_ucode.h"
@@ -595,35 +595,118 @@
void
cpu_topology_set(struct cpu_info *ci, int package_id, int core_id, int smt_id)
{
+ enum cpu_rel rel;
cpu_topology_present = true;
ci->ci_package_id = package_id;
ci->ci_core_id = core_id;
ci->ci_smt_id = smt_id;
- ci->ci_package_cpus = ci;
- ci->ci_npackage_cpus = 1;
- ci->ci_core_cpus = ci;
- ci->ci_ncore_cpus = 1;
+ for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) {
+ ci->ci_sibling[rel] = ci;
+ ci->ci_nsibling[rel] = 1;
+ }
+}
+
+/*
+ * Link a CPU into the given circular list.
+ */
+static void
+cpu_topology_link(struct cpu_info *ci, struct cpu_info *ci2, enum cpu_rel rel)
+{
+ struct cpu_info *ci3;
+
+ /* Walk to the end of the existing circular list and append. */
+ for (ci3 = ci2;; ci3 = ci3->ci_sibling[rel]) {
+ ci3->ci_nsibling[rel]++;
+ if (ci3->ci_sibling[rel] == ci2) {
+ break;
+ }
+ }
+ ci->ci_sibling[rel] = ci2;
+ ci3->ci_sibling[rel] = ci;
+ ci->ci_nsibling[rel] = ci3->ci_nsibling[rel];
+}
+
+/*
+ * Find peer CPUs in other packages.
+ */
+static void
+cpu_topology_peers(void)
+{
+ CPU_INFO_ITERATOR cii, cii2;
+ struct cpu_info *ci, *ci2;
+
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ if (ci->ci_nsibling[CPUREL_PEER] > 1) {
+ /* Already linked. */
+ continue;
+ }
+ for (CPU_INFO_FOREACH(cii2, ci2)) {
+ if (ci != ci2 &&
+ ci->ci_package_id != ci2->ci_package_id &&
+ ci->ci_core_id == ci2->ci_core_id &&
+ ci->ci_smt_id == ci2->ci_smt_id) {
+ cpu_topology_link(ci, ci2, CPUREL_PEER);
+ break;
+ }
+ }
+ }
+}
+
+/*
+ * Print out the toplogy lists.
+ */
+static void
+cpu_topology_print(void)
+{
+#ifdef DEBUG
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *ci, *ci2;
+ const char *names[] = { "core", "package", "peer" };
+ enum cpu_rel rel;
+ int i;
+
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) {
+ printf("%s has %dx %s siblings: ", cpu_name(ci),
+ ci->ci_nsibling[rel], names[rel]);
+ ci2 = ci->ci_sibling[rel];
+ i = 0;
+ do {
+ printf(" %s", cpu_name(ci2));
+ ci2 = ci2->ci_sibling[rel];
+ } while (++i < 64 && ci2 != ci->ci_sibling[rel]);
+ if (i == 64) {
+ printf(" GAVE UP");
+ }
+ printf("\n");
+ }
+ }
+#endif /* DEBUG */
}
/*
* Fake up toplogy info if we have none, or if what we got was bogus.
+ * Don't override ci_package_id, etc, if cpu_topology_present is set.
+ * MD code also uses these.
*/
static void
cpu_topology_fake(void)
{
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
+ enum cpu_rel rel;
for (CPU_INFO_FOREACH(cii, ci)) {
- ci->ci_package_id = cpu_index(ci);
- ci->ci_core_id = 0;
- ci->ci_smt_id = 0;
- ci->ci_ncore_cpus = 1;
- ci->ci_core_cpus = ci;
- ci->ci_package_cpus = ci;
- ci->ci_npackage_cpus = 1;
+ for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) {
+ ci->ci_sibling[rel] = ci;
+ ci->ci_nsibling[rel] = 1;
+ }
+ if (!cpu_topology_present) {
+ ci->ci_package_id = cpu_index(ci);
+ }
}
+ cpu_topology_print();
}
/*
@@ -634,20 +717,16 @@
cpu_topology_init(void)
{
CPU_INFO_ITERATOR cii, cii2;
- struct cpu_info *ci, *ci2, *ci3;
+ struct cpu_info *ci, *ci2;
+ int ncore, npackage, npeer;
+ bool symmetric;
if (!cpu_topology_present) {
cpu_topology_fake();
return;
}
- for (CPU_INFO_FOREACH(cii, ci)) {
- ci->ci_ncore_cpus = 1;
- ci->ci_core_cpus = ci;
- ci->ci_package_cpus = ci;
- ci->ci_npackage_cpus = 1;
- }
-
+ /* Find siblings in same core and package. */
for (CPU_INFO_FOREACH(cii, ci)) {
for (CPU_INFO_FOREACH(cii2, ci2)) {
/* Avoid bad things happening. */
@@ -664,39 +743,42 @@
ci2->ci_package_id != ci->ci_package_id) {
continue;
}
- /*
- * Find CPUs in the same core. Walk to the end of
- * the existing circular list and append.
- */
- if (ci->ci_ncore_cpus == 1 &&
+ /* Find CPUs in the same core. */
+ if (ci->ci_nsibling[CPUREL_CORE] == 1 &&
ci->ci_core_id == ci2->ci_core_id) {
- for (ci3 = ci2;; ci3 = ci3->ci_core_cpus) {
- ci3->ci_ncore_cpus++;
- if (ci3->ci_core_cpus == ci2) {
- break;
- }
- }
- ci->ci_core_cpus = ci2;
- ci3->ci_core_cpus = ci;
- ci->ci_ncore_cpus = ci3->ci_ncore_cpus;
+ cpu_topology_link(ci, ci2, CPUREL_CORE);
}
- /* Same, but for package. */
- if (ci->ci_npackage_cpus == 1) {
- for (ci3 = ci2;; ci3 = ci3->ci_package_cpus) {
- ci3->ci_npackage_cpus++;
- if (ci3->ci_package_cpus == ci2) {
- break;
- }
- }
- ci->ci_package_cpus = ci2;
- ci3->ci_package_cpus = ci;
- ci->ci_npackage_cpus = ci3->ci_npackage_cpus;
+ /* Find CPUs in the same package. */
+ if (ci->ci_nsibling[CPUREL_PACKAGE] == 1) {
+ cpu_topology_link(ci, ci2, CPUREL_PACKAGE);
}
- if (ci->ci_ncore_cpus > 1 && ci->ci_npackage_cpus > 1) {
+ if (ci->ci_nsibling[CPUREL_CORE] > 1 &&
+ ci->ci_nsibling[CPUREL_PACKAGE] > 1) {
break;
}
}
}
+
+ /* Find peers in other packages. */
+ cpu_topology_peers();
+
+ /* Determine whether the topology is bogus/symmetric. */
+ npackage = curcpu()->ci_nsibling[CPUREL_PACKAGE];
+ ncore = curcpu()->ci_nsibling[CPUREL_CORE];
+ npeer = curcpu()->ci_nsibling[CPUREL_PEER];
+ symmetric = true;
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ if (npackage != ci->ci_nsibling[CPUREL_PACKAGE] ||
+ ncore != ci->ci_nsibling[CPUREL_CORE] ||
+ npeer != ci->ci_nsibling[CPUREL_PEER]) {
+ symmetric = false;
+ }
+ }
+ cpu_topology_print();
+ if (symmetric == false) {
+ printf("cpu_topology_init: not symmetric, faking it\n");
+ cpu_topology_fake();
+ }
}
#ifdef CPU_UCODE
diff -r dcde80983d59 -r 2bac1cd9832d sys/kern/kern_runq.c
--- a/sys/kern/kern_runq.c Tue Dec 03 22:22:36 2019 +0000
+++ b/sys/kern/kern_runq.c Tue Dec 03 22:28:41 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_runq.c,v 1.52 2019/12/01 15:34:46 ad Exp $ */
+/* $NetBSD: kern_runq.c,v 1.53 2019/12/03 22:28:41 ad Exp $ */
/*-
* Copyright (c) 2019 The NetBSD Foundation, Inc.
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_runq.c,v 1.52 2019/12/01 15:34:46 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_runq.c,v 1.53 2019/12/03 22:28:41 ad Exp $");
#include "opt_dtrace.h"
@@ -79,15 +79,6 @@
#include <sys/atomic.h>
/*
- * Priority related definitions.
- */
-#define PRI_TS_COUNT (NPRI_USER)
-#define PRI_RT_COUNT (PRI_COUNT - PRI_TS_COUNT)
-#define PRI_HTS_RANGE (PRI_TS_COUNT / 10)
-
-#define PRI_HIGHEST_TS (MAXPRI_USER)
-
-/*
* Bits per map.
*/
#define BITMAP_BITS (32)
@@ -95,34 +86,9 @@
#define BITMAP_MSB (0x80000000U)
#define BITMAP_MASK (BITMAP_BITS - 1)
-/*
- * Structures, runqueue.
- */
-
const int schedppq = 1;
-typedef struct {
- TAILQ_HEAD(, lwp) q_head;
-} queue_t;
-
-typedef struct {
- /* Bitmap */
- uint32_t r_bitmap[PRI_COUNT >> BITMAP_SHIFT];
- /* Counters */
- u_int r_count; /* Count of the threads */
- u_int r_avgcount; /* Average count of threads (* 256) */
- u_int r_mcount; /* Count of migratable threads */
- /* Runqueues */
- queue_t r_rt_queue[PRI_RT_COUNT];
- queue_t r_ts_queue[PRI_TS_COUNT];
Home |
Main Index |
Thread Index |
Old Index