Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys - Convert x86 MD code, mainly pmap(9) e.g. TLB shootdown...
details: https://anonhg.NetBSD.org/src/rev/f0474f9f65ec
branches: trunk
changeset: 778929:f0474f9f65ec
user: rmind <rmind%NetBSD.org@localhost>
date: Fri Apr 20 22:23:24 2012 +0000
description:
- Convert x86 MD code, mainly pmap(9) e.g. TLB shootdown code, to use
kcpuset(9) and thus replace hardcoded CPU bitmasks. This removes the
limitation of maximum CPUs.
- Support up to 256 CPUs on amd64 architecture by default.
Bug fixes, improvements, completion of Xen part and testing on 64-core
AMD Opteron(tm) Processor 6282 SE (also, as Xen HVM domU with 128 CPUs)
by Manuel Bouyer.
diffstat:
sys/arch/amd64/amd64/genassym.cf | 4 +-
sys/arch/amd64/amd64/mptramp.S | 4 +-
sys/arch/amd64/include/param.h | 7 +-
sys/arch/i386/i386/genassym.cf | 4 +-
sys/arch/i386/i386/mptramp.S | 6 +-
sys/arch/i386/include/param.h | 9 +-
sys/arch/x86/acpi/acpi_wakeup.c | 23 ++-
sys/arch/x86/include/cpu.h | 6 +-
sys/arch/x86/include/cpuvar.h | 6 +-
sys/arch/x86/include/pmap.h | 11 +-
sys/arch/x86/x86/cpu.c | 33 ++---
sys/arch/x86/x86/mtrr_i686.c | 57 ++++-----
sys/arch/x86/x86/pmap.c | 175 ++++++++++++++-----------------
sys/arch/x86/x86/pmap_tlb.c | 215 ++++++++++++++++++++------------------
sys/arch/xen/include/xenpmap.h | 11 +-
sys/arch/xen/x86/cpu.c | 30 +---
sys/arch/xen/x86/x86_xpmap.c | 38 +++++-
sys/arch/xen/x86/xen_pmap.c | 12 +-
sys/kern/subr_kcpuset.c | 31 +++--
sys/kern/sys_sched.c | 6 +-
sys/sys/kcpuset.h | 3 +-
21 files changed, 353 insertions(+), 338 deletions(-)
diffs (truncated from 1762 to 300 lines):
diff -r b70783d899fe -r f0474f9f65ec sys/arch/amd64/amd64/genassym.cf
--- a/sys/arch/amd64/amd64/genassym.cf Fri Apr 20 21:57:33 2012 +0000
+++ b/sys/arch/amd64/amd64/genassym.cf Fri Apr 20 22:23:24 2012 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.49 2011/12/07 15:47:41 cegger Exp $
+# $NetBSD: genassym.cf,v 1.50 2012/04/20 22:23:24 rmind Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -228,12 +228,10 @@
define CPU_INFO_WANT_PMAPLOAD offsetof(struct cpu_info, ci_want_pmapload)
define CPU_INFO_TLBSTATE offsetof(struct cpu_info, ci_tlbstate)
define TLBSTATE_VALID TLBSTATE_VALID
-define CPU_INFO_TLB_EVCNT offsetof(struct cpu_info, ci_tlb_evcnt)
define CPU_INFO_CURLWP offsetof(struct cpu_info, ci_curlwp)
define CPU_INFO_CURLDT offsetof(struct cpu_info, ci_curldt)
define CPU_INFO_IDLELWP offsetof(struct cpu_info, ci_data.cpu_idlelwp)
define CPU_INFO_PMAP offsetof(struct cpu_info, ci_pmap)
-define CPU_INFO_CPUMASK offsetof(struct cpu_info, ci_cpumask)
define CPU_INFO_RSP0 offsetof(struct cpu_info, ci_tss.tss_rsp0)
define CPU_INFO_NSYSCALL offsetof(struct cpu_info, ci_data.cpu_nsyscall)
define CPU_INFO_NTRAP offsetof(struct cpu_info, ci_data.cpu_ntrap)
diff -r b70783d899fe -r f0474f9f65ec sys/arch/amd64/amd64/mptramp.S
--- a/sys/arch/amd64/amd64/mptramp.S Fri Apr 20 21:57:33 2012 +0000
+++ b/sys/arch/amd64/amd64/mptramp.S Fri Apr 20 22:23:24 2012 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: mptramp.S,v 1.13 2012/04/19 18:00:34 jym Exp $ */
+/* $NetBSD: mptramp.S,v 1.14 2012/04/20 22:23:24 rmind Exp $ */
/*-
* Copyright (c) 2000 The NetBSD Foundation, Inc.
@@ -108,7 +108,6 @@
#define HALTT(x,y) /**/
#endif
- .globl _C_LABEL(idle_loop)
.global _C_LABEL(cpu_spinup_trampoline)
.global _C_LABEL(cpu_spinup_trampoline_end)
.global _C_LABEL(cpu_hatch)
@@ -252,7 +251,6 @@
movl PCB_CR0(%rsi),%eax
movq %rax,%cr0
call _C_LABEL(cpu_hatch)
- jmp _C_LABEL(idle_loop)
.data
_C_LABEL(mp_pdirpa):
diff -r b70783d899fe -r f0474f9f65ec sys/arch/amd64/include/param.h
--- a/sys/arch/amd64/include/param.h Fri Apr 20 21:57:33 2012 +0000
+++ b/sys/arch/amd64/include/param.h Fri Apr 20 22:23:24 2012 +0000
@@ -1,7 +1,12 @@
-/* $NetBSD: param.h,v 1.17 2012/02/04 17:56:16 para Exp $ */
+/* $NetBSD: param.h,v 1.18 2012/04/20 22:23:24 rmind Exp $ */
#ifdef __x86_64__
+#ifndef XEN
+/* Must be defined before cpu.h */
+#define MAXCPUS 256
+#endif
+
#ifdef _KERNEL
#include <machine/cpu.h>
#endif
diff -r b70783d899fe -r f0474f9f65ec sys/arch/i386/i386/genassym.cf
--- a/sys/arch/i386/i386/genassym.cf Fri Apr 20 21:57:33 2012 +0000
+++ b/sys/arch/i386/i386/genassym.cf Fri Apr 20 22:23:24 2012 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.91 2011/12/07 15:47:42 cegger Exp $
+# $NetBSD: genassym.cf,v 1.92 2012/04/20 22:23:24 rmind Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -287,13 +287,11 @@
define CPU_INFO_WANT_PMAPLOAD offsetof(struct cpu_info, ci_want_pmapload)
define CPU_INFO_TLBSTATE offsetof(struct cpu_info, ci_tlbstate)
define TLBSTATE_VALID TLBSTATE_VALID
-define CPU_INFO_TLB_EVCNT offsetof(struct cpu_info, ci_tlb_evcnt)
define CPU_INFO_CURLWP offsetof(struct cpu_info, ci_curlwp)
define CPU_INFO_FPCURLWP offsetof(struct cpu_info, ci_fpcurlwp)
define CPU_INFO_CURLDT offsetof(struct cpu_info, ci_curldt)
define CPU_INFO_IDLELWP offsetof(struct cpu_info, ci_data.cpu_idlelwp)
define CPU_INFO_PMAP offsetof(struct cpu_info, ci_pmap)
-define CPU_INFO_CPUMASK offsetof(struct cpu_info, ci_cpumask)
define CPU_INFO_TSS offsetof(struct cpu_info, ci_tss)
define CPU_INFO_TSS_SEL offsetof(struct cpu_info, ci_tss_sel)
define CPU_INFO_ESP0 offsetof(struct cpu_info, ci_tss.tss_esp0)
diff -r b70783d899fe -r f0474f9f65ec sys/arch/i386/i386/mptramp.S
--- a/sys/arch/i386/i386/mptramp.S Fri Apr 20 21:57:33 2012 +0000
+++ b/sys/arch/i386/i386/mptramp.S Fri Apr 20 22:23:24 2012 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: mptramp.S,v 1.23 2012/04/19 18:00:35 jym Exp $ */
+/* $NetBSD: mptramp.S,v 1.24 2012/04/20 22:23:24 rmind Exp $ */
/*-
* Copyright (c) 2000 The NetBSD Foundation, Inc.
@@ -76,7 +76,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: mptramp.S,v 1.23 2012/04/19 18:00:35 jym Exp $");
+__KERNEL_RCSID(0, "$NetBSD: mptramp.S,v 1.24 2012/04/20 22:23:24 rmind Exp $");
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -271,8 +271,6 @@
HALTT(0x30,%ecx)
pushl %ecx
call _C_LABEL(cpu_hatch)
- HALT(0x33)
- jmp _C_LABEL(idle_loop)
.data
_C_LABEL(mp_pdirpa):
diff -r b70783d899fe -r f0474f9f65ec sys/arch/i386/include/param.h
--- a/sys/arch/i386/include/param.h Fri Apr 20 21:57:33 2012 +0000
+++ b/sys/arch/i386/include/param.h Fri Apr 20 22:23:24 2012 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: param.h,v 1.76 2012/02/10 17:35:49 para Exp $ */
+/* $NetBSD: param.h,v 1.77 2012/04/20 22:23:24 rmind Exp $ */
/*-
* Copyright (c) 1990 The Regents of the University of California.
@@ -41,6 +41,13 @@
* Machine dependent constants for Intel 386.
*/
+/*
+ * MAXCPUS must be defined before cpu.h inclusion. Note: i386 might
+ * support more CPUs, but due to the limited KVA space available on
+ * i386, such support would be inefficient. Use amd64 instead.
+ */
+#define MAXCPUS 32
+
#ifdef _KERNEL
#include <machine/cpu.h>
#endif
diff -r b70783d899fe -r f0474f9f65ec sys/arch/x86/acpi/acpi_wakeup.c
--- a/sys/arch/x86/acpi/acpi_wakeup.c Fri Apr 20 21:57:33 2012 +0000
+++ b/sys/arch/x86/acpi/acpi_wakeup.c Fri Apr 20 22:23:24 2012 +0000
@@ -1,7 +1,7 @@
-/* $NetBSD: acpi_wakeup.c,v 1.30 2012/04/10 13:48:24 jruoho Exp $ */
+/* $NetBSD: acpi_wakeup.c,v 1.31 2012/04/20 22:23:24 rmind Exp $ */
/*-
- * Copyright (c) 2002 The NetBSD Foundation, Inc.
+ * Copyright (c) 2002, 2011 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: acpi_wakeup.c,v 1.30 2012/04/10 13:48:24 jruoho Exp $");
+__KERNEL_RCSID(0, "$NetBSD: acpi_wakeup.c,v 1.31 2012/04/20 22:23:24 rmind Exp $");
/*-
* Copyright (c) 2001 Takanori Watanabe <takawata%jp.freebsd.org@localhost>
@@ -61,11 +61,15 @@
* FreeBSD: src/sys/i386/acpica/acpi_wakeup.c,v 1.9 2002/01/10 03:26:46 wes Exp
*/
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: acpi_wakeup.c,v 1.31 2012/04/20 22:23:24 rmind Exp $");
+
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/bus.h>
-#include <sys/proc.h>
+#include <sys/cpu.h>
+#include <sys/kcpuset.h>
#include <sys/sysctl.h>
#include <uvm/uvm_extern.h>
@@ -209,7 +213,7 @@
#ifdef MULTIPROCESSOR
if (!CPU_IS_PRIMARY(ci)) {
atomic_and_32(&ci->ci_flags, ~CPUF_RUNNING);
- atomic_and_32(&cpus_running, ~ci->ci_cpumask);
+ kcpuset_atomic_clear(kcpuset_running, cpu_index(ci));
ACPI_FLUSH_CPU_CACHE();
@@ -277,7 +281,7 @@
#endif
atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
- atomic_or_32(&cpus_running, ci->ci_cpumask);
+ kcpuset_atomic_set(kcpuset_running, cpu_index(ci));
tsc_sync_ap(ci);
x86_enable_intr();
@@ -291,6 +295,7 @@
#ifdef MULTIPROCESSOR
struct cpu_info *ci;
CPU_INFO_ITERATOR cii;
+ cpuid_t cid;
#endif
KASSERT(acpi_wakeup_paddr != 0);
@@ -312,10 +317,12 @@
x86_disable_intr();
#ifdef MULTIPROCESSOR
- /* Save and suspend Application Processors */
+ /* Save and suspend Application Processors. */
x86_broadcast_ipi(X86_IPI_ACPI_CPU_SLEEP);
- while (cpus_running != curcpu()->ci_cpumask)
+ cid = cpu_index(curcpu());
+ while (!kcpuset_isotherset(kcpuset_running, cid)) {
delay(1);
+ }
#endif
if (acpi_md_sleep_prepare(state))
diff -r b70783d899fe -r f0474f9f65ec sys/arch/x86/include/cpu.h
--- a/sys/arch/x86/include/cpu.h Fri Apr 20 21:57:33 2012 +0000
+++ b/sys/arch/x86/include/cpu.h Fri Apr 20 22:23:24 2012 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.49 2012/03/02 16:43:31 bouyer Exp $ */
+/* $NetBSD: cpu.h,v 1.50 2012/04/20 22:23:24 rmind Exp $ */
/*-
* Copyright (c) 1990 The Regents of the University of California.
@@ -105,7 +105,7 @@
int ci_fpsaving; /* save in progress */
int ci_fpused; /* XEN: FPU was used by curlwp */
cpuid_t ci_cpuid; /* our CPU ID */
- int ci_cpumask; /* (1 << CPU ID) */
+ int _unused;
uint32_t ci_acpiid; /* our ACPI/MADT ID */
uint32_t ci_initapicid; /* our intitial APIC ID */
@@ -323,8 +323,6 @@
void cpu_broadcast_halt(void);
void cpu_kick(struct cpu_info *);
-extern uint32_t cpus_attached;
-
#define curcpu() x86_curcpu()
#define curlwp x86_curlwp()
#define curpcb ((struct pcb *)lwp_getpcb(curlwp))
diff -r b70783d899fe -r f0474f9f65ec sys/arch/x86/include/cpuvar.h
--- a/sys/arch/x86/include/cpuvar.h Fri Apr 20 21:57:33 2012 +0000
+++ b/sys/arch/x86/include/cpuvar.h Fri Apr 20 22:23:24 2012 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpuvar.h,v 1.45 2011/08/13 12:37:30 cherry Exp $ */
+/* $NetBSD: cpuvar.h,v 1.46 2012/04/20 22:23:24 rmind Exp $ */
/*-
* Copyright (c) 2000, 2007 The NetBSD Foundation, Inc.
@@ -95,13 +95,11 @@
};
#ifdef _KERNEL
-
+#include <sys/kcpuset.h>
#if defined(_KERNEL_OPT)
#include "opt_multiprocessor.h"
#endif /* defined(_KERNEL_OPT) */
-extern uint32_t cpus_running;
-
int x86_ipi(int, int, int);
void x86_self_ipi(int);
int x86_ipi_init(int);
diff -r b70783d899fe -r f0474f9f65ec sys/arch/x86/include/pmap.h
--- a/sys/arch/x86/include/pmap.h Fri Apr 20 21:57:33 2012 +0000
+++ b/sys/arch/x86/include/pmap.h Fri Apr 20 22:23:24 2012 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.51 2012/03/11 16:28:02 jym Exp $ */
+/* $NetBSD: pmap.h,v 1.52 2012/04/20 22:23:24 rmind Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -108,6 +108,8 @@
#if defined(_KERNEL)
+#include <sys/kcpuset.h>
+
/*
* pmap data structures: see pmap.c for details of locking.
*/
@@ -162,10 +164,10 @@
union descriptor *pm_ldt; /* user-set LDT */
size_t pm_ldt_len; /* size of LDT in bytes */
int pm_ldt_sel; /* LDT selector */
- uint32_t pm_cpus; /* mask of CPUs using pmap */
- uint32_t pm_kernel_cpus; /* mask of CPUs using kernel part
+ kcpuset_t *pm_cpus; /* mask of CPUs using pmap */
+ kcpuset_t *pm_kernel_cpus; /* mask of CPUs using kernel part
of pmap */
- uint32_t pm_xen_ptp_cpus; /* mask of CPUs which have this pmap's
+ kcpuset_t *pm_xen_ptp_cpus; /* mask of CPUs which have this pmap's
ptp mapped */
uint64_t pm_ncsw; /* for assertions */
struct vm_page *pm_gc_ptp; /* pages from pmap g/c */
Home |
Main Index |
Thread Index |
Old Index