Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/arch Add PCID support in SVS. This avoids TLB flushes du...
details: https://anonhg.NetBSD.org/src/rev/328dbf389f0c
branches: trunk
changeset: 451647:328dbf389f0c
user: maxv <maxv%NetBSD.org@localhost>
date: Wed May 29 16:54:41 2019 +0000
description:
Add PCID support in SVS. This avoids TLB flushes during kernel<->user
transitions, which greatly reduces the performance penalty introduced by
SVS.
We use two ASIDs, 0 (kern) and 1 (user), and use invpcid to flush pages
in both ASIDs.
The read-only machdep.svs.pcid={0,1} sysctl is added, and indicates whether
SVS+PCID is in use.
diffstat:
sys/arch/amd64/amd64/cpufunc.S | 31 +++++++++++++++++++++++++---
sys/arch/amd64/amd64/genassym.cf | 10 ++++++++-
sys/arch/amd64/include/pmap.h | 3 +-
sys/arch/x86/include/cpufunc.h | 26 +++++++++++++++++++++++-
sys/arch/x86/include/pmap.h | 5 +++-
sys/arch/x86/include/specialreg.h | 9 +++++++-
sys/arch/x86/x86/cpu.c | 10 +++++++-
sys/arch/x86/x86/svs.c | 42 ++++++++++++++++++++++++++++++++++----
sys/arch/x86/x86/x86_machdep.c | 12 ++++++++--
9 files changed, 129 insertions(+), 19 deletions(-)
diffs (truncated from 365 to 300 lines):
diff -r 46bc9f34ed3f -r 328dbf389f0c sys/arch/amd64/amd64/cpufunc.S
--- a/sys/arch/amd64/amd64/cpufunc.S Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/amd64/amd64/cpufunc.S Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.S,v 1.40 2019/05/19 08:17:02 maxv Exp $ */
+/* $NetBSD: cpufunc.S,v 1.41 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -41,6 +41,7 @@
#include <machine/segments.h>
#include "opt_xen.h"
+#include "opt_svs.h"
#include "assym.h"
@@ -65,6 +66,17 @@
#ifndef XENPV
ENTRY(invlpg)
+#ifdef SVS
+ movb _C_LABEL(svs_pcid),%al
+ testb %al,%al
+ jz 1f
+ pushq %rdi
+ pushq $PMAP_PCID_USER
+ movq $INVPCID_ADDRESS,%rax
+ invpcid (%rsp),%rax
+ addq $16,%rsp
+1: /* FALLTHROUGH */
+#endif
invlpg (%rdi)
ret
END(invlpg)
@@ -112,7 +124,7 @@
ENTRY(tlbflushg)
movq %cr4, %rax
testq $CR4_PGE, %rax
- jz 1f
+ jz tlbflush
movq %rax, %rdx
andq $~CR4_PGE, %rdx
movq %rdx, %cr4
@@ -121,8 +133,19 @@
END(tlbflushg)
ENTRY(tlbflush)
-1:
- movq %cr3, %rax
+#ifdef SVS
+ movb _C_LABEL(svs_pcid),%al
+ testb %al,%al
+ jz 1f
+ xorq %rax,%rax
+ pushq %rax
+ pushq %rax
+ movq $INVPCID_ALL_NONGLOBAL,%rax
+ invpcid (%rsp),%rax
+ addq $16,%rsp
+ ret
+#endif
+1: movq %cr3, %rax
movq %rax, %cr3
ret
END(tlbflush)
diff -r 46bc9f34ed3f -r 328dbf389f0c sys/arch/amd64/amd64/genassym.cf
--- a/sys/arch/amd64/amd64/genassym.cf Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/amd64/amd64/genassym.cf Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.75 2019/05/27 17:32:36 maxv Exp $
+# $NetBSD: genassym.cf,v 1.76 2019/05/29 16:54:41 maxv Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -276,6 +276,14 @@
define ACPI_SUSPEND_CR4 offsetof(struct cpu_info, ci_suspend_cr4)
define ACPI_SUSPEND_CR8 offsetof(struct cpu_info, ci_suspend_cr8)
+define INVPCID_ADDRESS INVPCID_ADDRESS
+define INVPCID_CONTEXT INVPCID_CONTEXT
+define INVPCID_ALL INVPCID_ALL
+define INVPCID_ALL_NONGLOBAL INVPCID_ALL_NONGLOBAL
+
+define PMAP_PCID_USER PMAP_PCID_USER
+define PMAP_PCID_KERN PMAP_PCID_KERN
+
if NIOAPIC > 0
define PIC_IOAPIC offsetof(struct pic, pic_ioapic)
define IOAPIC_SC_REG offsetof(struct ioapic_softc, sc_reg)
diff -r 46bc9f34ed3f -r 328dbf389f0c sys/arch/amd64/include/pmap.h
--- a/sys/arch/amd64/include/pmap.h Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/amd64/include/pmap.h Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.60 2019/03/09 09:09:56 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.61 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -179,6 +179,7 @@
void svs_pdir_switch(struct pmap *);
void svs_init(void);
extern bool svs_enabled;
+extern bool svs_pcid;
#include <x86/pmap.h>
diff -r 46bc9f34ed3f -r 328dbf389f0c sys/arch/x86/include/cpufunc.h
--- a/sys/arch/x86/include/cpufunc.h Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/x86/include/cpufunc.h Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.h,v 1.30 2019/05/11 13:40:26 christos Exp $ */
+/* $NetBSD: cpufunc.h,v 1.31 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2019 The NetBSD Foundation, Inc.
@@ -65,6 +65,30 @@
void wbinvd(void);
void breakpoint(void);
+#define INVPCID_ADDRESS 0
+#define INVPCID_CONTEXT 1
+#define INVPCID_ALL 2
+#define INVPCID_ALL_NONGLOBAL 3
+
+static inline void
+invpcid(register_t op, uint64_t pcid, vaddr_t va)
+{
+ struct {
+ uint64_t pcid;
+ uint64_t addr;
+ } desc = {
+ .pcid = pcid,
+ .addr = va
+ };
+
+ asm volatile (
+ "invpcid %[desc],%[op]"
+ :
+ : [desc] "m" (desc), [op] "r" (op)
+ : "memory"
+ );
+}
+
static inline uint64_t
rdtsc(void)
{
diff -r 46bc9f34ed3f -r 328dbf389f0c sys/arch/x86/include/pmap.h
--- a/sys/arch/x86/include/pmap.h Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/x86/include/pmap.h Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.100 2019/03/10 16:30:01 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.101 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -210,6 +210,9 @@
extern struct pcpu_area *pcpuarea;
+#define PMAP_PCID_KERN 0
+#define PMAP_PCID_USER 1
+
/*
* pmap data structures: see pmap.c for details of locking.
*/
diff -r 46bc9f34ed3f -r 328dbf389f0c sys/arch/x86/include/specialreg.h
--- a/sys/arch/x86/include/specialreg.h Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/x86/include/specialreg.h Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: specialreg.h,v 1.146 2019/05/18 08:17:39 maxv Exp $ */
+/* $NetBSD: specialreg.h,v 1.147 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 2014-2019 The NetBSD Foundation, Inc.
@@ -88,6 +88,13 @@
#define CCR1_RPL 0x01 /* enables RPLSET and RPLVAL# pins */
/*
+ * CR3
+ */
+#define CR3_PCID __BITS(11,0)
+#define CR3_PA __BITS(62,12)
+#define CR3_NO_TLB_FLUSH __BIT(63)
+
+/*
* CR4
*/
#define CR4_VME 0x00000001 /* virtual 8086 mode extension enable */
diff -r 46bc9f34ed3f -r 328dbf389f0c sys/arch/x86/x86/cpu.c
--- a/sys/arch/x86/x86/cpu.c Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/x86/x86/cpu.c Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $ */
+/* $NetBSD: cpu.c,v 1.171 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 2000-2012 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.171 2019/05/29 16:54:41 maxv Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -613,6 +613,12 @@
if (cpu_feature[5] & CPUID_SEF_SMAP)
cr4 |= CR4_SMAP;
+#ifdef SVS
+ /* If PCID is supported, enable it */
+ if (svs_pcid)
+ cr4 |= CR4_PCIDE;
+#endif
+
if (cr4) {
cr4 |= rcr4();
lcr4(cr4);
diff -r 46bc9f34ed3f -r 328dbf389f0c sys/arch/x86/x86/svs.c
--- a/sys/arch/x86/x86/svs.c Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/x86/x86/svs.c Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $ */
+/* $NetBSD: svs.c,v 1.29 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.29 2019/05/29 16:54:41 maxv Exp $");
#include "opt_svs.h"
@@ -228,6 +228,10 @@
*/
bool svs_enabled __read_mostly = false;
+bool svs_pcid __read_mostly = false;
+
+static uint64_t svs_pcid_kcr3 __read_mostly;
+static uint64_t svs_pcid_ucr3 __read_mostly;
struct svs_utls {
paddr_t kpdirpa;
@@ -405,6 +409,19 @@
}
static void
+svs_pcid_init(struct cpu_info *ci)
+{
+ if (!svs_pcid) {
+ return;
+ }
+
+ svs_pcid_ucr3 = __SHIFTIN(PMAP_PCID_USER, CR3_PCID) | CR3_NO_TLB_FLUSH;
+ svs_pcid_kcr3 = __SHIFTIN(PMAP_PCID_KERN, CR3_PCID) | CR3_NO_TLB_FLUSH;
+
+ ci->ci_svs_updirpa |= svs_pcid_ucr3;
+}
+
+static void
svs_range_add(struct cpu_info *ci, vaddr_t va, size_t size, bool global)
{
size_t i, n;
@@ -454,6 +471,8 @@
svs_rsp0_init(ci);
svs_utls_init(ci);
+
+ svs_pcid_init(ci);
}
void
@@ -523,11 +542,14 @@
utls->scratch = 0;
/*
- * Enter the user rsp0. We don't need to flush the TLB here, since
- * the user page tables are not loaded.
+ * Enter the user rsp0. If we're using PCID we must flush the user VA,
+ * if we aren't it will be flushed during the next CR3 reload.
*/
pte = ci->ci_svs_rsp0_pte;
*pte = L1_BASE[pl1_i(va)];
+ if (svs_pcid) {
+ invpcid(INVPCID_ADDRESS, PMAP_PCID_USER, ci->ci_svs_rsp0);
+ }
}
static inline pt_entry_t
@@ -558,7 +580,7 @@
/* Update the info in the UTLS page */
utls = (struct svs_utls *)ci->ci_svs_utls;
- utls->kpdirpa = pmap_pdirpa(pmap, 0);
Home |
Main Index |
Thread Index |
Old Index