Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/arch Add PCID support in SVS. This avoids TLB flushes du...



details:   https://anonhg.NetBSD.org/src/rev/23bf7d396085
branches:  trunk
changeset: 999352:23bf7d396085
user:      maxv <maxv%NetBSD.org@localhost>
date:      Wed May 29 16:54:41 2019 +0000

description:
Add PCID support in SVS. This avoids TLB flushes during kernel<->user
transitions, which greatly reduces the performance penalty introduced by
SVS.

We use two ASIDs, 0 (kern) and 1 (user), and use invpcid to flush pages
in both ASIDs.

The read-only machdep.svs.pcid={0,1} sysctl is added, and indicates whether
SVS+PCID is in use.

diffstat:

 sys/arch/amd64/amd64/cpufunc.S    |  31 +++++++++++++++++++++++++---
 sys/arch/amd64/amd64/genassym.cf  |  10 ++++++++-
 sys/arch/amd64/include/pmap.h     |   3 +-
 sys/arch/x86/include/cpufunc.h    |  26 +++++++++++++++++++++++-
 sys/arch/x86/include/pmap.h       |   5 +++-
 sys/arch/x86/include/specialreg.h |   9 +++++++-
 sys/arch/x86/x86/cpu.c            |  10 +++++++-
 sys/arch/x86/x86/svs.c            |  42 ++++++++++++++++++++++++++++++++++----
 sys/arch/x86/x86/x86_machdep.c    |  12 ++++++++--
 9 files changed, 129 insertions(+), 19 deletions(-)

diffs (truncated from 365 to 300 lines):

diff -r 61c41fc5ee2b -r 23bf7d396085 sys/arch/amd64/amd64/cpufunc.S
--- a/sys/arch/amd64/amd64/cpufunc.S    Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/amd64/amd64/cpufunc.S    Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpufunc.S,v 1.40 2019/05/19 08:17:02 maxv Exp $        */
+/*     $NetBSD: cpufunc.S,v 1.41 2019/05/29 16:54:41 maxv Exp $        */
 
 /*
  * Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -41,6 +41,7 @@
 #include <machine/segments.h>
 
 #include "opt_xen.h"
+#include "opt_svs.h"
 
 #include "assym.h"
 
@@ -65,6 +66,17 @@
 
 #ifndef XENPV
 ENTRY(invlpg)
+#ifdef SVS
+       movb    _C_LABEL(svs_pcid),%al
+       testb   %al,%al
+       jz      1f
+       pushq   %rdi
+       pushq   $PMAP_PCID_USER
+       movq    $INVPCID_ADDRESS,%rax
+       invpcid (%rsp),%rax
+       addq    $16,%rsp
+1:     /* FALLTHROUGH */
+#endif
        invlpg  (%rdi)
        ret
 END(invlpg)
@@ -112,7 +124,7 @@
 ENTRY(tlbflushg)
        movq    %cr4, %rax
        testq   $CR4_PGE, %rax
-       jz      1f
+       jz      tlbflush
        movq    %rax, %rdx
        andq    $~CR4_PGE, %rdx
        movq    %rdx, %cr4
@@ -121,8 +133,19 @@
 END(tlbflushg)
 
 ENTRY(tlbflush)
-1:
-       movq    %cr3, %rax
+#ifdef SVS
+       movb    _C_LABEL(svs_pcid),%al
+       testb   %al,%al
+       jz      1f
+       xorq    %rax,%rax
+       pushq   %rax
+       pushq   %rax
+       movq    $INVPCID_ALL_NONGLOBAL,%rax
+       invpcid (%rsp),%rax
+       addq    $16,%rsp
+       ret
+#endif
+1:     movq    %cr3, %rax
        movq    %rax, %cr3
        ret
 END(tlbflush)
diff -r 61c41fc5ee2b -r 23bf7d396085 sys/arch/amd64/amd64/genassym.cf
--- a/sys/arch/amd64/amd64/genassym.cf  Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/amd64/amd64/genassym.cf  Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: genassym.cf,v 1.75 2019/05/27 17:32:36 maxv Exp $
+#      $NetBSD: genassym.cf,v 1.76 2019/05/29 16:54:41 maxv Exp $
 
 #
 # Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -276,6 +276,14 @@
 define ACPI_SUSPEND_CR4        offsetof(struct cpu_info, ci_suspend_cr4)
 define ACPI_SUSPEND_CR8        offsetof(struct cpu_info, ci_suspend_cr8)
 
+define INVPCID_ADDRESS         INVPCID_ADDRESS
+define INVPCID_CONTEXT         INVPCID_CONTEXT
+define INVPCID_ALL             INVPCID_ALL
+define INVPCID_ALL_NONGLOBAL   INVPCID_ALL_NONGLOBAL
+
+define PMAP_PCID_USER          PMAP_PCID_USER
+define PMAP_PCID_KERN          PMAP_PCID_KERN
+
 if NIOAPIC > 0
 define         PIC_IOAPIC              offsetof(struct pic, pic_ioapic)
 define         IOAPIC_SC_REG           offsetof(struct ioapic_softc, sc_reg)
diff -r 61c41fc5ee2b -r 23bf7d396085 sys/arch/amd64/include/pmap.h
--- a/sys/arch/amd64/include/pmap.h     Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/amd64/include/pmap.h     Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap.h,v 1.60 2019/03/09 09:09:56 maxv Exp $   */
+/*     $NetBSD: pmap.h,v 1.61 2019/05/29 16:54:41 maxv Exp $   */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -179,6 +179,7 @@
 void svs_pdir_switch(struct pmap *);
 void svs_init(void);
 extern bool svs_enabled;
+extern bool svs_pcid;
 
 #include <x86/pmap.h>
 
diff -r 61c41fc5ee2b -r 23bf7d396085 sys/arch/x86/include/cpufunc.h
--- a/sys/arch/x86/include/cpufunc.h    Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/x86/include/cpufunc.h    Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpufunc.h,v 1.30 2019/05/11 13:40:26 christos Exp $    */
+/*     $NetBSD: cpufunc.h,v 1.31 2019/05/29 16:54:41 maxv Exp $        */
 
 /*
  * Copyright (c) 1998, 2007, 2019 The NetBSD Foundation, Inc.
@@ -65,6 +65,30 @@
 void   wbinvd(void);
 void   breakpoint(void);
 
+#define INVPCID_ADDRESS                0
+#define INVPCID_CONTEXT                1
+#define INVPCID_ALL            2
+#define INVPCID_ALL_NONGLOBAL  3
+
+static inline void
+invpcid(register_t op, uint64_t pcid, vaddr_t va)
+{
+       struct {
+               uint64_t pcid;
+               uint64_t addr;
+       } desc = {
+               .pcid = pcid,
+               .addr = va
+       };
+
+       asm volatile (
+               "invpcid %[desc],%[op]"
+               :
+               : [desc] "m" (desc), [op] "r" (op)
+               : "memory"
+       );
+}
+
 static inline uint64_t
 rdtsc(void)
 {
diff -r 61c41fc5ee2b -r 23bf7d396085 sys/arch/x86/include/pmap.h
--- a/sys/arch/x86/include/pmap.h       Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/x86/include/pmap.h       Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap.h,v 1.100 2019/03/10 16:30:01 maxv Exp $  */
+/*     $NetBSD: pmap.h,v 1.101 2019/05/29 16:54:41 maxv Exp $  */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -210,6 +210,9 @@
 
 extern struct pcpu_area *pcpuarea;
 
+#define PMAP_PCID_KERN 0
+#define PMAP_PCID_USER 1
+
 /*
  * pmap data structures: see pmap.c for details of locking.
  */
diff -r 61c41fc5ee2b -r 23bf7d396085 sys/arch/x86/include/specialreg.h
--- a/sys/arch/x86/include/specialreg.h Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/x86/include/specialreg.h Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: specialreg.h,v 1.146 2019/05/18 08:17:39 maxv Exp $    */
+/*     $NetBSD: specialreg.h,v 1.147 2019/05/29 16:54:41 maxv Exp $    */
 
 /*
  * Copyright (c) 2014-2019 The NetBSD Foundation, Inc.
@@ -88,6 +88,13 @@
 #define CCR1_RPL       0x01    /* enables RPLSET and RPLVAL# pins */
 
 /*
+ * CR3
+ */
+#define CR3_PCID               __BITS(11,0)
+#define CR3_PA                 __BITS(62,12)
+#define CR3_NO_TLB_FLUSH       __BIT(63)
+
+/*
  * CR4
  */
 #define CR4_VME                0x00000001 /* virtual 8086 mode extension enable */
diff -r 61c41fc5ee2b -r 23bf7d396085 sys/arch/x86/x86/cpu.c
--- a/sys/arch/x86/x86/cpu.c    Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/x86/x86/cpu.c    Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $   */
+/*     $NetBSD: cpu.c,v 1.171 2019/05/29 16:54:41 maxv Exp $   */
 
 /*
  * Copyright (c) 2000-2012 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.171 2019/05/29 16:54:41 maxv Exp $");
 
 #include "opt_ddb.h"
 #include "opt_mpbios.h"                /* for MPDEBUG */
@@ -613,6 +613,12 @@
        if (cpu_feature[5] & CPUID_SEF_SMAP)
                cr4 |= CR4_SMAP;
 
+#ifdef SVS
+       /* If PCID is supported, enable it */
+       if (svs_pcid)
+               cr4 |= CR4_PCIDE;
+#endif
+
        if (cr4) {
                cr4 |= rcr4();
                lcr4(cr4);
diff -r 61c41fc5ee2b -r 23bf7d396085 sys/arch/x86/x86/svs.c
--- a/sys/arch/x86/x86/svs.c    Wed May 29 14:28:37 2019 +0000
+++ b/sys/arch/x86/x86/svs.c    Wed May 29 16:54:41 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $    */
+/*     $NetBSD: svs.c,v 1.29 2019/05/29 16:54:41 maxv Exp $    */
 
 /*
  * Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.29 2019/05/29 16:54:41 maxv Exp $");
 
 #include "opt_svs.h"
 
@@ -228,6 +228,10 @@
  */
 
 bool svs_enabled __read_mostly = false;
+bool svs_pcid __read_mostly = false;
+
+static uint64_t svs_pcid_kcr3 __read_mostly;
+static uint64_t svs_pcid_ucr3 __read_mostly;
 
 struct svs_utls {
        paddr_t kpdirpa;
@@ -405,6 +409,19 @@
 }
 
 static void
+svs_pcid_init(struct cpu_info *ci)
+{
+       if (!svs_pcid) {
+               return;
+       }
+
+       svs_pcid_ucr3 = __SHIFTIN(PMAP_PCID_USER, CR3_PCID) | CR3_NO_TLB_FLUSH;
+       svs_pcid_kcr3 = __SHIFTIN(PMAP_PCID_KERN, CR3_PCID) | CR3_NO_TLB_FLUSH;
+
+       ci->ci_svs_updirpa |= svs_pcid_ucr3;
+}
+
+static void
 svs_range_add(struct cpu_info *ci, vaddr_t va, size_t size, bool global)
 {
        size_t i, n;
@@ -454,6 +471,8 @@
 
        svs_rsp0_init(ci);
        svs_utls_init(ci);
+
+       svs_pcid_init(ci);
 }
 
 void
@@ -523,11 +542,14 @@
        utls->scratch = 0;
 
        /*
-        * Enter the user rsp0. We don't need to flush the TLB here, since
-        * the user page tables are not loaded.
+        * Enter the user rsp0. If we're using PCID we must flush the user VA,
+        * if we aren't it will be flushed during the next CR3 reload.
         */
        pte = ci->ci_svs_rsp0_pte;
        *pte = L1_BASE[pl1_i(va)];
+       if (svs_pcid) {
+               invpcid(INVPCID_ADDRESS, PMAP_PCID_USER, ci->ci_svs_rsp0);
+       }
 }
 
 static inline pt_entry_t
@@ -558,7 +580,7 @@
 
        /* Update the info in the UTLS page */
        utls = (struct svs_utls *)ci->ci_svs_utls;
-       utls->kpdirpa = pmap_pdirpa(pmap, 0);



Home | Main Index | Thread Index | Old Index