[src/trunk]: src/sys/arch Handle NMIs correctly when SVS is enabled. We store...

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/2922103c2973
branches:  trunk
changeset: 320576:2922103c2973
user:      maxv <maxv%NetBSD.org@localhost>
date:      Thu Jul 12 19:48:16 2018 +0000

description:
Handle NMIs correctly when SVS is enabled. We store the kernel's CR3 at the
top of the NMI stack, and we unconditionally switch to it, because we don't
know with which page tables we received the NMI. Hotpatch the whole thing as
usual.

This restores the ability to use PMCs on Intel CPUs.

diffstat:

 sys/arch/amd64/amd64/amd64_trap.S |   8 ++++----
 sys/arch/amd64/amd64/locore.S     |  22 +++++++++++++++++++++-
 sys/arch/amd64/amd64/machdep.c    |  16 ++++++++++++----
 sys/arch/amd64/include/frameasm.h |  20 +++++++++++++++++++-
 sys/arch/x86/x86/svs.c            |  28 ++++++++++++++++++++++++++--
 5 files changed, 82 insertions(+), 12 deletions(-)

diffs (287 lines):

diff -r b08fa56d3e87 -r 2922103c2973 sys/arch/amd64/amd64/amd64_trap.S
--- a/sys/arch/amd64/amd64/amd64_trap.S Thu Jul 12 18:39:09 2018 +0000
+++ b/sys/arch/amd64/amd64/amd64_trap.S Thu Jul 12 19:48:16 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: amd64_trap.S,v 1.42 2018/05/25 15:33:56 maxv Exp $     */
+/*     $NetBSD: amd64_trap.S,v 1.43 2018/07/12 19:48:16 maxv Exp $     */
 
 /*
  * Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
@@ -254,7 +254,6 @@
        subq    $TF_REGSIZE,%rsp
        INTR_SAVE_GPRS
        IBRS_ENTER
-       SVS_ENTER_ALTSTACK
        cld
        SMAP_ENABLE
        movw    %gs,TF_GS(%rsp)
@@ -262,6 +261,8 @@
        movw    %es,TF_ES(%rsp)
        movw    %ds,TF_DS(%rsp)
 
+       SVS_ENTER_NMI
+
        movl    $MSR_GSBASE,%ecx
        rdmsr
        cmpl    $VM_MIN_KERNEL_ADDRESS_HIGH32,%edx
@@ -271,7 +272,6 @@
        movq    %rsp,%rdi
        incq    CPUVAR(NTRAP)
        call    _C_LABEL(nmitrap)
-       SVS_LEAVE_ALTSTACK
        swapgs
        jmp     .Lnmileave
 
@@ -279,9 +279,9 @@
        movq    %rsp,%rdi
        incq    CPUVAR(NTRAP)
        call    _C_LABEL(nmitrap)
-       SVS_LEAVE_ALTSTACK
 
 .Lnmileave:
+       SVS_LEAVE_NMI
        IBRS_LEAVE
        INTR_RESTORE_GPRS
        addq    $TF_REGSIZE+16,%rsp
diff -r b08fa56d3e87 -r 2922103c2973 sys/arch/amd64/amd64/locore.S
--- a/sys/arch/amd64/amd64/locore.S     Thu Jul 12 18:39:09 2018 +0000
+++ b/sys/arch/amd64/amd64/locore.S     Thu Jul 12 19:48:16 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: locore.S,v 1.169 2018/07/09 18:52:04 maxv Exp $        */
+/*     $NetBSD: locore.S,v 1.170 2018/07/12 19:48:16 maxv Exp $        */
 
 /*
  * Copyright-o-rama!
@@ -1633,6 +1633,13 @@
 1234:
 LABEL(svs_enter_altstack_end)
 
+LABEL(svs_enter_nmi)
+       movq    %cr3,%rax
+       movq    %rax,(FRAMESIZE+1*8)(%rsp)      /* nmistore->scratch */
+       movq    (FRAMESIZE+0*8)(%rsp),%rax      /* nmistore->cr3 */
+       movq    %rax,%cr3
+LABEL(svs_enter_nmi_end)
+
 LABEL(svs_leave)
        testb   $SEL_UPL,TF_CS(%rsp)
        jz      1234f
@@ -1650,6 +1657,11 @@
 1234:
 LABEL(svs_leave_altstack_end)
 
+LABEL(svs_leave_nmi)
+       movq    (FRAMESIZE+1*8)(%rsp),%rax      /* nmistore->scratch */
+       movq    %rax,%cr3
+LABEL(svs_leave_nmi_end)
+
 LABEL(nosvs_enter)
        NOSVS_ENTER
 LABEL(nosvs_enter_end)
@@ -1658,6 +1670,10 @@
        NOSVS_ENTER_ALTSTACK
 LABEL(nosvs_enter_altstack_end)
 
+LABEL(nosvs_enter_nmi)
+       NOSVS_ENTER_NMI
+LABEL(nosvs_enter_nmi_end)
+
 LABEL(nosvs_leave)
        NOSVS_LEAVE
 LABEL(nosvs_leave_end)
@@ -1665,6 +1681,10 @@
 LABEL(nosvs_leave_altstack)
        NOSVS_LEAVE_ALTSTACK
 LABEL(nosvs_leave_altstack_end)
+
+LABEL(nosvs_leave_nmi)
+       NOSVS_LEAVE_NMI
+LABEL(nosvs_leave_nmi_end)
 #endif
 
        .globl  ibrs_enter, ibrs_enter_end
diff -r b08fa56d3e87 -r 2922103c2973 sys/arch/amd64/amd64/machdep.c
--- a/sys/arch/amd64/amd64/machdep.c    Thu Jul 12 18:39:09 2018 +0000
+++ b/sys/arch/amd64/amd64/machdep.c    Thu Jul 12 19:48:16 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: machdep.c,v 1.305 2018/06/20 11:49:37 maxv Exp $       */
+/*     $NetBSD: machdep.c,v 1.306 2018/07/12 19:48:16 maxv Exp $       */
 
 /*
  * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.305 2018/06/20 11:49:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.306 2018/07/12 19:48:16 maxv Exp $");
 
 #include "opt_modular.h"
 #include "opt_user_ldt.h"
@@ -281,6 +281,11 @@
 
 struct pool x86_dbregspl;
 
+struct nmistore {
+       uint64_t cr3;
+       uint64_t scratch;
+} __packed;
+
 /*
  * Size of memory segments, before any memory is stolen.
  */
@@ -506,6 +511,7 @@
        const cpuid_t cid = cpu_index(ci);
 #endif
        struct cpu_tss *cputss;
+       struct nmistore *store;
        uintptr_t p;
 
 #ifdef __HAVE_PCPU_AREA
@@ -533,13 +539,15 @@
 #endif
        cputss->tss.tss_ist[1] = p + PAGE_SIZE - 16;
 
-       /* NMI */
+       /* NMI - store a structure at the top of the stack */
 #ifdef __HAVE_PCPU_AREA
        p = (vaddr_t)&pcpuarea->ent[cid].ist2;
 #else
        p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO);
 #endif
-       cputss->tss.tss_ist[2] = p + PAGE_SIZE - 16;
+       cputss->tss.tss_ist[2] = p + PAGE_SIZE - sizeof(struct nmistore);
+       store = (struct nmistore *)(p + PAGE_SIZE - sizeof(struct nmistore));
+       store->cr3 = pmap_pdirpa(pmap_kernel(), 0);
 
        /* DB */
 #ifdef __HAVE_PCPU_AREA
diff -r b08fa56d3e87 -r 2922103c2973 sys/arch/amd64/include/frameasm.h
--- a/sys/arch/amd64/include/frameasm.h Thu Jul 12 18:39:09 2018 +0000
+++ b/sys/arch/amd64/include/frameasm.h Thu Jul 12 19:48:16 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: frameasm.h,v 1.38 2018/03/28 16:02:49 maxv Exp $       */
+/*     $NetBSD: frameasm.h,v 1.39 2018/07/12 19:48:16 maxv Exp $       */
 
 #ifndef _AMD64_MACHINE_FRAMEASM_H
 #define _AMD64_MACHINE_FRAMEASM_H
@@ -46,6 +46,8 @@
 #define HP_NAME_SVS_LEAVE_ALT  8
 #define HP_NAME_IBRS_ENTER     9
 #define HP_NAME_IBRS_LEAVE     10
+#define HP_NAME_SVS_ENTER_NMI  11
+#define HP_NAME_SVS_LEAVE_NMI  12
 
 #define HOTPATCH(name, size) \
 123:                                           ; \
@@ -165,6 +167,22 @@
        HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES)    ; \
        NOSVS_LEAVE_ALTSTACK
 
+#define SVS_ENTER_NMI_BYTES    22
+#define NOSVS_ENTER_NMI \
+       .byte 0xEB, (SVS_ENTER_NMI_BYTES-2)     /* jmp */       ; \
+       .fill   (SVS_ENTER_NMI_BYTES-2),1,0xCC
+#define SVS_ENTER_NMI \
+       HOTPATCH(HP_NAME_SVS_ENTER_NMI, SVS_ENTER_NMI_BYTES)    ; \
+       NOSVS_ENTER_NMI
+
+#define SVS_LEAVE_NMI_BYTES    11
+#define NOSVS_LEAVE_NMI \
+       .byte 0xEB, (SVS_LEAVE_NMI_BYTES-2)     /* jmp */       ; \
+       .fill   (SVS_LEAVE_NMI_BYTES-2),1,0xCC
+#define SVS_LEAVE_NMI \
+       HOTPATCH(HP_NAME_SVS_LEAVE_NMI, SVS_LEAVE_NMI_BYTES)    ; \
+       NOSVS_LEAVE_NMI
+
 #else
 #define SVS_ENTER      /* nothing */
 #define SVS_LEAVE      /* nothing */
diff -r b08fa56d3e87 -r 2922103c2973 sys/arch/x86/x86/svs.c
--- a/sys/arch/x86/x86/svs.c    Thu Jul 12 18:39:09 2018 +0000
+++ b/sys/arch/x86/x86/svs.c    Thu Jul 12 19:48:16 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: svs.c,v 1.18 2018/04/26 18:54:09 alnsn Exp $   */
+/*     $NetBSD: svs.c,v 1.19 2018/07/12 19:48:16 maxv Exp $    */
 
 /*
  * Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.18 2018/04/26 18:54:09 alnsn Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.19 2018/07/12 19:48:16 maxv Exp $");
 
 #include "opt_svs.h"
 
@@ -571,8 +571,10 @@
 {
        extern uint8_t svs_enter, svs_enter_end;
        extern uint8_t svs_enter_altstack, svs_enter_altstack_end;
+       extern uint8_t svs_enter_nmi, svs_enter_nmi_end;
        extern uint8_t svs_leave, svs_leave_end;
        extern uint8_t svs_leave_altstack, svs_leave_altstack_end;
+       extern uint8_t svs_leave_nmi, svs_leave_nmi_end;
        u_long psl, cr0;
        uint8_t *bytes;
        size_t size;
@@ -590,6 +592,11 @@
            (size_t)&svs_enter_altstack;
        x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
 
+       bytes = &svs_enter_nmi;
+       size = (size_t)&svs_enter_nmi_end -
+           (size_t)&svs_enter_nmi;
+       x86_hotpatch(HP_NAME_SVS_ENTER_NMI, bytes, size);
+
        bytes = &svs_leave;
        size = (size_t)&svs_leave_end - (size_t)&svs_leave;
        x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
@@ -599,6 +606,11 @@
            (size_t)&svs_leave_altstack;
        x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
 
+       bytes = &svs_leave_nmi;
+       size = (size_t)&svs_leave_nmi_end -
+           (size_t)&svs_leave_nmi;
+       x86_hotpatch(HP_NAME_SVS_LEAVE_NMI, bytes, size);
+
        x86_patch_window_close(psl, cr0);
 }
 
@@ -607,8 +619,10 @@
 {
        extern uint8_t nosvs_enter, nosvs_enter_end;
        extern uint8_t nosvs_enter_altstack, nosvs_enter_altstack_end;
+       extern uint8_t nosvs_enter_nmi, nosvs_enter_nmi_end;
        extern uint8_t nosvs_leave, nosvs_leave_end;
        extern uint8_t nosvs_leave_altstack, nosvs_leave_altstack_end;
+       extern uint8_t nosvs_leave_nmi, nosvs_leave_nmi_end;
        u_long psl, cr0;
        uint8_t *bytes;
        size_t size;
@@ -624,6 +638,11 @@
            (size_t)&nosvs_enter_altstack;
        x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
 
+       bytes = &nosvs_enter_nmi;
+       size = (size_t)&nosvs_enter_nmi_end -
+           (size_t)&nosvs_enter_nmi;
+       x86_hotpatch(HP_NAME_SVS_ENTER_NMI, bytes, size);
+
        bytes = &nosvs_leave;
        size = (size_t)&nosvs_leave_end - (size_t)&nosvs_leave;
        x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
@@ -633,6 +652,11 @@
            (size_t)&nosvs_leave_altstack;
        x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
 
+       bytes = &nosvs_leave_nmi;
+       size = (size_t)&nosvs_leave_nmi_end -
+           (size_t)&nosvs_leave_nmi;
+       x86_hotpatch(HP_NAME_SVS_LEAVE_NMI, bytes, size);
+
        x86_patch_window_close(psl, cr0);
 }