Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/arch Add the IBRS mitigation for SpectreV2 on amd64.



details:   https://anonhg.NetBSD.org/src/rev/0e69d006e48b
branches:  trunk
changeset: 360734:0e69d006e48b
user:      maxv <maxv%NetBSD.org@localhost>
date:      Wed Mar 28 16:02:49 2018 +0000

description:
Add the IBRS mitigation for SpectreV2 on amd64.

Different operations are performed during context transitions:

        user->kernel: IBRS <- 1
        kernel->user: IBRS <- 0

And during context switches:

        user->user:   IBPB <- 0
        kernel->user: IBPB <- 0
        [user->kernel:IBPB <- 0 this one may not be needed]

We use two macros, IBRS_ENTER and IBRS_LEAVE, to set the IBRS bit. The
thing is hotpatched for better performance, like SVS.

The idea is that IBRS is a "privileged" bit, which is set to 1 in kernel
mode and 0 in user mode. To protect the branch predictor between user
processes (which are of the same privilege), we use the IBPB barrier.

The Intel manual also talks about (MWAIT/HLT)+HyperThreading, and says
that when using either of the two instructions IBRS must be disabled for
better performance on the core. I'm not totally sure about this part, so
I'm not adding it now.

IBRS is available only when the Intel microcode update is applied. The
mitigation must be enabled manually with machdep.spectreV2.mitigated.

Tested by msaitoh a week ago (but I adapted a few things since). Probably
more changes to come.

diffstat:

 sys/arch/amd64/amd64/amd64_trap.S |    6 +-
 sys/arch/amd64/amd64/locore.S     |   31 ++++++-
 sys/arch/amd64/include/frameasm.h |   25 +++++-
 sys/arch/x86/x86/spectre.c        |  172 +++++++++++++++++++++++++++++++++++--
 4 files changed, 220 insertions(+), 14 deletions(-)

diffs (truncated from 433 to 300 lines):

diff -r 975150290108 -r 0e69d006e48b sys/arch/amd64/amd64/amd64_trap.S
--- a/sys/arch/amd64/amd64/amd64_trap.S Wed Mar 28 15:34:29 2018 +0000
+++ b/sys/arch/amd64/amd64/amd64_trap.S Wed Mar 28 16:02:49 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: amd64_trap.S,v 1.39 2018/03/20 18:27:58 maxv Exp $     */
+/*     $NetBSD: amd64_trap.S,v 1.40 2018/03/28 16:02:49 maxv Exp $     */
 
 /*
  * Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
@@ -134,6 +134,7 @@
        ZTRAP_NJ(T_NMI)
        subq    $TF_REGSIZE,%rsp
        INTR_SAVE_GPRS
+       IBRS_ENTER
        SVS_ENTER_ALTSTACK
        cld
        SMAP_ENABLE
@@ -162,6 +163,7 @@
        SVS_LEAVE_ALTSTACK
 
 .Lnmileave:
+       IBRS_LEAVE
        INTR_RESTORE_GPRS
        addq    $TF_REGSIZE+16,%rsp
        iretq
@@ -231,6 +233,7 @@
        TRAP_NJ(T_DOUBLEFLT)
        subq    $TF_REGSIZE,%rsp
        INTR_SAVE_GPRS
+       IBRS_ENTER
        SVS_ENTER_ALTSTACK
        testb   $SEL_UPL,TF_CS(%rsp)
        jz      1f
@@ -248,6 +251,7 @@
        call    _C_LABEL(doubletrap)
 
        SVS_LEAVE_ALTSTACK
+       IBRS_LEAVE
        INTR_RESTORE_GPRS
 
        testb   $SEL_UPL,TF_CS(%rsp)
diff -r 975150290108 -r 0e69d006e48b sys/arch/amd64/amd64/locore.S
--- a/sys/arch/amd64/amd64/locore.S     Wed Mar 28 15:34:29 2018 +0000
+++ b/sys/arch/amd64/amd64/locore.S     Wed Mar 28 16:02:49 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: locore.S,v 1.159 2018/03/20 18:27:58 maxv Exp $        */
+/*     $NetBSD: locore.S,v 1.160 2018/03/28 16:02:49 maxv Exp $        */
 
 /*
  * Copyright-o-rama!
@@ -1098,6 +1098,10 @@
        popq    %rdx
 #endif
 
+       pushq   %rdx
+       callq   _C_LABEL(speculation_barrier)
+       popq    %rdx
+
        /* Switch to newlwp's stack. */
        movq    L_PCB(%r12),%r14
        movq    PCB_RSP(%r14),%rsp
@@ -1413,6 +1417,7 @@
        cld
 #endif
        INTR_SAVE_GPRS
+       IBRS_ENTER
        movw    $GSEL(GUDATA_SEL, SEL_UPL),TF_DS(%rsp)
        movw    $GSEL(GUDATA_SEL, SEL_UPL),TF_ES(%rsp)
        movw    $0,TF_FS(%rsp)
@@ -1464,6 +1469,7 @@
        _ALIGN_TEXT
 LABEL(syscall_sysret)
        SVS_LEAVE
+       IBRS_LEAVE
        INTR_RESTORE_GPRS
        SWAPGS
 #ifndef XEN
@@ -1554,6 +1560,7 @@
 LABEL(intrfastexit)
        NOT_XEN(cli;)
        SVS_LEAVE
+       IBRS_LEAVE
        INTR_RESTORE_GPRS
        addq    $(TF_REGSIZE+16),%rsp   /* iret frame */
 
@@ -1649,3 +1656,25 @@
        NOSVS_LEAVE_ALTSTACK
 LABEL(nosvs_leave_altstack_end)
 #endif
+
+       .globl  ibrs_enter, ibrs_enter_end
+       .globl  ibrs_leave, ibrs_leave_end
+
+       /* IBRS <- 1 */
+LABEL(ibrs_enter)
+       movl    $MSR_IA32_SPEC_CTRL,%ecx
+       movl    $IA32_SPEC_CTRL_IBRS,%eax
+       movl    $(IA32_SPEC_CTRL_IBRS >> 32),%edx
+       wrmsr
+LABEL(ibrs_enter_end)
+
+       /* IBRS <- 0 */
+LABEL(ibrs_leave)
+       testb   $SEL_UPL,TF_CS(%rsp)
+       jz      1234f
+       movl    $MSR_IA32_SPEC_CTRL,%ecx
+       xorl    %eax,%eax
+       xorl    %edx,%edx
+       wrmsr
+1234:
+LABEL(ibrs_leave_end)
diff -r 975150290108 -r 0e69d006e48b sys/arch/amd64/include/frameasm.h
--- a/sys/arch/amd64/include/frameasm.h Wed Mar 28 15:34:29 2018 +0000
+++ b/sys/arch/amd64/include/frameasm.h Wed Mar 28 16:02:49 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: frameasm.h,v 1.37 2018/02/25 13:14:27 maxv Exp $       */
+/*     $NetBSD: frameasm.h,v 1.38 2018/03/28 16:02:49 maxv Exp $       */
 
 #ifndef _AMD64_MACHINE_FRAMEASM_H
 #define _AMD64_MACHINE_FRAMEASM_H
@@ -44,6 +44,8 @@
 #define HP_NAME_SVS_LEAVE      6
 #define HP_NAME_SVS_ENTER_ALT  7
 #define HP_NAME_SVS_LEAVE_ALT  8
+#define HP_NAME_IBRS_ENTER     9
+#define HP_NAME_IBRS_LEAVE     10
 
 #define HOTPATCH(name, size) \
 123:                                           ; \
@@ -61,6 +63,26 @@
        HOTPATCH(HP_NAME_STAC, 3)               ; \
        .byte 0x0F, 0x1F, 0x00                  ; \
 
+/*
+ * IBRS
+ */
+
+#define IBRS_ENTER_BYTES       17
+#define IBRS_ENTER \
+       HOTPATCH(HP_NAME_IBRS_ENTER, IBRS_ENTER_BYTES)          ; \
+       NOIBRS_ENTER
+#define NOIBRS_ENTER \
+       .byte 0xEB, (IBRS_ENTER_BYTES-2)        /* jmp */       ; \
+       .fill   (IBRS_ENTER_BYTES-2),1,0xCC
+
+#define IBRS_LEAVE_BYTES       21
+#define IBRS_LEAVE \
+       HOTPATCH(HP_NAME_IBRS_LEAVE, IBRS_LEAVE_BYTES)          ; \
+       NOIBRS_LEAVE
+#define NOIBRS_LEAVE \
+       .byte 0xEB, (IBRS_LEAVE_BYTES-2)        /* jmp */       ; \
+       .fill   (IBRS_LEAVE_BYTES-2),1,0xCC
+
 #define        SWAPGS  NOT_XEN(swapgs)
 
 /*
@@ -158,6 +180,7 @@
        testb   $SEL_UPL,TF_CS(%rsp)    ; \
        je      98f                     ; \
        SWAPGS                          ; \
+       IBRS_ENTER                      ; \
        SVS_ENTER                       ; \
        movw    %gs,TF_GS(%rsp)         ; \
        movw    %fs,TF_FS(%rsp)         ; \
diff -r 975150290108 -r 0e69d006e48b sys/arch/x86/x86/spectre.c
--- a/sys/arch/x86/x86/spectre.c        Wed Mar 28 15:34:29 2018 +0000
+++ b/sys/arch/x86/x86/spectre.c        Wed Mar 28 16:02:49 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: spectre.c,v 1.1 2018/03/28 14:56:59 maxv Exp $ */
+/*     $NetBSD: spectre.c,v 1.2 2018/03/28 16:02:49 maxv Exp $ */
 
 /*
  * Copyright (c) 2018 NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.1 2018/03/28 14:56:59 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.2 2018/03/28 16:02:49 maxv Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -45,6 +45,7 @@
 #include <machine/cpufunc.h>
 #include <machine/cpuvar.h>
 #include <machine/specialreg.h>
+#include <machine/frameasm.h>
 
 #include <x86/cputypes.h>
 
@@ -57,13 +58,46 @@
 bool spec_mitigation_enabled __read_mostly = false;
 static enum spec_mitigation mitigation_method = MITIGATION_NONE;
 
+void speculation_barrier(struct lwp *, struct lwp *);
+
+void
+speculation_barrier(struct lwp *oldlwp, struct lwp *newlwp)
+{
+       if (!spec_mitigation_enabled)
+               return;
+
+       /*
+        * From kernel thread to kernel thread, no need for a barrier.
+        */
+       if ((oldlwp->l_flag & LW_SYSTEM) &&
+           (newlwp->l_flag & LW_SYSTEM))
+               return;
+
+       switch (mitigation_method) {
+       case MITIGATION_INTEL_IBRS:
+               wrmsr(MSR_IA32_PRED_CMD, IA32_PRED_CMD_IBPB);
+               break;
+       default:
+               /* nothing */
+               break;
+       }
+}
+
 static void
 speculation_detect_method(void)
 {
        struct cpu_info *ci = curcpu();
+       u_int descs[4];
 
        if (cpu_vendor == CPUVENDOR_INTEL) {
-               /* TODO: detect MITIGATION_INTEL_IBRS */
+               if (cpuid_level >= 7) {
+                       x86_cpuid(7, descs);
+                       if (descs[3] & CPUID_SEF_IBRS) {
+                               /* descs[3] = %edx */
+                               mitigation_method = MITIGATION_INTEL_IBRS;
+                               return;
+                       }
+               }
                mitigation_method = MITIGATION_NONE;
        } else if (cpu_vendor == CPUVENDOR_AMD) {
                /*
@@ -88,6 +122,118 @@
        }
 }
 
+/* -------------------------------------------------------------------------- */
+
+#ifdef __x86_64__
+static volatile unsigned long ibrs_cpu_barrier1 __cacheline_aligned;
+static volatile unsigned long ibrs_cpu_barrier2 __cacheline_aligned;
+
+static void
+ibrs_enable_hotpatch(void)
+{
+       extern uint8_t ibrs_enter, ibrs_enter_end;
+       extern uint8_t ibrs_leave, ibrs_leave_end;
+       u_long psl, cr0;
+       uint8_t *bytes;
+       size_t size;
+
+       x86_patch_window_open(&psl, &cr0);
+
+       bytes = &ibrs_enter;
+       size = (size_t)&ibrs_enter_end - (size_t)&ibrs_enter;
+       x86_hotpatch(HP_NAME_IBRS_ENTER, bytes, size);
+
+       bytes = &ibrs_leave;
+       size = (size_t)&ibrs_leave_end - (size_t)&ibrs_leave;
+       x86_hotpatch(HP_NAME_IBRS_LEAVE, bytes, size);
+
+       x86_patch_window_close(psl, cr0);
+}
+
+static void
+ibrs_change_cpu(void *arg1, void *arg2)
+{
+       struct cpu_info *ci = curcpu();
+       bool enabled = (bool)arg1;
+       u_long psl;
+
+       psl = x86_read_psl();
+       x86_disable_intr();
+
+       atomic_dec_ulong(&ibrs_cpu_barrier1);
+       while (atomic_cas_ulong(&ibrs_cpu_barrier1, 0, 0) != 0) {
+               x86_pause();
+       }
+
+       /* cpu0 is the one that does the hotpatch job */
+       if (ci == &cpu_info_primary) {
+               if (enabled) {
+                       ibrs_enable_hotpatch();
+               } else {
+                       /* TODO */
+               }
+       }
+
+       atomic_dec_ulong(&ibrs_cpu_barrier2);
+       while (atomic_cas_ulong(&ibrs_cpu_barrier2, 0, 0) != 0) {
+               x86_pause();
+       }
+
+       /* Write back and invalidate cache, flush pipelines. */



Home | Main Index | Thread Index | Old Index