Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/arch Add the IBRS mitigation for SpectreV2 on amd64.
details: https://anonhg.NetBSD.org/src/rev/f779a400c689
branches: trunk
changeset: 321622:f779a400c689
user: maxv <maxv%NetBSD.org@localhost>
date: Wed Mar 28 16:02:49 2018 +0000
description:
Add the IBRS mitigation for SpectreV2 on amd64.
Different operations are performed during context transitions:
user->kernel: IBRS <- 1
kernel->user: IBRS <- 0
And during context switches:
user->user: IBPB <- 0
kernel->user: IBPB <- 0
[user->kernel:IBPB <- 0 this one may not be needed]
We use two macros, IBRS_ENTER and IBRS_LEAVE, to set the IBRS bit. The
thing is hotpatched for better performance, like SVS.
The idea is that IBRS is a "privileged" bit, which is set to 1 in kernel
mode and 0 in user mode. To protect the branch predictor between user
processes (which are of the same privilege), we use the IBPB barrier.
The Intel manual also talks about (MWAIT/HLT)+HyperThreading, and says
that when using either of the two instructions IBRS must be disabled for
better performance on the core. I'm not totally sure about this part, so
I'm not adding it now.
IBRS is available only when the Intel microcode update is applied. The
mitigation must be enabled manually with machdep.spectreV2.mitigated.
Tested by msaitoh a week ago (but I adapted a few things since). Probably
more changes to come.
diffstat:
sys/arch/amd64/amd64/amd64_trap.S | 6 +-
sys/arch/amd64/amd64/locore.S | 31 ++++++-
sys/arch/amd64/include/frameasm.h | 25 +++++-
sys/arch/x86/x86/spectre.c | 172 +++++++++++++++++++++++++++++++++++--
4 files changed, 220 insertions(+), 14 deletions(-)
diffs (truncated from 433 to 300 lines):
diff -r 5085914e21ce -r f779a400c689 sys/arch/amd64/amd64/amd64_trap.S
--- a/sys/arch/amd64/amd64/amd64_trap.S Wed Mar 28 15:34:29 2018 +0000
+++ b/sys/arch/amd64/amd64/amd64_trap.S Wed Mar 28 16:02:49 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: amd64_trap.S,v 1.39 2018/03/20 18:27:58 maxv Exp $ */
+/* $NetBSD: amd64_trap.S,v 1.40 2018/03/28 16:02:49 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
@@ -134,6 +134,7 @@
ZTRAP_NJ(T_NMI)
subq $TF_REGSIZE,%rsp
INTR_SAVE_GPRS
+ IBRS_ENTER
SVS_ENTER_ALTSTACK
cld
SMAP_ENABLE
@@ -162,6 +163,7 @@
SVS_LEAVE_ALTSTACK
.Lnmileave:
+ IBRS_LEAVE
INTR_RESTORE_GPRS
addq $TF_REGSIZE+16,%rsp
iretq
@@ -231,6 +233,7 @@
TRAP_NJ(T_DOUBLEFLT)
subq $TF_REGSIZE,%rsp
INTR_SAVE_GPRS
+ IBRS_ENTER
SVS_ENTER_ALTSTACK
testb $SEL_UPL,TF_CS(%rsp)
jz 1f
@@ -248,6 +251,7 @@
call _C_LABEL(doubletrap)
SVS_LEAVE_ALTSTACK
+ IBRS_LEAVE
INTR_RESTORE_GPRS
testb $SEL_UPL,TF_CS(%rsp)
diff -r 5085914e21ce -r f779a400c689 sys/arch/amd64/amd64/locore.S
--- a/sys/arch/amd64/amd64/locore.S Wed Mar 28 15:34:29 2018 +0000
+++ b/sys/arch/amd64/amd64/locore.S Wed Mar 28 16:02:49 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.159 2018/03/20 18:27:58 maxv Exp $ */
+/* $NetBSD: locore.S,v 1.160 2018/03/28 16:02:49 maxv Exp $ */
/*
* Copyright-o-rama!
@@ -1098,6 +1098,10 @@
popq %rdx
#endif
+ pushq %rdx
+ callq _C_LABEL(speculation_barrier)
+ popq %rdx
+
/* Switch to newlwp's stack. */
movq L_PCB(%r12),%r14
movq PCB_RSP(%r14),%rsp
@@ -1413,6 +1417,7 @@
cld
#endif
INTR_SAVE_GPRS
+ IBRS_ENTER
movw $GSEL(GUDATA_SEL, SEL_UPL),TF_DS(%rsp)
movw $GSEL(GUDATA_SEL, SEL_UPL),TF_ES(%rsp)
movw $0,TF_FS(%rsp)
@@ -1464,6 +1469,7 @@
_ALIGN_TEXT
LABEL(syscall_sysret)
SVS_LEAVE
+ IBRS_LEAVE
INTR_RESTORE_GPRS
SWAPGS
#ifndef XEN
@@ -1554,6 +1560,7 @@
LABEL(intrfastexit)
NOT_XEN(cli;)
SVS_LEAVE
+ IBRS_LEAVE
INTR_RESTORE_GPRS
addq $(TF_REGSIZE+16),%rsp /* iret frame */
@@ -1649,3 +1656,25 @@
NOSVS_LEAVE_ALTSTACK
LABEL(nosvs_leave_altstack_end)
#endif
+
+ .globl ibrs_enter, ibrs_enter_end
+ .globl ibrs_leave, ibrs_leave_end
+
+ /* IBRS <- 1 */
+LABEL(ibrs_enter)
+ movl $MSR_IA32_SPEC_CTRL,%ecx
+ movl $IA32_SPEC_CTRL_IBRS,%eax
+ movl $(IA32_SPEC_CTRL_IBRS >> 32),%edx
+ wrmsr
+LABEL(ibrs_enter_end)
+
+ /* IBRS <- 0 */
+LABEL(ibrs_leave)
+ testb $SEL_UPL,TF_CS(%rsp)
+ jz 1234f
+ movl $MSR_IA32_SPEC_CTRL,%ecx
+ xorl %eax,%eax
+ xorl %edx,%edx
+ wrmsr
+1234:
+LABEL(ibrs_leave_end)
diff -r 5085914e21ce -r f779a400c689 sys/arch/amd64/include/frameasm.h
--- a/sys/arch/amd64/include/frameasm.h Wed Mar 28 15:34:29 2018 +0000
+++ b/sys/arch/amd64/include/frameasm.h Wed Mar 28 16:02:49 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.37 2018/02/25 13:14:27 maxv Exp $ */
+/* $NetBSD: frameasm.h,v 1.38 2018/03/28 16:02:49 maxv Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -44,6 +44,8 @@
#define HP_NAME_SVS_LEAVE 6
#define HP_NAME_SVS_ENTER_ALT 7
#define HP_NAME_SVS_LEAVE_ALT 8
+#define HP_NAME_IBRS_ENTER 9
+#define HP_NAME_IBRS_LEAVE 10
#define HOTPATCH(name, size) \
123: ; \
@@ -61,6 +63,26 @@
HOTPATCH(HP_NAME_STAC, 3) ; \
.byte 0x0F, 0x1F, 0x00 ; \
+/*
+ * IBRS
+ */
+
+#define IBRS_ENTER_BYTES 17
+#define IBRS_ENTER \
+ HOTPATCH(HP_NAME_IBRS_ENTER, IBRS_ENTER_BYTES) ; \
+ NOIBRS_ENTER
+#define NOIBRS_ENTER \
+ .byte 0xEB, (IBRS_ENTER_BYTES-2) /* jmp */ ; \
+ .fill (IBRS_ENTER_BYTES-2),1,0xCC
+
+#define IBRS_LEAVE_BYTES 21
+#define IBRS_LEAVE \
+ HOTPATCH(HP_NAME_IBRS_LEAVE, IBRS_LEAVE_BYTES) ; \
+ NOIBRS_LEAVE
+#define NOIBRS_LEAVE \
+ .byte 0xEB, (IBRS_LEAVE_BYTES-2) /* jmp */ ; \
+ .fill (IBRS_LEAVE_BYTES-2),1,0xCC
+
#define SWAPGS NOT_XEN(swapgs)
/*
@@ -158,6 +180,7 @@
testb $SEL_UPL,TF_CS(%rsp) ; \
je 98f ; \
SWAPGS ; \
+ IBRS_ENTER ; \
SVS_ENTER ; \
movw %gs,TF_GS(%rsp) ; \
movw %fs,TF_FS(%rsp) ; \
diff -r 5085914e21ce -r f779a400c689 sys/arch/x86/x86/spectre.c
--- a/sys/arch/x86/x86/spectre.c Wed Mar 28 15:34:29 2018 +0000
+++ b/sys/arch/x86/x86/spectre.c Wed Mar 28 16:02:49 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: spectre.c,v 1.1 2018/03/28 14:56:59 maxv Exp $ */
+/* $NetBSD: spectre.c,v 1.2 2018/03/28 16:02:49 maxv Exp $ */
/*
* Copyright (c) 2018 NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.1 2018/03/28 14:56:59 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.2 2018/03/28 16:02:49 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -45,6 +45,7 @@
#include <machine/cpufunc.h>
#include <machine/cpuvar.h>
#include <machine/specialreg.h>
+#include <machine/frameasm.h>
#include <x86/cputypes.h>
@@ -57,13 +58,46 @@
bool spec_mitigation_enabled __read_mostly = false;
static enum spec_mitigation mitigation_method = MITIGATION_NONE;
+void speculation_barrier(struct lwp *, struct lwp *);
+
+void
+speculation_barrier(struct lwp *oldlwp, struct lwp *newlwp)
+{
+ if (!spec_mitigation_enabled)
+ return;
+
+ /*
+ * From kernel thread to kernel thread, no need for a barrier.
+ */
+ if ((oldlwp->l_flag & LW_SYSTEM) &&
+ (newlwp->l_flag & LW_SYSTEM))
+ return;
+
+ switch (mitigation_method) {
+ case MITIGATION_INTEL_IBRS:
+ wrmsr(MSR_IA32_PRED_CMD, IA32_PRED_CMD_IBPB);
+ break;
+ default:
+ /* nothing */
+ break;
+ }
+}
+
static void
speculation_detect_method(void)
{
struct cpu_info *ci = curcpu();
+ u_int descs[4];
if (cpu_vendor == CPUVENDOR_INTEL) {
- /* TODO: detect MITIGATION_INTEL_IBRS */
+ if (cpuid_level >= 7) {
+ x86_cpuid(7, descs);
+ if (descs[3] & CPUID_SEF_IBRS) {
+ /* descs[3] = %edx */
+ mitigation_method = MITIGATION_INTEL_IBRS;
+ return;
+ }
+ }
mitigation_method = MITIGATION_NONE;
} else if (cpu_vendor == CPUVENDOR_AMD) {
/*
@@ -88,6 +122,118 @@
}
}
+/* -------------------------------------------------------------------------- */
+
+#ifdef __x86_64__
+static volatile unsigned long ibrs_cpu_barrier1 __cacheline_aligned;
+static volatile unsigned long ibrs_cpu_barrier2 __cacheline_aligned;
+
+static void
+ibrs_enable_hotpatch(void)
+{
+ extern uint8_t ibrs_enter, ibrs_enter_end;
+ extern uint8_t ibrs_leave, ibrs_leave_end;
+ u_long psl, cr0;
+ uint8_t *bytes;
+ size_t size;
+
+ x86_patch_window_open(&psl, &cr0);
+
+ bytes = &ibrs_enter;
+ size = (size_t)&ibrs_enter_end - (size_t)&ibrs_enter;
+ x86_hotpatch(HP_NAME_IBRS_ENTER, bytes, size);
+
+ bytes = &ibrs_leave;
+ size = (size_t)&ibrs_leave_end - (size_t)&ibrs_leave;
+ x86_hotpatch(HP_NAME_IBRS_LEAVE, bytes, size);
+
+ x86_patch_window_close(psl, cr0);
+}
+
+static void
+ibrs_change_cpu(void *arg1, void *arg2)
+{
+ struct cpu_info *ci = curcpu();
+ bool enabled = (bool)arg1;
+ u_long psl;
+
+ psl = x86_read_psl();
+ x86_disable_intr();
+
+ atomic_dec_ulong(&ibrs_cpu_barrier1);
+ while (atomic_cas_ulong(&ibrs_cpu_barrier1, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ /* cpu0 is the one that does the hotpatch job */
+ if (ci == &cpu_info_primary) {
+ if (enabled) {
+ ibrs_enable_hotpatch();
+ } else {
+ /* TODO */
+ }
+ }
+
+ atomic_dec_ulong(&ibrs_cpu_barrier2);
+ while (atomic_cas_ulong(&ibrs_cpu_barrier2, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ /* Write back and invalidate cache, flush pipelines. */
Home |
Main Index |
Thread Index |
Old Index