Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/netbsd-8]: src/sys/arch Pull up the following, via patch, requested by m...
details: https://anonhg.NetBSD.org/src/rev/49cfecfafb22
branches: netbsd-8
changeset: 320120:49cfecfafb22
user: martin <martin%NetBSD.org@localhost>
date: Sat Jun 23 11:39:01 2018 +0000
description:
Pull up the following, via patch, requested by maxv in ticket #897:
sys/arch/amd64/amd64/locore.S 1.166 (patch)
sys/arch/i386/i386/locore.S 1.157 (patch)
sys/arch/x86/include/cpu.h 1.92 (patch)
sys/arch/x86/include/fpu.h 1.9 (patch)
sys/arch/x86/x86/fpu.c 1.33-1.39 (patch)
sys/arch/x86/x86/identcpu.c 1.72 (patch)
sys/arch/x86/x86/vm_machdep.c 1.34 (patch)
sys/arch/x86/x86/x86_machdep.c 1.116,1.117 (patch)
Support eager fpu switch, to work around INTEL-SA-00145.
Provide a sysctl machdep.fpu_eager, which gets automatically
initialized to 1 on affected CPUs.
diffstat:
sys/arch/amd64/amd64/locore.S | 16 ++-
sys/arch/i386/i386/locore.S | 17 ++-
sys/arch/x86/include/cpu.h | 3 +-
sys/arch/x86/include/fpu.h | 4 +-
sys/arch/x86/x86/fpu.c | 268 +++++++++++++++++++++++++++++++++++-----
sys/arch/x86/x86/identcpu.c | 52 +++++++-
sys/arch/x86/x86/vm_machdep.c | 8 +-
sys/arch/x86/x86/x86_machdep.c | 9 +-
8 files changed, 329 insertions(+), 48 deletions(-)
diffs (truncated from 602 to 300 lines):
diff -r 0afef4bb2fff -r 49cfecfafb22 sys/arch/amd64/amd64/locore.S
--- a/sys/arch/amd64/amd64/locore.S Sat Jun 23 11:14:23 2018 +0000
+++ b/sys/arch/amd64/amd64/locore.S Sat Jun 23 11:39:01 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.123.6.6 2018/05/05 15:00:29 martin Exp $ */
+/* $NetBSD: locore.S,v 1.123.6.7 2018/06/23 11:39:01 martin Exp $ */
/*
* Copyright-o-rama!
@@ -1094,6 +1094,20 @@
popq %rdx
#endif
+#ifndef XEN
+ /* RDI/RSI got clobbered. */
+ movq %r13,%rdi
+ movq %r12,%rsi
+
+ pushq %rdx
+ movb _C_LABEL(x86_fpu_eager),%dl
+ testb %dl,%dl
+ jz .Lno_eagerfpu
+ callq _C_LABEL(fpu_eagerswitch)
+.Lno_eagerfpu:
+ popq %rdx
+#endif
+
/* Switch to newlwp's stack. */
movq L_PCB(%r12),%r14
movq PCB_RSP(%r14),%rsp
diff -r 0afef4bb2fff -r 49cfecfafb22 sys/arch/i386/i386/locore.S
--- a/sys/arch/i386/i386/locore.S Sat Jun 23 11:14:23 2018 +0000
+++ b/sys/arch/i386/i386/locore.S Sat Jun 23 11:39:01 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.145.6.3 2018/03/13 15:47:44 martin Exp $ */
+/* $NetBSD: locore.S,v 1.145.6.4 2018/06/23 11:39:02 martin Exp $ */
/*
* Copyright-o-rama!
@@ -128,7 +128,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: locore.S,v 1.145.6.3 2018/03/13 15:47:44 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: locore.S,v 1.145.6.4 2018/06/23 11:39:02 martin Exp $");
#include "opt_compat_oldboot.h"
#include "opt_copy_symtab.h"
@@ -1089,6 +1089,19 @@
movl %ebp,PCB_EBP(%eax)
skip_save:
+#ifndef XEN
+ pushl %edx
+ movb _C_LABEL(x86_fpu_eager),%dl
+ testb %dl,%dl
+ jz .Lno_eagerfpu
+ pushl %edi
+ pushl %esi
+ call _C_LABEL(fpu_eagerswitch)
+ addl $8,%esp
+.Lno_eagerfpu:
+ popl %edx
+#endif
+
/* Switch to newlwp's stack. */
movl L_PCB(%edi),%ebx
movl PCB_EBP(%ebx),%ebp
diff -r 0afef4bb2fff -r 49cfecfafb22 sys/arch/x86/include/cpu.h
--- a/sys/arch/x86/include/cpu.h Sat Jun 23 11:14:23 2018 +0000
+++ b/sys/arch/x86/include/cpu.h Sat Jun 23 11:39:01 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.71.2.6 2018/06/09 15:12:21 martin Exp $ */
+/* $NetBSD: cpu.h,v 1.71.2.7 2018/06/23 11:39:02 martin Exp $ */
/*-
* Copyright (c) 1990 The Regents of the University of California.
@@ -424,6 +424,7 @@
#define FPU_SAVE_XSAVEOPT 3
extern unsigned int x86_fpu_save_size;
extern uint64_t x86_xsave_features;
+extern bool x86_fpu_eager;
extern void (*x86_cpu_idle)(void);
#define cpu_idle() (*x86_cpu_idle)()
diff -r 0afef4bb2fff -r 49cfecfafb22 sys/arch/x86/include/fpu.h
--- a/sys/arch/x86/include/fpu.h Sat Jun 23 11:14:23 2018 +0000
+++ b/sys/arch/x86/include/fpu.h Sat Jun 23 11:39:01 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: fpu.h,v 1.6 2014/02/25 22:16:52 dsl Exp $ */
+/* $NetBSD: fpu.h,v 1.6.28.1 2018/06/23 11:39:02 martin Exp $ */
#ifndef _X86_FPU_H_
#define _X86_FPU_H_
@@ -15,6 +15,8 @@
void fpusave_lwp(struct lwp *, bool);
void fpusave_cpu(bool);
+void fpu_eagerswitch(struct lwp *, struct lwp *);
+
void fpu_set_default_cw(struct lwp *, unsigned int);
void fputrap(struct trapframe *);
diff -r 0afef4bb2fff -r 49cfecfafb22 sys/arch/x86/x86/fpu.c
--- a/sys/arch/x86/x86/fpu.c Sat Jun 23 11:14:23 2018 +0000
+++ b/sys/arch/x86/x86/fpu.c Sat Jun 23 11:39:01 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: fpu.c,v 1.12.8.1 2017/12/21 19:33:15 snj Exp $ */
+/* $NetBSD: fpu.c,v 1.12.8.2 2018/06/23 11:39:02 martin Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc. All
@@ -96,7 +96,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.12.8.1 2017/12/21 19:33:15 snj Exp $");
+__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.12.8.2 2018/06/23 11:39:02 martin Exp $");
#include "opt_multiprocessor.h"
@@ -107,6 +107,8 @@
#include <sys/file.h>
#include <sys/proc.h>
#include <sys/kernel.h>
+#include <sys/sysctl.h>
+#include <sys/xcall.h>
#include <machine/cpu.h>
#include <machine/intr.h>
@@ -125,6 +127,8 @@
#define stts() HYPERVISOR_fpu_taskswitch(1)
#endif
+bool x86_fpu_eager __read_mostly = false;
+
static inline union savefpu *
process_fpframe(struct lwp *lwp)
{
@@ -228,6 +232,89 @@
}
static void
+fpu_clear_amd(void)
+{
+ /*
+ * AMD FPUs do not restore FIP, FDP, and FOP on fxrstor and xrstor
+ * when FSW.ES=0, leaking other threads' execution history.
+ *
+ * Clear them manually by loading a zero (fldummy). We do this
+ * unconditionally, regardless of FSW.ES.
+ *
+ * Before that, clear the ES bit in the x87 status word if it is
+ * currently set, in order to avoid causing a fault in the
+ * upcoming load.
+ *
+ * Newer generations of AMD CPUs have CPUID_Fn80000008_EBX[2],
+ * which indicates that FIP/FDP/FOP are restored (same behavior
+ * as Intel). We're not using it though.
+ */
+ if (fngetsw() & 0x80)
+ fnclex();
+ fldummy();
+}
+
+static void
+fpu_save(struct lwp *l)
+{
+ struct pcb *pcb = lwp_getpcb(l);
+
+ if (i386_use_fxsave) {
+ if (x86_xsave_features != 0)
+ xsave(&pcb->pcb_savefpu, x86_xsave_features);
+ else
+ fxsave(&pcb->pcb_savefpu);
+ } else {
+ fnsave(&pcb->pcb_savefpu);
+ }
+}
+
+static void
+fpu_restore(struct lwp *l)
+{
+ struct pcb *pcb = lwp_getpcb(l);
+
+ if (i386_use_fxsave) {
+ if (x86_xsave_features != 0) {
+ xrstor(&pcb->pcb_savefpu, x86_xsave_features);
+ } else {
+ fpu_clear_amd();
+ fxrstor(&pcb->pcb_savefpu);
+ }
+ } else {
+ frstor(&pcb->pcb_savefpu);
+ }
+}
+
+static void
+fpu_eagerrestore(struct lwp *l)
+{
+ struct pcb *pcb = lwp_getpcb(l);
+ struct cpu_info *ci = curcpu();
+
+ clts();
+ KASSERT(ci->ci_fpcurlwp == NULL);
+ KASSERT(pcb->pcb_fpcpu == NULL);
+ ci->ci_fpcurlwp = l;
+ pcb->pcb_fpcpu = ci;
+ fpu_restore(l);
+}
+
+void
+fpu_eagerswitch(struct lwp *oldlwp, struct lwp *newlwp)
+{
+ int s;
+
+ s = splhigh();
+ fpusave_cpu(true);
+ if (!(newlwp->l_flag & LW_SYSTEM))
+ fpu_eagerrestore(newlwp);
+ splx(s);
+}
+
+/* -------------------------------------------------------------------------- */
+
+static void
send_sigill(void *rip)
{
/* No fpu (486SX) - send SIGILL */
@@ -358,6 +445,11 @@
pcb = lwp_getpcb(l);
fl = ci->ci_fpcurlwp;
if (fl != NULL) {
+ if (__predict_false(x86_fpu_eager)) {
+ panic("%s: FPU busy with EagerFPU enabled",
+ __func__);
+ }
+
/*
* It seems we can get here on Xen even if we didn't
* switch lwp. In this case do nothing
@@ -373,6 +465,11 @@
/* Save our state if on a remote CPU. */
if (pcb->pcb_fpcpu != NULL) {
+ if (__predict_false(x86_fpu_eager)) {
+ panic("%s: LWP busy with EagerFPU enabled",
+ __func__);
+ }
+
/* Explicitly disable preemption before dropping spl. */
kpreempt_disable();
splx(s);
@@ -394,28 +491,7 @@
ci->ci_fpcurlwp = l;
pcb->pcb_fpcpu = ci;
- if (i386_use_fxsave) {
- if (x86_xsave_features != 0) {
- xrstor(&pcb->pcb_savefpu, x86_xsave_features);
- } else {
- /*
- * AMD FPU's do not restore FIP, FDP, and FOP on
- * fxrstor, leaking other process's execution history.
- * Clear them manually by loading a zero.
- *
- * Clear the ES bit in the x87 status word if it is
- * currently set, in order to avoid causing a fault
- * in the upcoming load.
- */
- if (fngetsw() & 0x80)
- fnclex();
- fldummy();
-
- fxrstor(&pcb->pcb_savefpu);
- }
- } else {
- frstor(&pcb->pcb_savefpu);
- }
+ fpu_restore(l);
KASSERT(ci == curcpu());
splx(s);
@@ -442,14 +518,7 @@
if (save) {
clts();
- if (i386_use_fxsave) {
- if (x86_xsave_features != 0)
- xsave(&pcb->pcb_savefpu, x86_xsave_features);
- else
- fxsave(&pcb->pcb_savefpu);
- } else {
- fnsave(&pcb->pcb_savefpu);
- }
+ fpu_save(l);
}
stts();
@@ -516,17 +585,28 @@
Home |
Main Index |
Thread Index |
Old Index