[src/trunk]: src/sys/arch/x86/x86 Need IPIs when enabling eager fpu switch, t...

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/fa542f35e5f9
branches:  trunk
changeset: 323481:fa542f35e5f9
user:      maxv <maxv%NetBSD.org@localhost>
date:      Sat Jun 16 17:11:13 2018 +0000

description:
Need IPIs when enabling eager fpu switch, to clear each fpu and get us
started. Otherwise it is possible that the first context switch on one of
the cpus will restore an invalid fpu state in the new lwp, if that lwp
had its fpu state stored on another cpu that didn't have time to do an
fpu save since eager-fpu was enabled.

Use barriers and all the related crap. The point is that we want to
ensure that no context switch occurs between [each fpu is cleared] and
[x86_fpu_eager is set to 'true'].

Also add KASSERTs.

diffstat:

 sys/arch/x86/x86/fpu.c         |  119 ++++++++++++++++++++++++++++++++++++++++-
 sys/arch/x86/x86/x86_machdep.c |   12 +--
 2 files changed, 121 insertions(+), 10 deletions(-)

diffs (191 lines):

diff -r 35d2c176c529 -r fa542f35e5f9 sys/arch/x86/x86/fpu.c
--- a/sys/arch/x86/x86/fpu.c    Sat Jun 16 15:18:33 2018 +0000
+++ b/sys/arch/x86/x86/fpu.c    Sat Jun 16 17:11:13 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: fpu.c,v 1.35 2018/06/16 05:52:17 maxv Exp $    */
+/*     $NetBSD: fpu.c,v 1.36 2018/06/16 17:11:13 maxv Exp $    */
 
 /*
  * Copyright (c) 2008 The NetBSD Foundation, Inc.  All
@@ -96,7 +96,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.35 2018/06/16 05:52:17 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.36 2018/06/16 17:11:13 maxv Exp $");
 
 #include "opt_multiprocessor.h"
 
@@ -107,6 +107,8 @@
 #include <sys/file.h>
 #include <sys/proc.h>
 #include <sys/kernel.h>
+#include <sys/sysctl.h>
+#include <sys/xcall.h>
 
 #include <machine/cpu.h>
 #include <machine/cpuvar.h>
@@ -339,6 +341,8 @@
        struct cpu_info *ci = curcpu();
 
        clts();
+       KASSERT(ci->ci_fpcurlwp == NULL);
+       KASSERT(pcb->pcb_fpcpu == NULL);
        ci->ci_fpcurlwp = l;
        pcb->pcb_fpcpu = ci;
        fpu_restore(l);
@@ -393,6 +397,10 @@
        if (!USERMODE(frame->tf_cs))
                panic("fpu trap from kernel, trapframe %p\n", frame);
 
+       if (__predict_false(x86_fpu_eager)) {
+               panic("%s: got DNA with EagerFPU enabled", __func__);
+       }
+
        /*
         * At this point, fpcurlwp should be curlwp.  If it wasn't, the TS bit
         * should be set, and we should have gotten a DNA exception.
@@ -890,3 +898,110 @@
                memcpy(fpregs, &fpu_save->sv_87, sizeof(fpu_save->sv_87));
        }
 }
+
+/* -------------------------------------------------------------------------- */
+
+static volatile unsigned long eagerfpu_cpu_barrier1 __cacheline_aligned;
+static volatile unsigned long eagerfpu_cpu_barrier2 __cacheline_aligned;
+
+static void
+eager_change_cpu(void *arg1, void *arg2)
+{
+       struct cpu_info *ci = curcpu();
+       bool enabled = (bool)arg1;
+       int s;
+
+       s = splhigh();
+
+       /* Rendez-vous 1. */
+       atomic_dec_ulong(&eagerfpu_cpu_barrier1);
+       while (atomic_cas_ulong(&eagerfpu_cpu_barrier1, 0, 0) != 0) {
+               x86_pause();
+       }
+
+       fpusave_cpu(true);
+       if (ci == &cpu_info_primary) {
+               x86_fpu_eager = enabled;
+       }
+
+       /* Rendez-vous 2. */
+       atomic_dec_ulong(&eagerfpu_cpu_barrier2);
+       while (atomic_cas_ulong(&eagerfpu_cpu_barrier2, 0, 0) != 0) {
+               x86_pause();
+       }
+
+       splx(s);
+}
+
+static int
+eager_change(bool enabled)
+{
+       struct cpu_info *ci = NULL;
+       CPU_INFO_ITERATOR cii;
+       uint64_t xc;
+
+       mutex_enter(&cpu_lock);
+
+       /*
+        * We expect all the CPUs to be online.
+        */
+       for (CPU_INFO_FOREACH(cii, ci)) {
+               struct schedstate_percpu *spc = &ci->ci_schedstate;
+               if (spc->spc_flags & SPCF_OFFLINE) {
+                       printf("[!] cpu%d offline, EagerFPU not changed\n",
+                           cpu_index(ci));
+                       mutex_exit(&cpu_lock);
+                       return EOPNOTSUPP;
+               }
+       }
+
+       /* Initialize the barriers */
+       eagerfpu_cpu_barrier1 = ncpu;
+       eagerfpu_cpu_barrier2 = ncpu;
+
+       printf("[+] %s EagerFPU...",
+           enabled ? "Enabling" : "Disabling");
+       xc = xc_broadcast(0, eager_change_cpu,
+           (void *)enabled, NULL);
+       xc_wait(xc);
+       printf(" done!\n");
+
+       mutex_exit(&cpu_lock);
+
+       return 0;
+}
+
+static int
+sysctl_machdep_fpu_eager(SYSCTLFN_ARGS)
+{
+       struct sysctlnode node;
+       int error;
+       bool val;
+
+       val = *(bool *)rnode->sysctl_data;
+
+       node = *rnode;
+       node.sysctl_data = &val;
+
+       error = sysctl_lookup(SYSCTLFN_CALL(&node));
+       if (error != 0 || newp == NULL)
+               return error;
+
+       if (val == x86_fpu_eager)
+               return 0;
+       return eager_change(val);
+}
+
+void sysctl_eagerfpu_init(struct sysctllog **);
+
+void
+sysctl_eagerfpu_init(struct sysctllog **clog)
+{
+       sysctl_createv(clog, 0, NULL, NULL,
+                      CTLFLAG_READWRITE,
+                      CTLTYPE_BOOL, "fpu_eager",
+                      SYSCTL_DESCR("Whether the kernel uses Eager FPU Switch"),
+                      sysctl_machdep_fpu_eager, 0,
+                      &x86_fpu_eager, 0,
+                      CTL_MACHDEP, CTL_CREATE, CTL_EOL);
+}
diff -r 35d2c176c529 -r fa542f35e5f9 sys/arch/x86/x86/x86_machdep.c
--- a/sys/arch/x86/x86/x86_machdep.c    Sat Jun 16 15:18:33 2018 +0000
+++ b/sys/arch/x86/x86/x86_machdep.c    Sat Jun 16 17:11:13 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: x86_machdep.c,v 1.116 2018/06/14 14:36:46 maxv Exp $   */
+/*     $NetBSD: x86_machdep.c,v 1.117 2018/06/16 17:11:13 maxv Exp $   */
 
 /*-
  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.116 2018/06/14 14:36:46 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.117 2018/06/16 17:11:13 maxv Exp $");
 
 #include "opt_modular.h"
 #include "opt_physmem.h"
@@ -1277,12 +1277,8 @@
 #endif
 
 #ifndef XEN
-       sysctl_createv(clog, 0, NULL, NULL,
-                      CTLFLAG_READWRITE,
-                      CTLTYPE_BOOL, "fpu_eager",
-                      SYSCTL_DESCR("Whether the kernel uses Eager FPU Switch"),
-                      NULL, 0, &x86_fpu_eager, 0,
-                      CTL_MACHDEP, CTL_CREATE, CTL_EOL);
+       void sysctl_eagerfpu_init(struct sysctllog **);
+       sysctl_eagerfpu_init(clog);
 #endif
 
        /* None of these can ever change once the system has booted */