[src/trunk]: src/sys/arch/amd64/amd64 There is a bug in the way the secondary...

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/4fe9c41b1cdc
branches:  trunk
changeset: 345180:4fe9c41b1cdc
user:      maxv <maxv%NetBSD.org@localhost>
date:      Wed May 11 19:35:08 2016 +0000

description:
There is a bug in the way the secondary CPUs are launched on amd64.

When CPU0 is launched, EFER_NXE is enabled in it, and it allows it to
handle pages that have the NOX bit. When the secondary CPUs are
launched, however, EFER_NXE is enabled only after paging is set in their
%cr0. And therefore, between the moment when paging is enabled and the
moment when EFER_NXE is enabled, the secondary CPUs cannot access pages
that have the NOX bit - they crash if they try to.

The funny thing is that in order to enable EFER_NXE, the secondary CPUs
give a look at cpu_feature[2], which is in the DATA segment, which in
turn could have the NOX bit. In other words, the secondary CPUs crash if
the DATA segment is mapped with the NOX bit.

Fix this by enabling EFER_NXE in the secondary CPUs before enabling
paging. CPU0 initializes nox_flag to the 32bit version of PG_NX if NOX
is supported; the secondary CPUs then use nox_flag to know whether NOX
is supported.

nox_flag will be used for other purposes soon.

diffstat:

 sys/arch/amd64/amd64/locore.S  |  28 +++++++++++++++++++++++++---
 sys/arch/amd64/amd64/mptramp.S |  22 ++++++++++------------
 2 files changed, 35 insertions(+), 15 deletions(-)

diffs (122 lines):

diff -r bc1e0c7a7374 -r 4fe9c41b1cdc sys/arch/amd64/amd64/locore.S
--- a/sys/arch/amd64/amd64/locore.S     Wed May 11 18:33:40 2016 +0000
+++ b/sys/arch/amd64/amd64/locore.S     Wed May 11 19:35:08 2016 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: locore.S,v 1.85 2016/05/08 08:22:58 maxv Exp $ */
+/*     $NetBSD: locore.S,v 1.86 2016/05/11 19:35:08 maxv Exp $ */
 
 /*
  * Copyright-o-rama!
@@ -187,6 +187,9 @@
 #define        _RELOC(x)       ((x) - KERNBASE)
 #define        RELOC(x)        _RELOC(_C_LABEL(x))
 
+/* 32bit version of PG_NX */
+#define PG_NX32        0x80000000
+
 #if L2_SLOT_KERNBASE > 0
 #define TABLE_L2_ENTRIES (2 * (NKL2_KIMG_ENTRIES + 1))
 #else
@@ -286,6 +289,7 @@
 END(lapic_isr)
 #endif /* NLAPIC > 0 */
 
+       .globl  _C_LABEL(nox_flag)
        .globl  _C_LABEL(cpuid_level)
        .globl  _C_LABEL(esym)
        .globl  _C_LABEL(eblob)
@@ -298,6 +302,9 @@
        .globl  _C_LABEL(gdtstore)
        .globl  _C_LABEL(cputype)
 
+       .type   _C_LABEL(nox_flag), @object
+LABEL(nox_flag)                .long   0       /* 32bit NOX flag, set if supported */
+END(nox_flag)
        .type   _C_LABEL(cputype), @object
 LABEL(cputype)         .long   0       /* are we 80486, Pentium, or.. */
 END(cputype)
@@ -527,6 +534,16 @@
         */
        movl    $RELOC(tmpstk),%esp
 
+       /*
+        * Retrieve the NX/XD flag. We use the 32bit version of PG_NX.
+        */
+       movl    $0x80000001,%eax
+       cpuid
+       andl    $CPUID_NOX,%edx
+       jz      no_NOX
+       movl    $PG_NX32,RELOC(nox_flag)
+no_NOX:
+
 /*
  * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will
  * be referred to as: L4 -> L3 -> L2 -> L1.
@@ -707,13 +724,18 @@
        movl    %eax,%cr4
 
        /*
-        * 2. Set Long Mode Enable in EFER. Also enable the
-        *    syscall extensions.
+        * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions,
+        *    and NOX if available.
         */
        movl    $MSR_EFER,%ecx
        rdmsr
        xorl    %eax,%eax       /* XXX */
        orl     $(EFER_LME|EFER_SCE),%eax
+       movl    RELOC(nox_flag),%ebx
+       cmpl    $0,%ebx
+       je      skip_NOX
+       orl     $(EFER_NXE),%eax
+skip_NOX:
        wrmsr
 
        /*
diff -r bc1e0c7a7374 -r 4fe9c41b1cdc sys/arch/amd64/amd64/mptramp.S
--- a/sys/arch/amd64/amd64/mptramp.S    Wed May 11 18:33:40 2016 +0000
+++ b/sys/arch/amd64/amd64/mptramp.S    Wed May 11 19:35:08 2016 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: mptramp.S,v 1.20 2016/05/07 13:08:30 maxv Exp $        */
+/*     $NetBSD: mptramp.S,v 1.21 2016/05/11 19:35:08 maxv Exp $        */
 
 /*-
  * Copyright (c) 2000, 2016 The NetBSD Foundation, Inc.
@@ -170,10 +170,19 @@
 no_PSE:
        movl    %eax,%cr4
 
+       /*
+        * Set Long Mode Enable in EFER. Also enable the syscall extensions,
+        * and NOX if available.
+        */
        movl    $MSR_EFER,%ecx
        rdmsr
        xorl    %eax,%eax
        orl     $(EFER_LME|EFER_SCE),%eax
+       movl    RELOC(nox_flag),%ebx
+       cmpl    $0,%ebx
+       je      no_NOX
+       orl     $(EFER_NXE),%eax
+no_NOX:
        wrmsr
 
        /* Load %cr3. */
@@ -227,17 +236,6 @@
 
 
 _C_LABEL(cpu_spinup_trampoline_end):   /* end of code copied to MP_TRAMPOLINE */
-       /*
-        * If EFER_NXE is not enabled, fetching a page with a NX bit set
-        * will raise a #GP. Avoid that by setting the NXE feature now.
-        */
-       movl    _C_LABEL(cpu_feature)+2*4,%eax  /* cpu_feature[2] */
-       andl    $CPUID_NOX,%eax
-       jz      1f
-       movl    $MSR_EFER,%ecx
-       rdmsr
-       orl     $EFER_NXE,%eax  /* enable No-Execute feature */
-       wrmsr
 
 1:
        /* Don't touch lapic until BP has done init sequence. */