Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys Rewrite the FPU code on x86. This greatly simplifies the...
details: https://anonhg.NetBSD.org/src/rev/a108cb5a3496
branches: trunk
changeset: 455254:a108cb5a3496
user: maxv <maxv%NetBSD.org@localhost>
date: Sat Oct 12 06:31:03 2019 +0000
description:
Rewrite the FPU code on x86. This greatly simplifies the logic and removes
the dependency on IPL_HIGH. NVMM is updated accordingly. Posted on
port-amd64 a week ago.
Bump the kernel version to 9.99.16.
diffstat:
sys/arch/amd64/amd64/amd64_trap.S | 3 +-
sys/arch/amd64/amd64/genassym.cf | 5 +-
sys/arch/amd64/amd64/locore.S | 27 +--
sys/arch/amd64/amd64/machdep.c | 15 +-
sys/arch/amd64/amd64/spl.S | 3 +-
sys/arch/amd64/include/frameasm.h | 16 ++-
sys/arch/amd64/include/pcb.h | 3 +-
sys/arch/amd64/include/proc.h | 3 +-
sys/arch/i386/i386/genassym.cf | 7 +-
sys/arch/i386/i386/i386_trap.S | 15 +-
sys/arch/i386/i386/locore.S | 29 +---
sys/arch/i386/i386/machdep.c | 16 +-
sys/arch/i386/i386/spl.S | 5 +-
sys/arch/i386/include/frameasm.h | 18 ++-
sys/arch/i386/include/pcb.h | 3 +-
sys/arch/i386/include/proc.h | 4 +-
sys/arch/x86/acpi/acpi_wakeup.c | 8 +-
sys/arch/x86/include/cpu.h | 3 +-
sys/arch/x86/include/fpu.h | 8 +-
sys/arch/x86/x86/cpu.c | 17 +-
sys/arch/x86/x86/fpu.c | 279 ++++++++++++-------------------------
sys/arch/x86/x86/ipi.c | 6 +-
sys/arch/x86/x86/vm_machdep.c | 21 +--
sys/arch/xen/x86/cpu.c | 17 +-
sys/arch/xen/x86/xen_ipi.c | 8 +-
sys/dev/nvmm/x86/nvmm_x86_svm.c | 22 +-
sys/dev/nvmm/x86/nvmm_x86_vmx.c | 22 +-
sys/sys/param.h | 4 +-
28 files changed, 225 insertions(+), 362 deletions(-)
diffs (truncated from 1444 to 300 lines):
diff -r c6a2de3f633f -r a108cb5a3496 sys/arch/amd64/amd64/amd64_trap.S
--- a/sys/arch/amd64/amd64/amd64_trap.S Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/amd64/amd64_trap.S Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: amd64_trap.S,v 1.48 2019/05/18 13:32:12 maxv Exp $ */
+/* $NetBSD: amd64_trap.S,v 1.49 2019/10/12 06:31:03 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
@@ -672,6 +672,7 @@
jmp .Lalltraps_checkast /* re-check ASTs */
3: CHECK_DEFERRED_SWITCH
jnz 9f
+ HANDLE_DEFERRED_FPU
6:
#ifdef DIAGNOSTIC
diff -r c6a2de3f633f -r a108cb5a3496 sys/arch/amd64/amd64/genassym.cf
--- a/sys/arch/amd64/amd64/genassym.cf Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/amd64/genassym.cf Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.76 2019/05/29 16:54:41 maxv Exp $
+# $NetBSD: genassym.cf,v 1.77 2019/10/12 06:31:03 maxv Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -164,6 +164,7 @@
define MDL_IRET MDL_IRET
define MDL_COMPAT32 MDL_COMPAT32
+define MDL_FPU_IN_CPU MDL_FPU_IN_CPU
define P_FLAG offsetof(struct proc, p_flag)
define P_RASLIST offsetof(struct proc, p_raslist)
@@ -188,6 +189,7 @@
define PCB_COMPAT32 PCB_COMPAT32
define PCB_FS offsetof(struct pcb, pcb_fs)
define PCB_GS offsetof(struct pcb, pcb_gs)
+define PCB_SAVEFPU offsetof(struct pcb, pcb_savefpu)
define TF_RDI offsetof(struct trapframe, tf_rdi)
define TF_RSI offsetof(struct trapframe, tf_rsi)
@@ -244,7 +246,6 @@
define CPU_INFO_NTRAP offsetof(struct cpu_info, ci_data.cpu_ntrap)
define CPU_INFO_NINTR offsetof(struct cpu_info, ci_data.cpu_nintr)
define CPU_INFO_CURPRIORITY offsetof(struct cpu_info, ci_schedstate.spc_curpriority)
-define CPU_INFO_FPCURLWP offsetof(struct cpu_info, ci_fpcurlwp)
define CPU_INFO_GDT offsetof(struct cpu_info, ci_gdt)
define CPU_INFO_ILEVEL offsetof(struct cpu_info, ci_ilevel)
diff -r c6a2de3f633f -r a108cb5a3496 sys/arch/amd64/amd64/locore.S
--- a/sys/arch/amd64/amd64/locore.S Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/amd64/locore.S Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.188 2019/10/04 11:47:07 maxv Exp $ */
+/* $NetBSD: locore.S,v 1.189 2019/10/12 06:31:03 maxv Exp $ */
/*
* Copyright-o-rama!
@@ -1170,29 +1170,10 @@
movq %rax,TF_RIP(%rbx)
.Lno_RAS:
- /*
- * Restore cr0 including FPU state (may have CR0_TS set). Note that
- * IPL_SCHED prevents from FPU interrupt altering the LWP's saved cr0.
- */
#ifndef XENPV
+ /* Raise the IPL to IPL_HIGH. Dropping the priority is deferred until
+ * mi_switch(), when cpu_switchto() returns. XXX Still needed? */
movl $IPL_HIGH,CPUVAR(ILEVEL)
- movl PCB_CR0(%r14),%ecx /* has CR0_TS clear */
- movq %cr0,%rdx
-
- /*
- * If our floating point registers are on a different CPU,
- * set CR0_TS so we'll trap rather than reuse bogus state.
- */
- cmpq CPUVAR(FPCURLWP),%r12
- je .Lskip_TS
- orq $CR0_TS,%rcx
-.Lskip_TS:
-
- /* Reloading CR0 is very expensive - avoid if possible. */
- cmpq %rdx,%rcx
- je .Lskip_CR0
- movq %rcx,%cr0
-.Lskip_CR0:
/* The 32bit LWPs are handled differently. */
testl $PCB_COMPAT32,PCB_FLAGS(%r14)
@@ -1305,6 +1286,8 @@
jne .Lspl_error
#endif
+ HANDLE_DEFERRED_FPU
+
/*
* Decide if we need to take a slow path. That's the case when we
* want to reload %cs and %ss on a 64bit LWP (MDL_IRET set), or when
diff -r c6a2de3f633f -r a108cb5a3496 sys/arch/amd64/amd64/machdep.c
--- a/sys/arch/amd64/amd64/machdep.c Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/amd64/machdep.c Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.336 2019/08/21 20:30:36 skrll Exp $ */
+/* $NetBSD: machdep.c,v 1.337 2019/10/12 06:31:03 maxv Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.336 2019/08/21 20:30:36 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.337 2019/10/12 06:31:03 maxv Exp $");
#include "opt_modular.h"
#include "opt_user_ldt.h"
@@ -442,18 +442,9 @@
uint64_t zero = 0;
/*
- * Raise the IPL to IPL_HIGH.
- * FPU IPIs can alter the LWP's saved cr0. Dropping the priority
- * is deferred until mi_switch(), when cpu_switchto() returns.
+ * Raise the IPL to IPL_HIGH. XXX Still needed?
*/
(void)splhigh();
- /*
- * If our floating point registers are on a different CPU,
- * set CR0_TS so we'll trap rather than reuse bogus state.
- */
- if (l != ci->ci_fpcurlwp) {
- HYPERVISOR_fpu_taskswitch(1);
- }
/* Update segment registers */
if (pcb->pcb_flags & PCB_COMPAT32) {
diff -r c6a2de3f633f -r a108cb5a3496 sys/arch/amd64/amd64/spl.S
--- a/sys/arch/amd64/amd64/spl.S Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/amd64/spl.S Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: spl.S,v 1.40 2019/02/14 08:18:25 cherry Exp $ */
+/* $NetBSD: spl.S,v 1.41 2019/10/12 06:31:03 maxv Exp $ */
/*
* Copyright (c) 2003 Wasabi Systems, Inc.
@@ -397,6 +397,7 @@
3:
CHECK_DEFERRED_SWITCH
jnz 9f
+ HANDLE_DEFERRED_FPU
6:
INTRFASTEXIT
9:
diff -r c6a2de3f633f -r a108cb5a3496 sys/arch/amd64/include/frameasm.h
--- a/sys/arch/amd64/include/frameasm.h Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/include/frameasm.h Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.44 2019/05/18 13:32:12 maxv Exp $ */
+/* $NetBSD: frameasm.h,v 1.45 2019/10/12 06:31:03 maxv Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -246,4 +246,18 @@
#define CHECK_ASTPENDING(reg) cmpl $0, L_MD_ASTPENDING(reg)
#define CLEAR_ASTPENDING(reg) movl $0, L_MD_ASTPENDING(reg)
+/*
+ * If the FPU state is not in the CPU, restore it. Executed with interrupts
+ * disabled.
+ *
+ * %r14 is curlwp, must not be modified
+ * %rbx must not be modified
+ */
+#define HANDLE_DEFERRED_FPU \
+ testl $MDL_FPU_IN_CPU,L_MD_FLAGS(%r14) ; \
+ jnz 1f ; \
+ call _C_LABEL(fpu_handle_deferred) ; \
+ orl $MDL_FPU_IN_CPU,L_MD_FLAGS(%r14) ; \
+1:
+
#endif /* _AMD64_MACHINE_FRAMEASM_H */
diff -r c6a2de3f633f -r a108cb5a3496 sys/arch/amd64/include/pcb.h
--- a/sys/arch/amd64/include/pcb.h Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/include/pcb.h Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pcb.h,v 1.29 2018/07/26 09:29:08 maxv Exp $ */
+/* $NetBSD: pcb.h,v 1.30 2019/10/12 06:31:03 maxv Exp $ */
/*
* Copyright (c) 1998 The NetBSD Foundation, Inc.
@@ -96,7 +96,6 @@
uint32_t pcb_unused[8]; /* unused */
- struct cpu_info *pcb_fpcpu; /* cpu holding our fp state. */
union savefpu pcb_savefpu __aligned(64); /* floating point state */
/* **** DO NOT ADD ANYTHING HERE **** */
};
diff -r c6a2de3f633f -r a108cb5a3496 sys/arch/amd64/include/proc.h
--- a/sys/arch/amd64/include/proc.h Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/include/proc.h Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: proc.h,v 1.22 2017/02/25 13:34:21 kamil Exp $ */
+/* $NetBSD: proc.h,v 1.23 2019/10/12 06:31:03 maxv Exp $ */
/*
* Copyright (c) 1991 Regents of the University of California.
@@ -55,6 +55,7 @@
#define MDL_COMPAT32 0x0008 /* i386, always return via iret */
#define MDL_IRET 0x0010 /* force return via iret, not sysret */
+#define MDL_FPU_IN_CPU 0x0020 /* the FPU state is in the CPU */
struct mdproc {
int md_flags;
diff -r c6a2de3f633f -r a108cb5a3496 sys/arch/i386/i386/genassym.cf
--- a/sys/arch/i386/i386/genassym.cf Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/i386/i386/genassym.cf Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.113 2019/03/09 08:42:25 maxv Exp $
+# $NetBSD: genassym.cf,v 1.114 2019/10/12 06:31:03 maxv Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -165,6 +165,7 @@
define L_FLAG offsetof(struct lwp, l_flag)
define L_PROC offsetof(struct lwp, l_proc)
define L_MD_REGS offsetof(struct lwp, l_md.md_regs)
+define L_MD_FLAGS offsetof(struct lwp, l_md.md_flags)
define L_CTXSWTCH offsetof(struct lwp, l_ctxswtch)
define L_MD_ASTPENDING offsetof(struct lwp, l_md.md_astpending)
define L_CPU offsetof(struct lwp, l_cpu)
@@ -176,6 +177,8 @@
define P_RASLIST offsetof(struct proc, p_raslist)
define P_MD_SYSCALL offsetof(struct proc, p_md.md_syscall)
+define MDL_FPU_IN_CPU MDL_FPU_IN_CPU
+
define LW_SYSTEM LW_SYSTEM
define GUFS_SEL GUFS_SEL
@@ -200,6 +203,7 @@
define PCB_FSD offsetof(struct pcb, pcb_fsd)
define PCB_GSD offsetof(struct pcb, pcb_gsd)
define PCB_IOMAP offsetof(struct pcb, pcb_iomap)
+define PCB_SAVEFPU offsetof(struct pcb, pcb_savefpu)
define TF_CS offsetof(struct trapframe, tf_cs)
define TF_EIP offsetof(struct trapframe, tf_eip)
@@ -251,7 +255,6 @@
define CPU_INFO_TLBSTATE offsetof(struct cpu_info, ci_tlbstate)
define TLBSTATE_VALID TLBSTATE_VALID
define CPU_INFO_CURLWP offsetof(struct cpu_info, ci_curlwp)
-define CPU_INFO_FPCURLWP offsetof(struct cpu_info, ci_fpcurlwp)
define CPU_INFO_CURLDT offsetof(struct cpu_info, ci_curldt)
define CPU_INFO_IDLELWP offsetof(struct cpu_info, ci_data.cpu_idlelwp)
define CPU_INFO_PMAP offsetof(struct cpu_info, ci_pmap)
diff -r c6a2de3f633f -r a108cb5a3496 sys/arch/i386/i386/i386_trap.S
--- a/sys/arch/i386/i386/i386_trap.S Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/i386/i386/i386_trap.S Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: i386_trap.S,v 1.19 2019/10/04 15:28:00 maxv Exp $ */
+/* $NetBSD: i386_trap.S,v 1.20 2019/10/12 06:31:03 maxv Exp $ */
/*
* Copyright 2002 (c) Wasabi Systems, Inc.
@@ -66,7 +66,7 @@
#if 0
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: i386_trap.S,v 1.19 2019/10/04 15:28:00 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: i386_trap.S,v 1.20 2019/10/12 06:31:03 maxv Exp $");
#endif
/*
@@ -436,10 +436,10 @@
#ifdef XEN
STIC(%eax)
- jz 6f
+ jz 22f
call _C_LABEL(stipending)
testl %eax,%eax
- jz 6f
+ jz 22f
/* process pending interrupts */
CLI(%eax)
movl CPUVAR(ILEVEL),%ebx
@@ -448,15 +448,18 @@
movl %ebx,%eax /* get cpl */
movl CPUVAR(XUNMASK)(,%eax,4),%eax
andl CPUVAR(XPENDING),%eax /* any non-masked bits left? */
- jz 7f
+ jz 11f
bsrl %eax,%eax
btrl %eax,CPUVAR(XPENDING)
movl CPUVAR(XSOURCES)(,%eax,4),%eax
jmp *IS_RESUME(%eax)
Home |
Main Index |
Thread Index |
Old Index