Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/arch/x86/x86 Use the hotpatch framework for LFENCE/MFENCE.
details: https://anonhg.NetBSD.org/src/rev/6809c7cec654
branches: trunk
changeset: 931322:6809c7cec654
user: maxv <maxv%NetBSD.org@localhost>
date: Sun Apr 26 14:49:17 2020 +0000
description:
Use the hotpatch framework for LFENCE/MFENCE.
diffstat:
common/lib/libc/arch/i386/atomic/atomic.S | 50 ++++++++++++++++------------
common/lib/libc/arch/x86_64/atomic/atomic.S | 48 +++++++++++++++------------
sys/arch/amd64/include/frameasm.h | 4 +-
sys/arch/i386/include/frameasm.h | 4 +-
sys/arch/x86/x86/patch.c | 35 ++++++++------------
5 files changed, 76 insertions(+), 65 deletions(-)
diffs (286 lines):
diff -r 5ec19c3748c0 -r 6809c7cec654 common/lib/libc/arch/i386/atomic/atomic.S
--- a/common/lib/libc/arch/i386/atomic/atomic.S Sun Apr 26 14:21:14 2020 +0000
+++ b/common/lib/libc/arch/i386/atomic/atomic.S Sun Apr 26 14:49:17 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic.S,v 1.27 2020/04/26 13:59:44 maxv Exp $ */
+/* $NetBSD: atomic.S,v 1.28 2020/04/26 14:49:17 maxv Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -45,11 +45,15 @@
#ifdef _HARDKERNEL
#include "opt_xen.h"
#include <machine/frameasm.h>
-#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
-#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a)
+#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
+#define HOTPATCH_SSE2_LFENCE HOTPATCH(HP_NAME_SSE2_LFENCE, 7);
+#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 7);
+#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a)
#else
-#define LOCK lock
-#define ENDLABEL(a) /* nothing */
+#define LOCK lock
+#define HOTPATCH_SSE2_LFENCE /* nothing */
+#define HOTPATCH_SSE2_MFENCE /* nothing */
+#define ENDLABEL(a) /* nothing */
#endif
.text
@@ -177,11 +181,12 @@
END(_atomic_cas_32_ni)
ENTRY(_membar_consumer)
+ HOTPATCH_SSE2_LFENCE
+ /* 7 bytes of instructions */
LOCK
addl $0, -4(%esp)
ret
END(_membar_consumer)
-ENDLABEL(membar_consumer_end)
ENTRY(_membar_producer)
/* A store is enough */
@@ -190,11 +195,12 @@
END(_membar_producer)
ENTRY(_membar_sync)
+ HOTPATCH_SSE2_MFENCE
+ /* 7 bytes of instructions */
LOCK
addl $0, -4(%esp)
ret
END(_membar_sync)
-ENDLABEL(membar_sync_end)
#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL)
#ifdef XENPV
@@ -257,20 +263,6 @@
ENDLABEL(_atomic_cas_cx8_end)
#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */
-#ifdef _HARDKERNEL
-ENTRY(sse2_lfence)
- lfence
- ret
-END(sse2_lfence)
-ENDLABEL(sse2_lfence_end)
-
-ENTRY(sse2_mfence)
- mfence
- ret
-END(sse2_mfence)
-ENDLABEL(sse2_mfence_end)
-#endif /* _HARDKERNEL */
-
ALIAS(atomic_add_32,_atomic_add_32)
ALIAS(atomic_add_int,_atomic_add_32)
ALIAS(atomic_add_long,_atomic_add_32)
@@ -402,3 +394,19 @@
STRONG_ALIAS(_membar_enter,_membar_consumer)
STRONG_ALIAS(_membar_exit,_membar_producer)
+
+#ifdef _HARDKERNEL
+ .section .rodata
+
+LABEL(sse2_lfence)
+ lfence
+ ret
+ nop; nop; nop;
+LABEL(sse2_lfence_end)
+
+LABEL(sse2_mfence)
+ mfence
+ ret
+ nop; nop; nop;
+LABEL(sse2_mfence_end)
+#endif /* _HARDKERNEL */
diff -r 5ec19c3748c0 -r 6809c7cec654 common/lib/libc/arch/x86_64/atomic/atomic.S
--- a/common/lib/libc/arch/x86_64/atomic/atomic.S Sun Apr 26 14:21:14 2020 +0000
+++ b/common/lib/libc/arch/x86_64/atomic/atomic.S Sun Apr 26 14:49:17 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic.S,v 1.21 2020/04/26 13:59:44 maxv Exp $ */
+/* $NetBSD: atomic.S,v 1.22 2020/04/26 14:49:17 maxv Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -40,11 +40,13 @@
#ifdef _HARDKERNEL
#include <machine/frameasm.h>
-#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
-#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a)
+#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
+#define HOTPATCH_SSE2_LFENCE HOTPATCH(HP_NAME_SSE2_LFENCE, 8);
+#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 8);
#else
-#define LOCK lock
-#define ENDLABEL(a) /* nothing */
+#define LOCK lock
+#define HOTPATCH_SSE2_LFENCE /* nothing */
+#define HOTPATCH_SSE2_MFENCE /* nothing */
#endif
.text
@@ -254,11 +256,12 @@
/* memory barriers */
ENTRY(_membar_consumer)
+ HOTPATCH_SSE2_LFENCE
+ /* 8 bytes of instructions */
LOCK
addq $0, -8(%rsp)
ret
END(_membar_consumer)
-ENDLABEL(membar_consumer_end)
ENTRY(_membar_producer)
/* A store is enough */
@@ -267,25 +270,12 @@
END(_membar_producer)
ENTRY(_membar_sync)
+ HOTPATCH_SSE2_MFENCE
+ /* 8 bytes of instructions */
LOCK
addq $0, -8(%rsp)
ret
END(_membar_sync)
-ENDLABEL(membar_sync_end)
-
-#ifdef _HARDKERNEL
-ENTRY(sse2_lfence)
- lfence
- ret
-END(sse2_lfence)
-ENDLABEL(sse2_lfence_end)
-
-ENTRY(sse2_mfence)
- mfence
- ret
-END(sse2_mfence)
-ENDLABEL(sse2_mfence_end)
-#endif /* _HARDKERNEL */
ALIAS(atomic_add_32,_atomic_add_32)
ALIAS(atomic_add_64,_atomic_add_64)
@@ -425,3 +415,19 @@
STRONG_ALIAS(_membar_enter,_membar_consumer)
STRONG_ALIAS(_membar_exit,_membar_producer)
+
+#ifdef _HARDKERNEL
+ .section .rodata
+
+LABEL(sse2_lfence)
+ lfence
+ ret
+ nop; nop; nop; nop;
+LABEL(sse2_lfence_end)
+
+LABEL(sse2_mfence)
+ mfence
+ ret
+ nop; nop; nop; nop;
+LABEL(sse2_mfence_end)
+#endif /* _HARDKERNEL */
diff -r 5ec19c3748c0 -r 6809c7cec654 sys/arch/amd64/include/frameasm.h
--- a/sys/arch/amd64/include/frameasm.h Sun Apr 26 14:21:14 2020 +0000
+++ b/sys/arch/amd64/include/frameasm.h Sun Apr 26 14:49:17 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.48 2020/04/25 15:26:16 bouyer Exp $ */
+/* $NetBSD: frameasm.h,v 1.49 2020/04/26 14:49:17 maxv Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -63,6 +63,8 @@
#define HP_NAME_SVS_ENTER_NMI 11
#define HP_NAME_SVS_LEAVE_NMI 12
#define HP_NAME_MDS_LEAVE 13
+#define HP_NAME_SSE2_LFENCE 14
+#define HP_NAME_SSE2_MFENCE 15
#define HOTPATCH(name, size) \
123: ; \
diff -r 5ec19c3748c0 -r 6809c7cec654 sys/arch/i386/include/frameasm.h
--- a/sys/arch/i386/include/frameasm.h Sun Apr 26 14:21:14 2020 +0000
+++ b/sys/arch/i386/include/frameasm.h Sun Apr 26 14:49:17 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.30 2020/04/25 15:26:17 bouyer Exp $ */
+/* $NetBSD: frameasm.h,v 1.31 2020/04/26 14:49:17 maxv Exp $ */
#ifndef _I386_FRAMEASM_H_
#define _I386_FRAMEASM_H_
@@ -48,6 +48,8 @@
#define HP_NAME_STAC 2
#define HP_NAME_NOLOCK 3
#define HP_NAME_RETFENCE 4
+#define HP_NAME_SSE2_LFENCE 5
+#define HP_NAME_SSE2_MFENCE 6
#define HOTPATCH(name, size) \
123: ; \
diff -r 5ec19c3748c0 -r 6809c7cec654 sys/arch/x86/x86/patch.c
--- a/sys/arch/x86/x86/patch.c Sun Apr 26 14:21:14 2020 +0000
+++ b/sys/arch/x86/x86/patch.c Sun Apr 26 14:49:17 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: patch.c,v 1.41 2020/04/26 13:37:14 maxv Exp $ */
+/* $NetBSD: patch.c,v 1.42 2020/04/26 14:49:17 maxv Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.41 2020/04/26 13:37:14 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.42 2020/04/26 14:49:17 maxv Exp $");
#include "opt_lockdebug.h"
#ifdef i386
@@ -69,15 +69,6 @@
void i686_mutex_spin_exit_end(void);
void i686_mutex_spin_exit_patch(void);
-void membar_consumer(void);
-void membar_consumer_end(void);
-void membar_sync(void);
-void membar_sync_end(void);
-void sse2_lfence(void);
-void sse2_lfence_end(void);
-void sse2_mfence(void);
-void sse2_mfence_end(void);
-
void _atomic_cas_64(void);
void _atomic_cas_64_end(void);
void _atomic_cas_cx8(void);
@@ -219,16 +210,18 @@
* ordinary non-temporal stores are always issued in
* program order to main memory and to other CPUs.
*/
- patchfunc(
- sse2_lfence, sse2_lfence_end,
- membar_consumer, membar_consumer_end,
- NULL
- );
- patchfunc(
- sse2_mfence, sse2_mfence_end,
- membar_sync, membar_sync_end,
- NULL
- );
+ extern uint8_t sse2_lfence, sse2_lfence_end;
+ extern uint8_t sse2_mfence, sse2_mfence_end;
+ uint8_t *bytes;
+ size_t size;
+
+ bytes = &sse2_lfence;
+ size = (size_t)&sse2_lfence_end - (size_t)&sse2_lfence;
+ x86_hotpatch(HP_NAME_SSE2_LFENCE, bytes, size);
+
+ bytes = &sse2_mfence;
+ size = (size_t)&sse2_mfence_end - (size_t)&sse2_mfence;
+ x86_hotpatch(HP_NAME_SSE2_MFENCE, bytes, size);
}
#ifdef i386
Home |
Main Index |
Thread Index |
Old Index