tech-kern archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
__read_mostly and __mp_friendly annotations
Hi guys,
below is a patch that introduces __read_mostly and __mp_friendly
annotations. These annotations allow infrequently modified data to be
grouped together and allow aligning data on cacheline boundaries. This
helps to mitigate false-sharing.
This patch has been passed around for about a year now so it's pretty
well reviewed but I just wanted to post it here in case anyone had any
comments.
I'll do another patch that uses these annotations soon.
Index: sys/arch/amd64/conf/kern.ldscript
===================================================================
RCS file: /cvsroot/src/sys/arch/amd64/conf/kern.ldscript,v
retrieving revision 1.4
diff -u -p -u -r1.4 kern.ldscript
--- sys/arch/amd64/conf/kern.ldscript 18 Oct 2007 15:28:33 -0000 1.4
+++ sys/arch/amd64/conf/kern.ldscript 21 May 2010 18:57:41 -0000
@@ -32,8 +32,22 @@ SECTIONS
AT (LOADADDR(.text) + (ADDR(.data) - ADDR(.text)))
{
*(.data)
- *(.data.*)
}
+
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.cacheline_aligned :
+ AT (LOADADDR(.text) + (ADDR(.data.cacheline_aligned) - ADDR(.text)))
+ {
+ *(.data.cacheline_aligned)
+ }
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.read_mostly :
+ AT (LOADADDR(.text) + (ADDR(.data.read_mostly) - ADDR(.text)))
+ {
+ *(.data.read_mostly)
+ }
+ . = ALIGN(64); /* COHERENCY_UNIT */
+
_edata = . ;
PROVIDE (edata = .) ;
__bss_start = . ;
Index: sys/arch/amd64/conf/kern.ldscript.2MB
===================================================================
RCS file: /cvsroot/src/sys/arch/amd64/conf/kern.ldscript.2MB,v
retrieving revision 1.2
diff -u -p -u -r1.2 kern.ldscript.2MB
--- sys/arch/amd64/conf/kern.ldscript.2MB 18 Oct 2007 15:28:33 -0000
1.2
+++ sys/arch/amd64/conf/kern.ldscript.2MB 21 May 2010 18:57:41 -0000
@@ -33,8 +33,20 @@ SECTIONS
AT (LOADADDR(.text) + (ADDR(.data) - ADDR(.text)))
{
*(.data)
- *(.data.*)
}
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.cacheline_aligned :
+ AT (LOADADDR(.text) + (ADDR(.data.cacheline_aligned) - ADDR(.text)))
+ {
+ *(.data.cacheline_aligned)
+ }
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.read_mostly :
+ AT (LOADADDR(.text) + (ADDR(.data.read_mostly) - ADDR(.text)))
+ {
+ *(.data.read_mostly)
+ }
+ . = ALIGN(64); /* COHERENCY_UNIT */
_edata = . ;
PROVIDE (edata = .) ;
__bss_start = . ;
Index: sys/arch/amd64/conf/kern.ldscript.Xen
===================================================================
RCS file: /cvsroot/src/sys/arch/amd64/conf/kern.ldscript.Xen,v
retrieving revision 1.2
diff -u -p -u -r1.2 kern.ldscript.Xen
--- sys/arch/amd64/conf/kern.ldscript.Xen 22 Nov 2007 16:16:44 -0000
1.2
+++ sys/arch/amd64/conf/kern.ldscript.Xen 21 May 2010 18:57:41 -0000
@@ -26,8 +26,20 @@ SECTIONS
AT (LOADADDR(.text) + (ADDR(.data) - ADDR(.text)))
{
*(.data)
- *(.data.*)
}
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.cacheline_aligned :
+ AT (LOADADDR(.text) + (ADDR(.data.cacheline_aligned) - ADDR(.text)))
+ {
+ *(.data.cacheline_aligned)
+ }
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.read_mostly :
+ AT (LOADADDR(.text) + (ADDR(.data.read_mostly) - ADDR(.text)))
+ {
+ *(.data.read_mostly)
+ }
+ . = ALIGN(64); /* COHERENCY_UNIT */
_edata = . ;
PROVIDE (edata = .) ;
__bss_start = . ;
Index: sys/arch/i386/conf/kern.ldscript
===================================================================
RCS file: /cvsroot/src/sys/arch/i386/conf/kern.ldscript,v
retrieving revision 1.7
diff -u -p -u -r1.7 kern.ldscript
--- sys/arch/i386/conf/kern.ldscript 18 Oct 2007 15:28:34 -0000 1.7
+++ sys/arch/i386/conf/kern.ldscript 21 May 2010 18:58:02 -0000
@@ -32,7 +32,19 @@ SECTIONS
AT (LOADADDR(.text) + (ADDR(.data) - ADDR(.text)))
{
*(.data)
- *(.data.*)
+ }
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.cacheline_aligned :
+ AT (LOADADDR(.text) + (ADDR(.data.cacheline_aligned) - ADDR(.text)))
+ {
+ *(.data.cacheline_aligned)
+ }
+
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.read_mostly :
+ AT (LOADADDR(.text) + (ADDR(.data.read_mostly) - ADDR(.text)))
+ {
+ *(.data.read_mostly)
}
_edata = . ;
PROVIDE (edata = .) ;
Index: sys/arch/i386/conf/kern.ldscript.Xen
===================================================================
RCS file: /cvsroot/src/sys/arch/i386/conf/kern.ldscript.Xen,v
retrieving revision 1.3
diff -u -p -u -r1.3 kern.ldscript.Xen
--- sys/arch/i386/conf/kern.ldscript.Xen 18 Oct 2007 15:28:34 -0000
1.3
+++ sys/arch/i386/conf/kern.ldscript.Xen 21 May 2010 18:58:02 -0000
@@ -14,7 +14,6 @@ SECTIONS
} =0
_etext = . ;
PROVIDE (etext = .) ;
-
.rodata :
{
*(.rodata)
Index: sys/arch/sparc64/conf/kern.ldscript
===================================================================
RCS file: /cvsroot/src/sys/arch/sparc64/conf/kern.ldscript,v
retrieving revision 1.10
diff -u -p -u -r1.10 kern.ldscript
--- sys/arch/sparc64/conf/kern.ldscript 17 Oct 2007 19:57:28 -0000 1.10
+++ sys/arch/sparc64/conf/kern.ldscript 21 May 2010 18:58:08 -0000
@@ -74,6 +74,10 @@ SECTIONS
CONSTRUCTORS
}
.data1 : { *(.data1) }
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.cacheline_aligned : { *(.data.cacheline_aligned) }
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.read_mostly : { *(.data.read_mostly) }
.ctors :
{
*(.ctors)
@@ -102,7 +106,11 @@ SECTIONS
. = ALIGN(64 / 8);
_end = . ;
PROVIDE (end = .);
- /* Stabs debugging sections. */
+ /* These must appear regardless of . */
+ .note.netbsd.ident :
+ {
+ KEEP(*(.note.netbsd.ident));
+ }
.stab 0 : { *(.stab) }
.stabstr 0 : { *(.stabstr) }
.stab.excl 0 : { *(.stab.excl) }
@@ -110,19 +118,12 @@ SECTIONS
.stab.index 0 : { *(.stab.index) }
.stab.indexstr 0 : { *(.stab.indexstr) }
.comment 0 : { *(.comment) }
- /* DWARF debug sections.
- Symbols in the DWARF debugging sections are relative to the beginning
- of the section so we begin them at 0. */
- /* DWARF 1 */
.debug 0 : { *(.debug) }
.line 0 : { *(.line) }
- /* GNU DWARF 1 extensions */
.debug_srcinfo 0 : { *(.debug_srcinfo) }
.debug_sfnames 0 : { *(.debug_sfnames) }
- /* DWARF 1.1 and DWARF 2 */
.debug_aranges 0 : { *(.debug_aranges) }
.debug_pubnames 0 : { *(.debug_pubnames) }
- /* DWARF 2 */
.debug_info 0 : { *(.debug_info) }
.debug_abbrev 0 : { *(.debug_abbrev) }
.debug_line 0 : { *(.debug_line) }
@@ -130,14 +131,8 @@ SECTIONS
.debug_str 0 : { *(.debug_str) }
.debug_loc 0 : { *(.debug_loc) }
.debug_macinfo 0 : { *(.debug_macinfo) }
- /* SGI/MIPS DWARF 2 extensions */
.debug_weaknames 0 : { *(.debug_weaknames) }
.debug_funcnames 0 : { *(.debug_funcnames) }
.debug_typenames 0 : { *(.debug_typenames) }
.debug_varnames 0 : { *(.debug_varnames) }
- /* These must appear regardless of . */
- .note.netbsd.ident :
- {
- KEEP(*(.note.netbsd.ident));
- }
}
Index: sys/arch/sparc64/conf/kern32.ldscript
===================================================================
RCS file: /cvsroot/src/sys/arch/sparc64/conf/kern32.ldscript,v
retrieving revision 1.9
diff -u -p -u -r1.9 kern32.ldscript
--- sys/arch/sparc64/conf/kern32.ldscript 17 Oct 2007 19:57:28 -0000
1.9
+++ sys/arch/sparc64/conf/kern32.ldscript 21 May 2010 18:58:08 -0000
@@ -75,6 +75,10 @@ SECTIONS
CONSTRUCTORS
}
.data1 : { *(.data1) }
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.cacheline_aligned : { *(.data_cacheline_aligned) }
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.read_mostly : { *(.data.read_mostly) }
.ctors :
{
*(.ctors)
Index: sys/modules/xldscripts/kmodule
===================================================================
RCS file: /cvsroot/src/sys/modules/xldscripts/kmodule,v
retrieving revision 1.1
diff -u -p -u -r1.1 kmodule
--- sys/modules/xldscripts/kmodule 10 Nov 2009 14:47:52 -0000 1.1
+++ sys/modules/xldscripts/kmodule 21 May 2010 18:58:36 -0000
@@ -6,11 +6,15 @@ SECTIONS
.data 0 :
{
*(.data)
- *(.data.*)
*(.bss)
*(.bss.*)
*(COMMON)
}
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.cacheline_aligned : { *(.data.cacheline_aligned) }
+ . = ALIGN(64); /* COHERENCY_UNIT */
+ .data.read_mostly : { *(.data.read_mostly) }
+ . = ALIGN(64); /* COHERENCY_UNIT */
/* Pre-loaded modules do not need the following. */
/DISCARD/ :
{
Index: sys/sys/cdefs_elf.h
===================================================================
RCS file: /cvsroot/src/sys/sys/cdefs_elf.h,v
retrieving revision 1.30
diff -u -p -u -r1.30 cdefs_elf.h
--- sys/sys/cdefs_elf.h 21 Jul 2008 15:22:19 -0000 1.30
+++ sys/sys/cdefs_elf.h 21 May 2010 18:58:40 -0000
@@ -30,6 +30,10 @@
#ifndef _SYS_CDEFS_ELF_H_
#define _SYS_CDEFS_ELF_H_
+#ifdef _KERNEL_OPT
+#include "opt_multiprocessor.h"
+#endif
+
#ifdef __LEADING_UNDERSCORE
#define _C_LABEL(x) __CONCAT(_,x)
#define _C_LABEL_STRING(x) "_"x
@@ -156,4 +160,38 @@
#define __link_set_count(set)
\
(__link_set_end(set) - __link_set_start(set))
+/*
+ * On multiprocessor systems we can gain an improvement in performance
+ * by being mindful of which cachelines data is placed in.
+ *
+ * __read_mostly:
+ *
+ * It makes sense to ensure that rarely modified data is not
+ * placed in the same cacheline as frequently modified data.
+ * To mitigate the phenomenon known as "false-sharing" we
+ * can annotate rarely modified variables with __read_mostly.
+ * All such variables are placed into the .data.read_mostly
+ * section in the kernel ELF.
+ *
+ * Prime candidates for __read_mostly annotation are variables
+ * which are hardly ever modified and which are used in code
+ * hot-paths, e.g. pmap_initialized.
+ *
+ * __mp_friendly:
+ *
+ * Some data structures (mainly locks) benefit from being aligned
+ * on a cacheline boundary, and having a cacheline to themselves.
+ * This way, the modification of other data items cannot adversely
+ * affect the lock and vice versa.
+ *
+ * Any variables annotated with __mp_friendly will be
+ * placed into the .data.cacheline_aligned ELF section.
+ */
+#define __read_mostly \
+ __attribute__((__section__(".data.read_mostly")))
+
+#define __mp_friendly \
+ __attribute__((__aligned__(COHERENCY_UNIT) \
+ __section__(".data.cacheline_aligned")))
+
#endif /* !_SYS_CDEFS_ELF_H_ */
Home |
Main Index |
Thread Index |
Old Index