tech-kern archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

__read_mostly and __mp_friendly annotations



Hi guys,

below is a patch that introduces __read_mostly and __mp_friendly
annotations. These annotations allow infrequently modified data to be
grouped together and allow aligning data on cacheline boundaries. This
helps to mitigate false-sharing.

This patch has been passed around for about a year now so it's pretty
well reviewed but I just wanted to post it here in case anyone had any
comments.

I'll do another patch that uses these annotations soon.


Index: sys/arch/amd64/conf/kern.ldscript
===================================================================
RCS file: /cvsroot/src/sys/arch/amd64/conf/kern.ldscript,v
retrieving revision 1.4
diff -u -p -u -r1.4 kern.ldscript
--- sys/arch/amd64/conf/kern.ldscript   18 Oct 2007 15:28:33 -0000      1.4
+++ sys/arch/amd64/conf/kern.ldscript   21 May 2010 18:57:41 -0000
@@ -32,8 +32,22 @@ SECTIONS
    AT (LOADADDR(.text) + (ADDR(.data) - ADDR(.text)))
    {
      *(.data)
-     *(.data.*)
    }
+
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+   .data.cacheline_aligned :
+   AT (LOADADDR(.text) + (ADDR(.data.cacheline_aligned) - ADDR(.text)))
+   {
+     *(.data.cacheline_aligned)
+   }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+   .data.read_mostly :
+   AT (LOADADDR(.text) + (ADDR(.data.read_mostly) - ADDR(.text)))
+   {
+     *(.data.read_mostly)
+   }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+
    _edata = . ;
    PROVIDE (edata = .) ;
    __bss_start = . ;
Index: sys/arch/amd64/conf/kern.ldscript.2MB
===================================================================
RCS file: /cvsroot/src/sys/arch/amd64/conf/kern.ldscript.2MB,v
retrieving revision 1.2
diff -u -p -u -r1.2 kern.ldscript.2MB
--- sys/arch/amd64/conf/kern.ldscript.2MB       18 Oct 2007 15:28:33 -0000      
1.2
+++ sys/arch/amd64/conf/kern.ldscript.2MB       21 May 2010 18:57:41 -0000
@@ -33,8 +33,20 @@ SECTIONS
    AT (LOADADDR(.text) + (ADDR(.data) - ADDR(.text)))
    {
      *(.data)
-     *(.data.*)
    }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+   .data.cacheline_aligned :
+   AT (LOADADDR(.text) + (ADDR(.data.cacheline_aligned) - ADDR(.text)))
+   {
+     *(.data.cacheline_aligned)
+   }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+   .data.read_mostly :
+   AT (LOADADDR(.text) + (ADDR(.data.read_mostly) - ADDR(.text)))
+   {
+     *(.data.read_mostly)
+   }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
    _edata = . ;
    PROVIDE (edata = .) ;
    __bss_start = . ;
Index: sys/arch/amd64/conf/kern.ldscript.Xen
===================================================================
RCS file: /cvsroot/src/sys/arch/amd64/conf/kern.ldscript.Xen,v
retrieving revision 1.2
diff -u -p -u -r1.2 kern.ldscript.Xen
--- sys/arch/amd64/conf/kern.ldscript.Xen       22 Nov 2007 16:16:44 -0000      
1.2
+++ sys/arch/amd64/conf/kern.ldscript.Xen       21 May 2010 18:57:41 -0000
@@ -26,8 +26,20 @@ SECTIONS
    AT (LOADADDR(.text) + (ADDR(.data) - ADDR(.text)))
    {
      *(.data)
-     *(.data.*)
    }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+   .data.cacheline_aligned :
+   AT (LOADADDR(.text) + (ADDR(.data.cacheline_aligned) - ADDR(.text)))
+   {
+     *(.data.cacheline_aligned)
+   }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+   .data.read_mostly :
+   AT (LOADADDR(.text) + (ADDR(.data.read_mostly) - ADDR(.text)))
+   {
+     *(.data.read_mostly)
+   }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
    _edata = . ;
    PROVIDE (edata = .) ;
    __bss_start = . ;
Index: sys/arch/i386/conf/kern.ldscript
===================================================================
RCS file: /cvsroot/src/sys/arch/i386/conf/kern.ldscript,v
retrieving revision 1.7
diff -u -p -u -r1.7 kern.ldscript
--- sys/arch/i386/conf/kern.ldscript    18 Oct 2007 15:28:34 -0000      1.7
+++ sys/arch/i386/conf/kern.ldscript    21 May 2010 18:58:02 -0000
@@ -32,7 +32,19 @@ SECTIONS
    AT (LOADADDR(.text) + (ADDR(.data) - ADDR(.text)))
    {
      *(.data)
-     *(.data.*)
+   }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+   .data.cacheline_aligned :
+   AT (LOADADDR(.text) + (ADDR(.data.cacheline_aligned) - ADDR(.text)))
+   { 
+     *(.data.cacheline_aligned)
+   }
+
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+   .data.read_mostly :
+   AT (LOADADDR(.text) + (ADDR(.data.read_mostly) - ADDR(.text)))
+   {
+     *(.data.read_mostly)
    }
    _edata = . ;
    PROVIDE (edata = .) ;
Index: sys/arch/i386/conf/kern.ldscript.Xen
===================================================================
RCS file: /cvsroot/src/sys/arch/i386/conf/kern.ldscript.Xen,v
retrieving revision 1.3
diff -u -p -u -r1.3 kern.ldscript.Xen
--- sys/arch/i386/conf/kern.ldscript.Xen        18 Oct 2007 15:28:34 -0000      
1.3
+++ sys/arch/i386/conf/kern.ldscript.Xen        21 May 2010 18:58:02 -0000
@@ -14,7 +14,6 @@ SECTIONS
    } =0
    _etext = . ;
    PROVIDE (etext = .) ;
-
    .rodata :
    {
      *(.rodata)
Index: sys/arch/sparc64/conf/kern.ldscript
===================================================================
RCS file: /cvsroot/src/sys/arch/sparc64/conf/kern.ldscript,v
retrieving revision 1.10
diff -u -p -u -r1.10 kern.ldscript
--- sys/arch/sparc64/conf/kern.ldscript 17 Oct 2007 19:57:28 -0000      1.10
+++ sys/arch/sparc64/conf/kern.ldscript 21 May 2010 18:58:08 -0000
@@ -74,6 +74,10 @@ SECTIONS
     CONSTRUCTORS
   }
   .data1   : { *(.data1) }
+  . = ALIGN(64);       /* COHERENCY_UNIT */
+  .data.cacheline_aligned      : { *(.data.cacheline_aligned) }
+  . = ALIGN(64);       /* COHERENCY_UNIT */
+  .data.read_mostly            : { *(.data.read_mostly) }
   .ctors         :
   {
     *(.ctors)
@@ -102,7 +106,11 @@ SECTIONS
   . = ALIGN(64 / 8);
   _end = . ;
   PROVIDE (end = .);
-  /* Stabs debugging sections.  */
+  /* These must appear regardless of  .  */
+  .note.netbsd.ident :
+  {
+    KEEP(*(.note.netbsd.ident));
+  }
   .stab 0 : { *(.stab) }
   .stabstr 0 : { *(.stabstr) }
   .stab.excl 0 : { *(.stab.excl) }
@@ -110,19 +118,12 @@ SECTIONS
   .stab.index 0 : { *(.stab.index) }
   .stab.indexstr 0 : { *(.stab.indexstr) }
   .comment 0 : { *(.comment) }
-  /* DWARF debug sections.
-     Symbols in the DWARF debugging sections are relative to the beginning
-     of the section so we begin them at 0.  */
-  /* DWARF 1 */
   .debug          0 : { *(.debug) }
   .line           0 : { *(.line) }
-  /* GNU DWARF 1 extensions */
   .debug_srcinfo  0 : { *(.debug_srcinfo) }
   .debug_sfnames  0 : { *(.debug_sfnames) }
-  /* DWARF 1.1 and DWARF 2 */
   .debug_aranges  0 : { *(.debug_aranges) }
   .debug_pubnames 0 : { *(.debug_pubnames) }
-  /* DWARF 2 */
   .debug_info     0 : { *(.debug_info) }
   .debug_abbrev   0 : { *(.debug_abbrev) }
   .debug_line     0 : { *(.debug_line) }
@@ -130,14 +131,8 @@ SECTIONS
   .debug_str      0 : { *(.debug_str) }
   .debug_loc      0 : { *(.debug_loc) }
   .debug_macinfo  0 : { *(.debug_macinfo) }
-  /* SGI/MIPS DWARF 2 extensions */
   .debug_weaknames 0 : { *(.debug_weaknames) }
   .debug_funcnames 0 : { *(.debug_funcnames) }
   .debug_typenames 0 : { *(.debug_typenames) }
   .debug_varnames  0 : { *(.debug_varnames) }
-  /* These must appear regardless of  .  */
-  .note.netbsd.ident :
-  {
-    KEEP(*(.note.netbsd.ident));
-  }
 }
Index: sys/arch/sparc64/conf/kern32.ldscript
===================================================================
RCS file: /cvsroot/src/sys/arch/sparc64/conf/kern32.ldscript,v
retrieving revision 1.9
diff -u -p -u -r1.9 kern32.ldscript
--- sys/arch/sparc64/conf/kern32.ldscript       17 Oct 2007 19:57:28 -0000      
1.9
+++ sys/arch/sparc64/conf/kern32.ldscript       21 May 2010 18:58:08 -0000
@@ -75,6 +75,10 @@ SECTIONS
     CONSTRUCTORS
   }
   .data1   : { *(.data1) }
+  . = ALIGN(64);       /* COHERENCY_UNIT */
+  .data.cacheline_aligned : { *(.data_cacheline_aligned) }
+  . = ALIGN(64);       /* COHERENCY_UNIT */
+  .data.read_mostly : { *(.data.read_mostly) }
   .ctors         :
   {
     *(.ctors)
Index: sys/modules/xldscripts/kmodule
===================================================================
RCS file: /cvsroot/src/sys/modules/xldscripts/kmodule,v
retrieving revision 1.1
diff -u -p -u -r1.1 kmodule
--- sys/modules/xldscripts/kmodule      10 Nov 2009 14:47:52 -0000      1.1
+++ sys/modules/xldscripts/kmodule      21 May 2010 18:58:36 -0000
@@ -6,11 +6,15 @@ SECTIONS
    .data 0 :
    {
      *(.data)
-     *(.data.*)
      *(.bss)
      *(.bss.*)
      *(COMMON)
    }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+   .data.cacheline_aligned     : { *(.data.cacheline_aligned) }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
+   .data.read_mostly           : { *(.data.read_mostly) }
+   . = ALIGN(64);      /* COHERENCY_UNIT */
    /* Pre-loaded modules do not need the following. */
    /DISCARD/ :
    {
Index: sys/sys/cdefs_elf.h
===================================================================
RCS file: /cvsroot/src/sys/sys/cdefs_elf.h,v
retrieving revision 1.30
diff -u -p -u -r1.30 cdefs_elf.h
--- sys/sys/cdefs_elf.h 21 Jul 2008 15:22:19 -0000      1.30
+++ sys/sys/cdefs_elf.h 21 May 2010 18:58:40 -0000
@@ -30,6 +30,10 @@
 #ifndef _SYS_CDEFS_ELF_H_
 #define        _SYS_CDEFS_ELF_H_
 
+#ifdef _KERNEL_OPT
+#include "opt_multiprocessor.h"
+#endif
+
 #ifdef __LEADING_UNDERSCORE
 #define        _C_LABEL(x)     __CONCAT(_,x)
 #define _C_LABEL_STRING(x)     "_"x
@@ -156,4 +160,38 @@
 #define        __link_set_count(set)                                           
\
        (__link_set_end(set) - __link_set_start(set))
 
+/*
+ * On multiprocessor systems we can gain an improvement in performance
+ * by being mindful of which cachelines data is placed in.
+ *
+ * __read_mostly:
+ *
+ *     It makes sense to ensure that rarely modified data is not
+ *     placed in the same cacheline as frequently modified data.
+ *     To mitigate the phenomenon known as "false-sharing" we
+ *     can annotate rarely modified variables with __read_mostly.
+ *     All such variables are placed into the .data.read_mostly
+ *     section in the kernel ELF.
+ *
+ *     Prime candidates for __read_mostly annotation are variables
+ *     which are hardly ever modified and which are used in code
+ *     hot-paths, e.g. pmap_initialized.
+ *
+ * __mp_friendly:
+ *
+ *     Some data structures (mainly locks) benefit from being aligned
+ *     on a cacheline boundary, and having a cacheline to themselves.
+ *     This way, the modification of other data items cannot adversely
+ *     affect the lock and vice versa.
+ *
+ *     Any variables annotated with __mp_friendly will be
+ *     placed into the .data.cacheline_aligned ELF section.
+ */
+#define        __read_mostly                                           \
+    __attribute__((__section__(".data.read_mostly")))
+
+#define        __mp_friendly                                           \
+    __attribute__((__aligned__(COHERENCY_UNIT)                 \
+                __section__(".data.cacheline_aligned")))
+
 #endif /* !_SYS_CDEFS_ELF_H_ */


Home | Main Index | Thread Index | Old Index