[src/netbsd-9]: src/sys/kern Pull up following revision(s) (requested by maxv...

[martin <martin%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/5487d9af82fc
branches:  netbsd-9
changeset: 458268:5487d9af82fc
user:      martin <martin%NetBSD.org@localhost>
date:      Sun Sep 01 10:56:00 2019 +0000

description:
Pull up following revision(s) (requested by maxv in ticket #129):

        sys/kern/subr_pool.c: revision 1.256
        sys/kern/subr_pool.c: revision 1.257

Kernel Heap Hardening: use bitmaps on all off-page pools. This migrates 29
MI pools on amd64 from linked lists to bitmaps, which have higher security
properties.

Then, change the computation of the size of the PH pools: take into account
the bitmap area available by default in the ph_u2 union, and don't go with
&phpool[>0] if &phpool[0] already has enough space to embed a bitmap.

The pools that are migrated in this change all use bitmaps small enough to
fit in &phpool[0], therefore there is no increase in memory consumption.

 -

Revert r1.254, put back || for KASAN, some destructors like lwp_dtor()
caused false positives. Needs more work.

diffstat:

 sys/kern/subr_pool.c |  45 +++++++++++++++++++++++++--------------------
 1 files changed, 25 insertions(+), 20 deletions(-)

diffs (123 lines):

diff -r 91f432088ab8 -r 5487d9af82fc sys/kern/subr_pool.c
--- a/sys/kern/subr_pool.c      Sun Sep 01 10:49:37 2019 +0000
+++ b/sys/kern/subr_pool.c      Sun Sep 01 10:56:00 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: subr_pool.c,v 1.252.2.1 2019/08/18 09:52:12 martin Exp $       */
+/*     $NetBSD: subr_pool.c,v 1.252.2.2 2019/09/01 10:56:00 martin Exp $       */
 
 /*
  * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018
@@ -33,7 +33,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.252.2.1 2019/08/18 09:52:12 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.252.2.2 2019/09/01 10:56:00 martin Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_ddb.h"
@@ -81,7 +81,7 @@
 #define        PHPOOL_MAX      8
 static struct pool phpool[PHPOOL_MAX];
 #define        PHPOOL_FREELIST_NELEM(idx) \
-       (((idx) == 0) ? 0 : BITMAP_SIZE * (1 << (idx)))
+       (((idx) == 0) ? BITMAP_MIN_SIZE : BITMAP_SIZE * (1 << (idx)))
 
 #if defined(KASAN)
 #define POOL_REDZONE
@@ -162,6 +162,7 @@
 typedef uint32_t pool_item_bitmap_t;
 #define        BITMAP_SIZE     (CHAR_BIT * sizeof(pool_item_bitmap_t))
 #define        BITMAP_MASK     (BITMAP_SIZE - 1)
+#define        BITMAP_MIN_SIZE (CHAR_BIT * sizeof(((struct pool_item_header *)NULL)->ph_u2))
 
 struct pool_item_header {
        /* Page headers */
@@ -201,6 +202,9 @@
 
 #define PHSIZE ALIGN(sizeof(struct pool_item_header))
 
+CTASSERT(offsetof(struct pool_item_header, ph_u2) +
+    BITMAP_MIN_SIZE / CHAR_BIT == sizeof(struct pool_item_header));
+
 #if defined(DIAGNOSTIC) && !defined(KASAN)
 #define POOL_CHECK_MAGIC
 #endif
@@ -588,13 +592,11 @@
                size_t sz;
 
                nelem = PHPOOL_FREELIST_NELEM(idx);
+               KASSERT(nelem != 0);
                snprintf(phpool_names[idx], sizeof(phpool_names[idx]),
                    "phpool-%d", nelem);
-               sz = sizeof(struct pool_item_header);
-               if (nelem) {
-                       sz = offsetof(struct pool_item_header,
-                           ph_bitmap[howmany(nelem, BITMAP_SIZE)]);
-               }
+               sz = offsetof(struct pool_item_header,
+                   ph_bitmap[howmany(nelem, BITMAP_SIZE)]);
                pool_init(&phpool[idx], sz, 0, 0, 0,
                    phpool_names[idx], &pool_allocator_meta, IPL_VM);
        }
@@ -657,12 +659,16 @@
        }
 
        /*
+        * If we're off-page, go with a bitmap.
+        */
+       if (!(pp->pr_roflags & PR_PHINPAGE)) {
+               return true;
+       }
+
+       /*
         * If we're on-page, and the page header can already contain a bitmap
         * big enough to cover all the items of the page, go with a bitmap.
         */
-       if (!(pp->pr_roflags & PR_PHINPAGE)) {
-               return false;
-       }
        bmapsize = roundup(PHSIZE, pp->pr_align) -
            offsetof(struct pool_item_header, ph_bitmap[0]);
        KASSERT(bmapsize % sizeof(pool_item_bitmap_t) == 0);
@@ -801,14 +807,15 @@
        }
 
        /*
-        * If we're off-page and use a bitmap, choose the appropriate pool to
-        * allocate page headers, whose size varies depending on the bitmap. If
-        * we're just off-page, take the first pool, no extra size. If we're
-        * on-page, nothing to do.
+        * If we're off-page, then we're using a bitmap; choose the appropriate
+        * pool to allocate page headers, whose size varies depending on the
+        * bitmap. If we're on-page, nothing to do.
         */
-       if (!(pp->pr_roflags & PR_PHINPAGE) && (pp->pr_roflags & PR_USEBMAP)) {
+       if (!(pp->pr_roflags & PR_PHINPAGE)) {
                int idx;
 
+               KASSERT(pp->pr_roflags & PR_USEBMAP);
+
                for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx);
                    idx++) {
                        /* nothing */
@@ -823,8 +830,6 @@
                            pp->pr_wchan, pp->pr_itemsperpage);
                }
                pp->pr_phpool = &phpool[idx];
-       } else if (!(pp->pr_roflags & PR_PHINPAGE)) {
-               pp->pr_phpool = &phpool[0];
        } else {
                pp->pr_phpool = NULL;
        }
@@ -3099,8 +3104,8 @@
 pool_cache_redzone_check(pool_cache_t pc, void *p)
 {
 #ifdef KASAN
-       /* If there is a ctor+dtor, leave the data as valid. */
-       if (__predict_false(pc_has_ctor(pc) && pc_has_dtor(pc))) {
+       /* If there is a ctor/dtor, leave the data as valid. */
+       if (__predict_false(pc_has_ctor(pc) || pc_has_dtor(pc))) {
                return;
        }
 #endif