[src/trunk]: src/sys Introduce POOL_QUARANTINE, a feature that creates a wind...

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/9c5c5e4bd95e
branches:  trunk
changeset: 455783:9c5c5e4bd95e
user:      maxv <maxv%NetBSD.org@localhost>
date:      Sat Apr 13 08:41:36 2019 +0000

description:
Introduce POOL_QUARANTINE, a feature that creates a window during which a
freed buffer cannot be reallocated. This greatly helps detecting
use-after-frees, because they are not short-lived anymore.

We maintain a per-pool fifo of 128 buffers. On each pool_put, we do a real
free of the oldest buffer, and insert the new buffer. Before insertion, we
mark the buffer as invalid with KASAN. On each pool_cache_put, we destruct
the object, so it lands in pool_put, and the quarantine is handled there.

POOL_QUARANTINE can be used in conjunction with KASAN to detect more
use-after-free bugs.

diffstat:

 sys/arch/amd64/conf/GENERIC |   6 ++-
 sys/conf/files              |   3 +-
 sys/kern/subr_pool.c        |  87 +++++++++++++++++++++++++++++++++++++++++++-
 sys/sys/pool.h              |  15 +++++++-
 4 files changed, 104 insertions(+), 7 deletions(-)

diffs (237 lines):

diff -r 0444c5a2809f -r 9c5c5e4bd95e sys/arch/amd64/conf/GENERIC
--- a/sys/arch/amd64/conf/GENERIC       Sat Apr 13 08:26:14 2019 +0000
+++ b/sys/arch/amd64/conf/GENERIC       Sat Apr 13 08:41:36 2019 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: GENERIC,v 1.521 2019/03/28 19:00:40 maxv Exp $
+# $NetBSD: GENERIC,v 1.522 2019/04/13 08:41:37 maxv Exp $
 #
 # GENERIC machine description file
 #
@@ -22,7 +22,7 @@
 
 options        INCLUDE_CONFIG_FILE     # embed config file in kernel binary
 
-#ident         "GENERIC-$Revision: 1.521 $"
+#ident         "GENERIC-$Revision: 1.522 $"
 
 maxusers       64              # estimated number of users
 
@@ -122,9 +122,11 @@
 options        KDTRACE_HOOKS   # kernel DTrace hooks
 
 # Kernel Address Sanitizer (kASan). You need to disable SVS to use it.
+# The quarantine is optional and can help KASAN find more use-after-frees.
 #makeoptions   KASAN=1         # Kernel Address Sanitizer
 #options       KASAN
 #no options    SVS
+#options       POOL_QUARANTINE
 
 # Kernel Info Leak Detector.
 #makeoptions   KLEAK=1
diff -r 0444c5a2809f -r 9c5c5e4bd95e sys/conf/files
--- a/sys/conf/files    Sat Apr 13 08:26:14 2019 +0000
+++ b/sys/conf/files    Sat Apr 13 08:41:36 2019 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: files,v 1.1233 2019/04/09 22:05:27 pgoyette Exp $
+#      $NetBSD: files,v 1.1234 2019/04/13 08:41:36 maxv Exp $
 #      @(#)files.newconf       7.5 (Berkeley) 5/10/93
 
 version        20171118
@@ -32,6 +32,7 @@
 defflag                                KASAN
 defflag                                KLEAK
 defflag                                KCOV
+defflag opt_pool.h             POOL_QUARANTINE
 
 defparam opt_copy_symtab.h     makeoptions_COPY_SYMTAB
 
diff -r 0444c5a2809f -r 9c5c5e4bd95e sys/kern/subr_pool.c
--- a/sys/kern/subr_pool.c      Sat Apr 13 08:26:14 2019 +0000
+++ b/sys/kern/subr_pool.c      Sat Apr 13 08:41:36 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: subr_pool.c,v 1.248 2019/04/07 09:20:04 maxv Exp $     */
+/*     $NetBSD: subr_pool.c,v 1.249 2019/04/13 08:41:36 maxv Exp $     */
 
 /*
  * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018
@@ -33,11 +33,12 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.248 2019/04/07 09:20:04 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.249 2019/04/13 08:41:36 maxv Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_ddb.h"
 #include "opt_lockdebug.h"
+#include "opt_pool.h"
 #include "opt_kleak.h"
 #endif
 
@@ -111,6 +112,19 @@
 #define pool_cache_kleak_fill(pc, ptr) __nothing
 #endif
 
+#ifdef POOL_QUARANTINE
+static void pool_quarantine_init(struct pool *);
+static void pool_quarantine_flush(struct pool *);
+static bool pool_put_quarantine(struct pool *, void *,
+    struct pool_pagelist *);
+static bool pool_cache_put_quarantine(pool_cache_t, void *, paddr_t);
+#else
+#define pool_quarantine_init(a)                        __nothing
+#define pool_quarantine_flush(a)               __nothing
+#define pool_put_quarantine(a, b, c)           false
+#define pool_cache_put_quarantine(a, b, c)     false
+#endif
+
 #define pc_has_ctor(pc) \
        (pc->pc_ctor != (int (*)(void *, void *, int))nullop)
 #define pc_has_dtor(pc) \
@@ -733,6 +747,7 @@
        pp->pr_drain_hook_arg = NULL;
        pp->pr_freecheck = NULL;
        pool_redzone_init(pp, size);
+       pool_quarantine_init(pp);
 
        /*
         * Decide whether to put the page header off-page to avoid wasting too
@@ -844,6 +859,8 @@
        struct pool_pagelist pq;
        struct pool_item_header *ph;
 
+       pool_quarantine_flush(pp);
+
        /* Remove from global pool list */
        mutex_enter(&pool_head_lock);
        while (pp->pr_refcnt != 0)
@@ -1184,7 +1201,9 @@
        LIST_INIT(&pq);
 
        mutex_enter(&pp->pr_lock);
-       pool_do_put(pp, v, &pq);
+       if (!pool_put_quarantine(pp, v, &pq)) {
+               pool_do_put(pp, v, &pq);
+       }
        mutex_exit(&pp->pr_lock);
 
        pr_pagelist_free(pp, &pq);
@@ -2586,6 +2605,10 @@
        pool_cache_redzone_check(pc, object);
        FREECHECK_IN(&pc->pc_freecheck, object);
 
+       if (pool_cache_put_quarantine(pc, object, pa)) {
+               return;
+       }
+
        /* Lock out interrupts and disable preemption. */
        s = splvm();
        while (/* CONSTCOND */ true) {
@@ -2850,6 +2873,64 @@
 }
 #endif
 
+#ifdef POOL_QUARANTINE
+static void
+pool_quarantine_init(struct pool *pp)
+{
+       pp->pr_quar.rotor = 0;
+       memset(&pp->pr_quar, 0, sizeof(pp->pr_quar));
+}
+
+static void
+pool_quarantine_flush(struct pool *pp)
+{
+       pool_quar_t *quar = &pp->pr_quar;
+       struct pool_pagelist pq;
+       size_t i;
+
+       LIST_INIT(&pq);
+
+       mutex_enter(&pp->pr_lock);
+       for (i = 0; i < POOL_QUARANTINE_DEPTH; i++) {
+               if (quar->list[i] == 0)
+                       continue;
+               pool_do_put(pp, (void *)quar->list[i], &pq);
+       }
+       mutex_exit(&pp->pr_lock);
+
+       pr_pagelist_free(pp, &pq);
+}
+
+static bool
+pool_put_quarantine(struct pool *pp, void *v, struct pool_pagelist *pq)
+{
+       pool_quar_t *quar = &pp->pr_quar;
+       uintptr_t old;
+
+       if (pp->pr_roflags & PR_NOTOUCH) {
+               return false;
+       }
+
+       pool_redzone_check(pp, v);
+
+       old = quar->list[quar->rotor];
+       quar->list[quar->rotor] = (uintptr_t)v;
+       quar->rotor = (quar->rotor + 1) % POOL_QUARANTINE_DEPTH;
+       if (old != 0) {
+               pool_do_put(pp, (void *)old, pq);
+       }
+
+       return true;
+}
+
+static bool
+pool_cache_put_quarantine(pool_cache_t pc, void *p, paddr_t pa)
+{
+       pool_cache_destruct_object(pc, p);
+       return true;
+}
+#endif
+
 #ifdef POOL_REDZONE
 #if defined(_LP64)
 # define PRIME 0x9e37fffffffc0000UL
diff -r 0444c5a2809f -r 9c5c5e4bd95e sys/sys/pool.h
--- a/sys/sys/pool.h    Sat Apr 13 08:26:14 2019 +0000
+++ b/sys/sys/pool.h    Sat Apr 13 08:41:36 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pool.h,v 1.87 2019/03/27 18:27:47 maxv Exp $   */
+/*     $NetBSD: pool.h,v 1.88 2019/04/13 08:41:37 maxv Exp $   */
 
 /*-
  * Copyright (c) 1997, 1998, 1999, 2000, 2007 The NetBSD Foundation, Inc.
@@ -81,6 +81,10 @@
 #include <sys/tree.h>
 #include <sys/callback.h>
 
+#ifdef _KERNEL_OPT
+#include "opt_pool.h"
+#endif
+
 #define        POOL_PADDR_INVALID      ((paddr_t) -1)
 
 struct pool;
@@ -101,6 +105,12 @@
 LIST_HEAD(pool_pagelist,pool_item_header);
 SPLAY_HEAD(phtree, pool_item_header);
 
+#define POOL_QUARANTINE_DEPTH  128
+typedef struct {
+       size_t rotor;
+       intptr_t list[POOL_QUARANTINE_DEPTH];
+} pool_quar_t;
+
 struct pool {
        TAILQ_ENTRY(pool)
                        pr_poollist;
@@ -198,6 +208,9 @@
        bool            pr_redzone;
        size_t          pr_reqsize;
        size_t          pr_reqsize_with_redzone;
+#ifdef POOL_QUARANTINE
+       pool_quar_t     pr_quar;
+#endif
 };
 
 /*