[src/trunk]: src/sys/kern Avoid integer overflow in kern_malloc(). Reported b...

[martin <martin%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/7836de17b2a4
branches:  trunk
changeset: 825683:7836de17b2a4
user:      martin <martin%NetBSD.org@localhost>
date:      Fri Jul 28 12:28:48 2017 +0000

description:
Avoid integer overflow in kern_malloc(). Reported by Ilja Van Sprundel.
XXX Time to kill malloc() completely!

diffstat:

 sys/kern/kern_malloc.c |  9 ++++++---
 1 files changed, 6 insertions(+), 3 deletions(-)

diffs (30 lines):

diff -r 59e42468f835 -r 7836de17b2a4 sys/kern/kern_malloc.c
--- a/sys/kern/kern_malloc.c    Fri Jul 28 10:34:58 2017 +0000
+++ b/sys/kern/kern_malloc.c    Fri Jul 28 12:28:48 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: kern_malloc.c,v 1.145 2015/02/06 18:21:29 maxv Exp $   */
+/*     $NetBSD: kern_malloc.c,v 1.146 2017/07/28 12:28:48 martin Exp $ */
 
 /*
  * Copyright (c) 1987, 1991, 1993
@@ -70,7 +70,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_malloc.c,v 1.145 2015/02/06 18:21:29 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_malloc.c,v 1.146 2017/07/28 12:28:48 martin Exp $");
 
 #include <sys/param.h>
 #include <sys/malloc.h>
@@ -105,7 +105,10 @@
        void *p;
 
        if (size >= PAGE_SIZE) {
-               allocsize = PAGE_SIZE + size; /* for page alignment */
+               if (size > (ULONG_MAX-PAGE_SIZE))
+                       allocsize = ULONG_MAX;  /* this will fail later */
+               else
+                       allocsize = PAGE_SIZE + size; /* for page alignment */
                hdroffset = PAGE_SIZE - sizeof(struct malloc_header);
        } else {
                allocsize = sizeof(struct malloc_header) + size;