[src/trunk]: src/sys/uvm mmap(2): Avoid overflow in rounding and checking size.

[riastradh <riastradh%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/2e1f72409ad0
branches:  trunk
changeset: 368337:2e1f72409ad0
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Wed Jul 06 00:40:16 2022 +0000

description:
mmap(2): Avoid overflow in rounding and checking size.

diffstat:

 sys/uvm/uvm_mmap.c |  28 ++++++++++++++++------------
 1 files changed, 16 insertions(+), 12 deletions(-)

diffs (58 lines):

diff -r f0178201f919 -r 2e1f72409ad0 sys/uvm/uvm_mmap.c
--- a/sys/uvm/uvm_mmap.c        Tue Jul 05 22:50:43 2022 +0000
+++ b/sys/uvm/uvm_mmap.c        Wed Jul 06 00:40:16 2022 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_mmap.c,v 1.180 2022/06/04 20:54:03 riastradh Exp $ */
+/*     $NetBSD: uvm_mmap.c,v 1.181 2022/07/06 00:40:16 riastradh Exp $ */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -46,7 +46,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_mmap.c,v 1.180 2022/06/04 20:54:03 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_mmap.c,v 1.181 2022/07/06 00:40:16 riastradh Exp $");
 
 #include "opt_compat_netbsd.h"
 #include "opt_pax.h"
@@ -274,7 +274,7 @@
        struct proc *p = l->l_proc;
        vaddr_t addr;
        off_t pos;
-       vsize_t size, pageoff, newsize;
+       vsize_t size, pageoff;
        vm_prot_t prot, maxprot, extraprot;
        int flags, fd, advice;
        vaddr_t defaddr = 0;    /* XXXGCC */
@@ -309,17 +309,21 @@
                return EINVAL;
 
        /*
-        * align file position and save offset.  adjust size.
+        * Align file position and save offset into page.  Adjust size
+        * so that it is an integral multiple of the page size.
         */
-
-       pageoff = (pos & PAGE_MASK);
-       pos    -= pageoff;
-       newsize = size + pageoff;               /* add offset */
-       newsize = (vsize_t)round_page(newsize); /* round up */
-
-       if (newsize < size)
+       pageoff = pos & PAGE_MASK;
+       pos -= pageoff;
+       CTASSERT(PAGE_MASK <= __type_max(vsize_t));
+       CTASSERT((__type_max(vsize_t) - PAGE_SIZE + 1) % PAGE_SIZE == 0);
+       if (size > __type_max(vsize_t) - PAGE_SIZE + 1 - pageoff)
                return ENOMEM;
-       size = newsize;
+       /*
+        * size + pageoff <= VSIZE_MAX + 1 - PAGE_SIZE, and the
+        * right-hand side is an integral multiple of the page size, so
+        * round_page(size + pageoff) <= VSIZE_MAX + 1 - PAGE_SIZE.
+        */
+       size = round_page(size + pageoff);
 
        /*
         * now check (MAP_FIXED) or get (!MAP_FIXED) the "addr"