[src/trunk]: src/lib/libc/gen fix off-by-one, found by asan.

[christos <christos%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/4a05383493f7
branches:  trunk
changeset: 821674:4a05383493f7
user:      christos <christos%NetBSD.org@localhost>
date:      Sun Feb 12 22:37:49 2017 +0000

description:
fix off-by-one, found by asan.

diffstat:

 lib/libc/gen/vis.c |  26 ++++++++++++++------------
 1 files changed, 14 insertions(+), 12 deletions(-)

diffs (71 lines):

diff -r 89e59a6b0ce2 -r 4a05383493f7 lib/libc/gen/vis.c
--- a/lib/libc/gen/vis.c        Sun Feb 12 21:52:46 2017 +0000
+++ b/lib/libc/gen/vis.c        Sun Feb 12 22:37:49 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: vis.c,v 1.71 2016/01/14 20:41:23 christos Exp $        */
+/*     $NetBSD: vis.c,v 1.72 2017/02/12 22:37:49 christos Exp $        */
 
 /*-
  * Copyright (c) 1989, 1993
@@ -57,7 +57,7 @@
 
 #include <sys/cdefs.h>
 #if defined(LIBC_SCCS) && !defined(lint)
-__RCSID("$NetBSD: vis.c,v 1.71 2016/01/14 20:41:23 christos Exp $");
+__RCSID("$NetBSD: vis.c,v 1.72 2017/02/12 22:37:49 christos Exp $");
 #endif /* LIBC_SCCS and not lint */
 #ifdef __FBSDID
 __FBSDID("$FreeBSD$");
@@ -405,6 +405,14 @@
        _DIAGASSERT(mbsrc != NULL || mblength == 0);
        _DIAGASSERT(mbextra != NULL);
 
+       mbslength = (ssize_t)mblength;
+       /*
+        * When inputing a single character, must also read in the
+        * next character for nextc, the look-ahead character.
+        */
+       if (mbslength == 1)
+               mbslength++;
+
        /*
         * Input (mbsrc) is a char string considered to be multibyte
         * characters.  The input loop will read this string pulling
@@ -421,12 +429,12 @@
        /* Allocate space for the wide char strings */
        psrc = pdst = extra = NULL;
        mdst = NULL;
-       if ((psrc = calloc(mblength + 1, sizeof(*psrc))) == NULL)
+       if ((psrc = calloc(mbslength + 1, sizeof(*psrc))) == NULL)
                return -1;
-       if ((pdst = calloc((4 * mblength) + 1, sizeof(*pdst))) == NULL)
+       if ((pdst = calloc((4 * mbslength) + 1, sizeof(*pdst))) == NULL)
                goto out;
        if (*mbdstp == NULL) {
-               if ((mdst = calloc((4 * mblength) + 1, sizeof(*mdst))) == NULL)
+               if ((mdst = calloc((4 * mbslength) + 1, sizeof(*mdst))) == NULL)
                        goto out;
                *mbdstp = mdst;
        }
@@ -449,13 +457,6 @@
         * stop at NULs because we may be processing a block of data
         * that includes NULs.
         */
-       mbslength = (ssize_t)mblength;
-       /*
-        * When inputing a single character, must also read in the
-        * next character for nextc, the look-ahead character.
-        */
-       if (mbslength == 1)
-               mbslength++;
        while (mbslength > 0) {
                /* Convert one multibyte character to wchar_t. */
                if (!cerr)
@@ -481,6 +482,7 @@
        }
        len = src - psrc;
        src = psrc;
+
        /*
         * In the single character input case, we will have actually
         * processed two characters, c and nextc.  Reset len back to