[src/trunk]: src/lib/libterminfo libterminfo: cast to uint16/32_t before conv...

[roy <roy%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/f0bef7588514
branches:  trunk
changeset: 934976:f0bef7588514
user:      roy <roy%NetBSD.org@localhost>
date:      Sun Jun 21 15:05:23 2020 +0000

description:
libterminfo: cast to uint16/32_t before conversion to preserve negativity

Otherwise the ABSENT_NUMERIC(-1) or CANCELLED_NUMERIC(-2) will be converted
incorrectly to size_t and then down to uint16/32_t.
Picked up by DIAGNOSTIC builds.

Thanks to Michael Forney for the fix for PR lib/52293.

diffstat:

 lib/libterminfo/compile.c      |   8 ++++----
 lib/libterminfo/term_private.h |  10 ++++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diffs (59 lines):

diff -r 3ad46e178f89 -r f0bef7588514 lib/libterminfo/compile.c
--- a/lib/libterminfo/compile.c Sun Jun 21 14:26:16 2020 +0000
+++ b/lib/libterminfo/compile.c Sun Jun 21 15:05:23 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: compile.c,v 1.25 2020/04/05 12:31:02 roy Exp $ */
+/* $NetBSD: compile.c,v 1.26 2020/06/21 15:05:23 roy Exp $ */
 
 /*
  * Copyright (c) 2009, 2010, 2011, 2020 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
 #endif
 
 #include <sys/cdefs.h>
-__RCSID("$NetBSD: compile.c,v 1.25 2020/04/05 12:31:02 roy Exp $");
+__RCSID("$NetBSD: compile.c,v 1.26 2020/06/21 15:05:23 roy Exp $");
 
 #if !HAVE_NBTOOL_CONFIG_H || HAVE_SYS_ENDIAN_H
 #include <sys/endian.h>
@@ -560,9 +560,9 @@
                return 0;
        _ti_encode_buf_16(tbuf, ind);
        if (len == sizeof(uint32_t))
-               _ti_encode_buf_32(tbuf, num);
+               _ti_encode_buf_32(tbuf, (uint32_t)num);
        else
-               _ti_encode_buf_16(tbuf, num);
+               _ti_encode_buf_16(tbuf, (uint16_t)num);
        tbuf->entries++;
        return 1;
 }
diff -r 3ad46e178f89 -r f0bef7588514 lib/libterminfo/term_private.h
--- a/lib/libterminfo/term_private.h    Sun Jun 21 14:26:16 2020 +0000
+++ b/lib/libterminfo/term_private.h    Sun Jun 21 15:05:23 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: term_private.h,v 1.18 2020/03/29 21:46:22 roy Exp $ */
+/* $NetBSD: term_private.h,v 1.19 2020/06/21 15:05:23 roy Exp $ */
 
 /*
  * Copyright (c) 2009, 2010, 2013, 2020 The NetBSD Foundation, Inc.
@@ -276,14 +276,16 @@
 }
 
 static __inline void
-_ti_encode_buf_num(TBUF *tbuf, size_t num, int rtype)
+_ti_encode_buf_num(TBUF *tbuf, int num, int rtype)
 {
        if (rtype == TERMINFO_RTYPE_O1) {
                if (num > INT16_MAX)
                        num = INT16_MAX;
-               _ti_encode_buf_16(tbuf, num);
+               _ti_encode_buf_16(tbuf, (uint16_t)num);
        } else {
-               _ti_encode_buf_32(tbuf, num);
+               if (num > INT32_MAX)
+                       num = INT32_MAX;
+               _ti_encode_buf_32(tbuf, (uint32_t)num);
        }
 }