Re: lib/57250: dtoa mishandles infinite doubles on 32bit big endian machines

[Havard Eidnes <he%NetBSD.org@localhost>]

Hi,

a slightly simpler diff with less code duplication.

- Havard

------------------------------

Index: include/extern.h
===================================================================
RCS file: /cvsroot/src/lib/libc/include/extern.h,v
retrieving revision 1.26
diff -u -p -r1.26 extern.h
--- include/extern.h	15 May 2020 14:37:21 -0000	1.26
+++ include/extern.h	4 Apr 2023 18:21:25 -0000
@@ -49,11 +49,17 @@ struct sigaction;
 int __sigaction_sigtramp(int, const struct sigaction *,
     struct sigaction *, const void *, int);
 
+/* is "long double" and "double" different? */
+#if (__LDBL_MANT_DIG__ != __DBL_MANT_DIG__) || \
+    (__LDBL_MAX_EXP__ != __DBL_MAX_EXP__)
+#define WIDE_DOUBLE
+#endif
+
 #ifdef WIDE_DOUBLE
-char *__hdtoa(double, const char *, int, int *, int *, char **);
 char *__hldtoa(long double, const char *, int, int *, int *,  char **);
 char *__ldtoa(long double *, int, int, int *, int *, char **);
 #endif
+char *__hdtoa(double, const char *, int, int *, int *, char **);
 
 #ifndef __LIBC12_SOURCE__
 struct syslog_data;
Index: stdio/Makefile.inc
===================================================================
RCS file: /cvsroot/src/lib/libc/stdio/Makefile.inc,v
retrieving revision 1.47
diff -u -p -r1.47 Makefile.inc
--- stdio/Makefile.inc	29 Dec 2015 17:55:23 -0000	1.47
+++ stdio/Makefile.inc	4 Apr 2023 18:21:25 -0000
@@ -4,8 +4,6 @@
 # stdio sources
 .PATH: ${.CURDIR}/stdio
 
-CPPFLAGS+=-DWIDE_DOUBLE
-
 SRCS+=	clrerr.c dprintf.c fclose.c fdopen.c feof.c ferror.c \
 	fflush.c fgetc.c fgetln.c fgetpos.c fgets.c fgetstr.c fgetwc.c \
 	fgetwln.c fgetws.c fileno.c findfp.c flags.c flockfile.c fopen.c \
Index: stdio/vfwprintf.c
===================================================================
RCS file: /cvsroot/src/lib/libc/stdio/vfwprintf.c,v
retrieving revision 1.39
diff -u -p -r1.39 vfwprintf.c
--- stdio/vfwprintf.c	19 Apr 2022 20:32:16 -0000	1.39
+++ stdio/vfwprintf.c	4 Apr 2023 18:21:25 -0000
@@ -650,16 +650,16 @@ WDECL(__vf,printf_unlocked_l)(FILE *fp, 
 	 */
 	char *decimal_point;	/* locale specific decimal point */
 #ifdef WIDE_DOUBLE
-	int signflag;		/* true if float is negative */
 	union {			/* floating point arguments %[aAeEfFgG] */
 		double dbl;
 		long double ldbl;
 	} fparg;
-	char *dtoaend;		/* pointer to end of converted digits */
 #else
 	double _double;		/* double precision arguments %[eEfgG] */
-	char softsign;		/* temporary negative sign for floats */
+	char softsign = 0;	/* temporary negative sign for floats */
 #endif
+	int signflag = 0;	/* true if float is negative */
+	char *dtoaend;		/* pointer to end of converted digits */
 	char *dtoaresult;	/* buffer allocated by dtoa */
 	int expt;		/* integer value of exponent */
 	char expchar;		/* exponent character: [eEpP\0] */
@@ -1058,7 +1058,6 @@ reswitch:	switch (ch) {
 			base = 10;
 			goto number;
 #ifndef NO_FLOATING_POINT
-#ifdef WIDE_DOUBLE
 		case 'a':
 		case 'A':
 			if (ch == 'a') {
@@ -1072,6 +1071,7 @@ reswitch:	switch (ch) {
 			}
 			if (prec >= 0)
 				prec++;
+#ifdef WIDE_DOUBLE
 			if (flags & LONGDBL) {
 				fparg.ldbl = GETARG(long double);
 				dtoaresult =
@@ -1083,6 +1083,12 @@ reswitch:	switch (ch) {
 				    __hdtoa(fparg.dbl, xdigs, prec,
 				        &expt, &signflag, &dtoaend);
 			}
+#else /* !WIDE_DOUBLE */
+			_double = GETARG(double);
+			dtoaresult =
+			    __hdtoa(_double, xdigs, prec,
+				    &expt, &signflag, &dtoaend);
+#endif /* WIDE_DOUBLE */
 			if (dtoaresult == NULL)
 				goto oomem;
 			
@@ -1107,6 +1113,8 @@ reswitch:	switch (ch) {
 				goto oomem;
 			__freedtoa(dtoaresult);
 			goto fp_common;
+#ifdef WIDE_DOUBLE
+
 		case 'e':
 		case 'E':
 			expchar = ch;
@@ -1170,7 +1178,7 @@ fp_common:
 				flags &= ~ZEROPAD;
 				break;
 			}
-#else
+#else /* !WIDE_DOUBLE */
 		case 'e':
 		case 'E':
 		case 'f':
@@ -1227,9 +1235,11 @@ fp_common:
 			if (result == NULL)
 				goto oomem;
 			__freedtoa(dtoaresult);
-			if (softsign)
+fp_common:
+			if (signflag || softsign)
 				sign = '-';
-#endif
+#endif /* WIDE_DOUBLE */
+
 			flags |= FPT;
 			if (ch == 'g' || ch == 'G') {
 				if (expt > -4 && expt <= prec) {