Re: lib/57250: dtoa mishandles infinite doubles on 32bit big endian machines

[Havard Eidnes <he%NetBSD.org@localhost>]

...and after Christos massaged the code to do more code sharing
(and delete some now-un-needed code), and fixing the
non-WIDE_DOUBLE build issues, this is the resulting diff.

- Havard

------------------------------

Index: include/extern.h
===================================================================
RCS file: /cvsroot/src/lib/libc/include/extern.h,v
retrieving revision 1.26
diff -u -p -r1.26 extern.h
--- include/extern.h	15 May 2020 14:37:21 -0000	1.26
+++ include/extern.h	4 Apr 2023 18:44:22 -0000
@@ -49,11 +49,17 @@ struct sigaction;
 int __sigaction_sigtramp(int, const struct sigaction *,
     struct sigaction *, const void *, int);
 
+/* is "long double" and "double" different? */
+#if (__LDBL_MANT_DIG__ != __DBL_MANT_DIG__) || \
+    (__LDBL_MAX_EXP__ != __DBL_MAX_EXP__)
+#define WIDE_DOUBLE
+#endif
+
 #ifdef WIDE_DOUBLE
-char *__hdtoa(double, const char *, int, int *, int *, char **);
 char *__hldtoa(long double, const char *, int, int *, int *,  char **);
 char *__ldtoa(long double *, int, int, int *, int *, char **);
 #endif
+char *__hdtoa(double, const char *, int, int *, int *, char **);
 
 #ifndef __LIBC12_SOURCE__
 struct syslog_data;
Index: stdio/Makefile.inc
===================================================================
RCS file: /cvsroot/src/lib/libc/stdio/Makefile.inc,v
retrieving revision 1.47
diff -u -p -r1.47 Makefile.inc
--- stdio/Makefile.inc	29 Dec 2015 17:55:23 -0000	1.47
+++ stdio/Makefile.inc	4 Apr 2023 18:44:22 -0000
@@ -4,8 +4,6 @@
 # stdio sources
 .PATH: ${.CURDIR}/stdio
 
-CPPFLAGS+=-DWIDE_DOUBLE
-
 SRCS+=	clrerr.c dprintf.c fclose.c fdopen.c feof.c ferror.c \
 	fflush.c fgetc.c fgetln.c fgetpos.c fgets.c fgetstr.c fgetwc.c \
 	fgetwln.c fgetws.c fileno.c findfp.c flags.c flockfile.c fopen.c \
Index: stdio/vfwprintf.c
===================================================================
RCS file: /cvsroot/src/lib/libc/stdio/vfwprintf.c,v
retrieving revision 1.39
diff -u -p -r1.39 vfwprintf.c
--- stdio/vfwprintf.c	19 Apr 2022 20:32:16 -0000	1.39
+++ stdio/vfwprintf.c	4 Apr 2023 18:44:22 -0000
@@ -581,9 +581,6 @@ WDECL(vf,printf_l)(FILE * __restrict fp,
 #define	DEFPREC		6
 
 static int exponent(CHAR_T *, int, int);
-#ifndef WIDE_DOUBLE
-static char *cvt(double, int, int, char *, int *, int, int *);
-#endif
 
 #endif /* !NO_FLOATING_POINT */
 
@@ -650,16 +647,15 @@ WDECL(__vf,printf_unlocked_l)(FILE *fp, 
 	 */
 	char *decimal_point;	/* locale specific decimal point */
 #ifdef WIDE_DOUBLE
-	int signflag;		/* true if float is negative */
 	union {			/* floating point arguments %[aAeEfFgG] */
 		double dbl;
 		long double ldbl;
 	} fparg;
-	char *dtoaend;		/* pointer to end of converted digits */
 #else
 	double _double;		/* double precision arguments %[eEfgG] */
-	char softsign;		/* temporary negative sign for floats */
 #endif
+	int signflag;		/* true if float is negative */
+	char *dtoaend;		/* pointer to end of converted digits */
 	char *dtoaresult;	/* buffer allocated by dtoa */
 	int expt;		/* integer value of exponent */
 	char expchar;		/* exponent character: [eEpP\0] */
@@ -1058,7 +1054,6 @@ reswitch:	switch (ch) {
 			base = 10;
 			goto number;
 #ifndef NO_FLOATING_POINT
-#ifdef WIDE_DOUBLE
 		case 'a':
 		case 'A':
 			if (ch == 'a') {
@@ -1072,6 +1067,7 @@ reswitch:	switch (ch) {
 			}
 			if (prec >= 0)
 				prec++;
+#ifdef WIDE_DOUBLE
 			if (flags & LONGDBL) {
 				fparg.ldbl = GETARG(long double);
 				dtoaresult =
@@ -1083,6 +1079,12 @@ reswitch:	switch (ch) {
 				    __hdtoa(fparg.dbl, xdigs, prec,
 				        &expt, &signflag, &dtoaend);
 			}
+#else
+			_double = GETARG(double);
+			dtoaresult =
+			  __hdtoa(_double, xdigs, prec,
+				  &expt, &signflag, &dtoaend);
+#endif /* WIDE_DOUBLE */
 			if (dtoaresult == NULL)
 				goto oomem;
 			
@@ -1127,6 +1129,7 @@ reswitch:	switch (ch) {
 fp_begin:
 			if (prec < 0)
 				prec = DEFPREC;
+#ifdef WIDE_DOUBLE
 			if (flags & LONGDBL) {
 				fparg.ldbl = GETARG(long double);
 				dtoaresult =
@@ -1140,6 +1143,14 @@ fp_begin:
 				if (expt == 9999)
 					expt = INT_MAX;
 			}
+#else
+			_double = GETARG(double);
+				dtoaresult =
+				    __dtoa(_double, expchar ? 2 : 3, prec,
+				    &expt, &signflag, &dtoaend);
+				if (expt == 9999)
+					expt = INT_MAX;
+#endif /* WIDE_DOUBLE */
 			if (dtoaresult == NULL)
 				goto oomem;
 			_DIAGASSERT(__type_fit(int, dtoaend - dtoaresult));
@@ -1170,67 +1181,8 @@ fp_common:
 				flags &= ~ZEROPAD;
 				break;
 			}
-#else
-		case 'e':
-		case 'E':
-		case 'f':
-		case 'F':
-		case 'g':
-		case 'G':
-			if (prec == -1) {
-				prec = DEFPREC;
-			} else if ((ch == 'g' || ch == 'G') && prec == 0) {
-				prec = 1;
-			}
-
-			if (flags & LONGDBL) {
-				_double = (double) GETARG(long double);
-			} else {
-				_double = GETARG(double);
-			}
-
-			/* do this before tricky precision changes */
-			if (isinf(_double)) {
-				if (_double < 0)
-					sign = '-';
-				if (ch == 'E' || ch == 'F' || ch == 'G')
-					result = STRCONST("INF");
-				else
-					result = STRCONST("inf");
-				size = 3;
-				flags &= ~ZEROPAD;
-				break;
-			}
-			if (isnan(_double)) {
-				if (ch == 'E' || ch == 'F' || ch == 'G')
-					result = STRCONST("NAN");
-				else
-					result = STRCONST("nan");
-				size = 3;
-				flags &= ~ZEROPAD;
-				break;
-			}
 
 			flags |= FPT;
-			dtoaresult = cvt(_double, prec, flags, &softsign,
-			    &expt, ch, &ndig);
-			if (dtoaresult == NULL)
-				goto oomem;
-			if (convbuf != NULL)
-				free(convbuf);
-#ifndef NARROW
-			result = convbuf = __mbsconv(dtoaresult, -1, loc);
-#else
-			/*XXX inefficient*/
-			result = convbuf = strdup(dtoaresult);
-#endif
-			if (result == NULL)
-				goto oomem;
-			__freedtoa(dtoaresult);
-			if (softsign)
-				sign = '-';
-#endif
-			flags |= FPT;
 			if (ch == 'g' || ch == 'G') {
 				if (expt > -4 && expt <= prec) {
 					/* Make %[gG] smell like %[fF] */
@@ -2001,57 +1953,7 @@ __grow_type_table (size_t nextarg, enum 
 	return 0;
 }
 
-
 #ifndef NO_FLOATING_POINT
-#ifndef WIDE_DOUBLE
-static char *
-cvt(double value, int ndigits, int flags, char *sign, int *decpt, int ch,
-    int *length)
-{
-	int mode, dsgn;
-	char *digits, *bp, *rve;
-
-	_DIAGASSERT(decpt != NULL);
-	_DIAGASSERT(length != NULL);
-	_DIAGASSERT(sign != NULL);
-
-	if (ch == 'f') {
-		mode = 3;		/* ndigits after the decimal point */
-	} else {
-		/* To obtain ndigits after the decimal point for the 'e' 
-		 * and 'E' formats, round to ndigits + 1 significant 
-		 * figures.
-		 */
-		if (ch == 'e' || ch == 'E') {
-			ndigits++;
-		}
-		mode = 2;		/* ndigits significant digits */
-	}
-
-	digits = __dtoa(value, mode, ndigits, decpt, &dsgn, &rve);
-	if (digits == NULL)
-		return NULL;
-	if (dsgn) {
-		value = -value;
-		*sign = '-';
-	} else
-		*sign = '\000';
-	if ((ch != 'g' && ch != 'G') || flags & ALT) {	/* Print trailing zeros */
-		bp = digits + ndigits;
-		if (ch == 'f') {
-			if (*digits == '0' && value)
-				*decpt = -ndigits + 1;
-			bp += *decpt;
-		}
-		if (value == 0)	/* kludge for __dtoa irregularity */
-			rve = bp;
-		while (rve < bp)
-			*rve++ = '0';
-	}
-	*length = rve - digits;
-	return digits;
-}
-#endif
 
 static int
 exponent(CHAR_T *p0, int expo, int fmtch)