Re: lib/57250: dtoa mishandles infinite doubles on 32bit big endian machines

[Havard Eidnes <he%NetBSD.org@localhost>]

Hi,

based on hints from Martin, and a suggestion that the non-
WIDE_DOUBLE code misses handling for %a and %A, I came up with
this diff which at least produces one correct result for %a on
NetBSD/macppc :)

"More testing required".

Regards,

- Havard

------------------------------

Index: include/extern.h
===================================================================
RCS file: /cvsroot/src/lib/libc/include/extern.h,v
retrieving revision 1.26
diff -u -p -r1.26 extern.h
--- include/extern.h	15 May 2020 14:37:21 -0000	1.26
+++ include/extern.h	3 Apr 2023 12:33:57 -0000
@@ -49,11 +49,17 @@ struct sigaction;
 int __sigaction_sigtramp(int, const struct sigaction *,
     struct sigaction *, const void *, int);
 
+/* is "long double" and "double" different? */
+#if (__LDBL_MANT_DIG__ != __DBL_MANT_DIG__) || \
+    (__LDBL_MAX_EXP__ != __DBL_MAX_EXP__)
+#define WIDE_DOUBLE
+#endif
+
 #ifdef WIDE_DOUBLE
-char *__hdtoa(double, const char *, int, int *, int *, char **);
 char *__hldtoa(long double, const char *, int, int *, int *,  char **);
 char *__ldtoa(long double *, int, int, int *, int *, char **);
 #endif
+char *__hdtoa(double, const char *, int, int *, int *, char **);
 
 #ifndef __LIBC12_SOURCE__
 struct syslog_data;
Index: stdio/Makefile.inc
===================================================================
RCS file: /cvsroot/src/lib/libc/stdio/Makefile.inc,v
retrieving revision 1.47
diff -u -p -r1.47 Makefile.inc
--- stdio/Makefile.inc	29 Dec 2015 17:55:23 -0000	1.47
+++ stdio/Makefile.inc	3 Apr 2023 12:33:57 -0000
@@ -4,8 +4,6 @@
 # stdio sources
 .PATH: ${.CURDIR}/stdio
 
-CPPFLAGS+=-DWIDE_DOUBLE
-
 SRCS+=	clrerr.c dprintf.c fclose.c fdopen.c feof.c ferror.c \
 	fflush.c fgetc.c fgetln.c fgetpos.c fgets.c fgetstr.c fgetwc.c \
 	fgetwln.c fgetws.c fileno.c findfp.c flags.c flockfile.c fopen.c \
Index: stdio/vfwprintf.c
===================================================================
RCS file: /cvsroot/src/lib/libc/stdio/vfwprintf.c,v
retrieving revision 1.39
diff -u -p -r1.39 vfwprintf.c
--- stdio/vfwprintf.c	19 Apr 2022 20:32:16 -0000	1.39
+++ stdio/vfwprintf.c	3 Apr 2023 12:33:57 -0000
@@ -650,16 +650,16 @@ WDECL(__vf,printf_unlocked_l)(FILE *fp, 
 	 */
 	char *decimal_point;	/* locale specific decimal point */
 #ifdef WIDE_DOUBLE
-	int signflag;		/* true if float is negative */
 	union {			/* floating point arguments %[aAeEfFgG] */
 		double dbl;
 		long double ldbl;
 	} fparg;
-	char *dtoaend;		/* pointer to end of converted digits */
 #else
 	double _double;		/* double precision arguments %[eEfgG] */
 	char softsign;		/* temporary negative sign for floats */
 #endif
+	int signflag;		/* true if float is negative */
+	char *dtoaend;		/* pointer to end of converted digits */
 	char *dtoaresult;	/* buffer allocated by dtoa */
 	int expt;		/* integer value of exponent */
 	char expchar;		/* exponent character: [eEpP\0] */
@@ -1170,7 +1170,64 @@ fp_common:
 				flags &= ~ZEROPAD;
 				break;
 			}
+#else /* ! WIDE_DOUBLE */
+		case 'a':
+		case 'A':
+			if (ch == 'a') {
+				ox[1] = 'x';
+				xdigs = xdigs_lower;
+				expchar = 'p';
+			} else {
+				ox[1] = 'X';
+				xdigs = xdigs_upper;
+				expchar = 'P';
+			}
+			if (prec >= 0)
+				prec++;
+				
+			_double = GETARG(double);
+			dtoaresult =
+			    __hdtoa(_double, xdigs, prec,
+				    &expt, &signflag, &dtoaend);
+			if (dtoaresult == NULL)
+				goto oomem;
+			
+			if (prec < 0) {
+				_DIAGASSERT(__type_fit(int,
+				    dtoaend - dtoaresult));
+				prec = (int)(dtoaend - dtoaresult);
+			}
+			if (expt == INT_MAX)
+				ox[1] = '\0';
+			_DIAGASSERT(__type_fit(int, dtoaend - dtoaresult));
+			ndig = (int)(dtoaend - dtoaresult);
+			if (convbuf != NULL)
+				free(convbuf);
+#ifndef NARROW
+			result = convbuf = __mbsconv(dtoaresult, -1, loc);
 #else
+			/*XXX inefficient*/
+			result = convbuf = strdup(dtoaresult);
+#endif
+			if (result == NULL)
+				goto oomem;
+			__freedtoa(dtoaresult);
+
+			if (signflag)
+				sign = '-';
+			if (expt == INT_MAX) {	/* inf or nan */
+				if (*result == 'N') {
+					result = (ch >= 'a') ? STRCONST("nan") :
+					    STRCONST("NAN");
+					sign = '\0';
+				} else
+					result = (ch >= 'a') ? STRCONST("inf") :
+					    STRCONST("INF");
+				size = 3;
+				flags &= ~ZEROPAD;
+				break;
+			}
+			goto fp_common_wrapup;
 		case 'e':
 		case 'E':
 		case 'f':
@@ -1229,7 +1286,8 @@ fp_common:
 			__freedtoa(dtoaresult);
 			if (softsign)
 				sign = '-';
-#endif
+#endif /* WIDE_DOUBLE */
+		fp_common_wrapup:
 			flags |= FPT;
 			if (ch == 'g' || ch == 'G') {
 				if (expt > -4 && expt <= prec) {