Re: lib/57250: dtoa mishandles infinite doubles on 32bit big endian machines

[Havard Eidnes <he%NetBSD.org@localhost>]

The following reply was made to PR lib/57250; it has been noted by GNATS.

From: Havard Eidnes <he%NetBSD.org@localhost>
To: gnats-bugs%netbsd.org@localhost, martin%duskware.de@localhost
Cc: lib-bug-people%netbsd.org@localhost, gnats-admin%netbsd.org@localhost,
 netbsd-bugs%netbsd.org@localhost, martin%NetBSD.org@localhost
Subject: Re: lib/57250: dtoa mishandles infinite doubles on 32bit big
 endian machines
Date: Mon, 03 Apr 2023 15:01:19 +0200 (CEST)

 Hi,
 
 based on hints from Martin, and a suggestion that the non-
 WIDE_DOUBLE code misses handling for %a and %A, I came up with
 this diff which at least produces one correct result for %a on
 NetBSD/macppc :)
 
 "More testing required".
 
 Regards,
 
 - Havard
 
 ------------------------------
 
 Index: include/extern.h
 ===================================================================
 RCS file: /cvsroot/src/lib/libc/include/extern.h,v
 retrieving revision 1.26
 diff -u -p -r1.26 extern.h
 --- include/extern.h	15 May 2020 14:37:21 -0000	1.26
 +++ include/extern.h	3 Apr 2023 12:33:57 -0000
 @@ -49,11 +49,17 @@ struct sigaction;
  int __sigaction_sigtramp(int, const struct sigaction *,
      struct sigaction *, const void *, int);
  
 +/* is "long double" and "double" different? */
 +#if (__LDBL_MANT_DIG__ != __DBL_MANT_DIG__) || \
 +    (__LDBL_MAX_EXP__ != __DBL_MAX_EXP__)
 +#define WIDE_DOUBLE
 +#endif
 +
  #ifdef WIDE_DOUBLE
 -char *__hdtoa(double, const char *, int, int *, int *, char **);
  char *__hldtoa(long double, const char *, int, int *, int *,  char **);
  char *__ldtoa(long double *, int, int, int *, int *, char **);
  #endif
 +char *__hdtoa(double, const char *, int, int *, int *, char **);
  
  #ifndef __LIBC12_SOURCE__
  struct syslog_data;
 Index: stdio/Makefile.inc
 ===================================================================
 RCS file: /cvsroot/src/lib/libc/stdio/Makefile.inc,v
 retrieving revision 1.47
 diff -u -p -r1.47 Makefile.inc
 --- stdio/Makefile.inc	29 Dec 2015 17:55:23 -0000	1.47
 +++ stdio/Makefile.inc	3 Apr 2023 12:33:57 -0000
 @@ -4,8 +4,6 @@
  # stdio sources
  .PATH: ${.CURDIR}/stdio
  
 -CPPFLAGS+=-DWIDE_DOUBLE
 -
  SRCS+=	clrerr.c dprintf.c fclose.c fdopen.c feof.c ferror.c \
  	fflush.c fgetc.c fgetln.c fgetpos.c fgets.c fgetstr.c fgetwc.c \
  	fgetwln.c fgetws.c fileno.c findfp.c flags.c flockfile.c fopen.c \
 Index: stdio/vfwprintf.c
 ===================================================================
 RCS file: /cvsroot/src/lib/libc/stdio/vfwprintf.c,v
 retrieving revision 1.39
 diff -u -p -r1.39 vfwprintf.c
 --- stdio/vfwprintf.c	19 Apr 2022 20:32:16 -0000	1.39
 +++ stdio/vfwprintf.c	3 Apr 2023 12:33:57 -0000
 @@ -650,16 +650,16 @@ WDECL(__vf,printf_unlocked_l)(FILE *fp, 
  	 */
  	char *decimal_point;	/* locale specific decimal point */
  #ifdef WIDE_DOUBLE
 -	int signflag;		/* true if float is negative */
  	union {			/* floating point arguments %[aAeEfFgG] */
  		double dbl;
  		long double ldbl;
  	} fparg;
 -	char *dtoaend;		/* pointer to end of converted digits */
  #else
  	double _double;		/* double precision arguments %[eEfgG] */
  	char softsign;		/* temporary negative sign for floats */
  #endif
 +	int signflag;		/* true if float is negative */
 +	char *dtoaend;		/* pointer to end of converted digits */
  	char *dtoaresult;	/* buffer allocated by dtoa */
  	int expt;		/* integer value of exponent */
  	char expchar;		/* exponent character: [eEpP\0] */
 @@ -1170,7 +1170,64 @@ fp_common:
  				flags &= ~ZEROPAD;
  				break;
  			}
 +#else /* ! WIDE_DOUBLE */
 +		case 'a':
 +		case 'A':
 +			if (ch == 'a') {
 +				ox[1] = 'x';
 +				xdigs = xdigs_lower;
 +				expchar = 'p';
 +			} else {
 +				ox[1] = 'X';
 +				xdigs = xdigs_upper;
 +				expchar = 'P';
 +			}
 +			if (prec >= 0)
 +				prec++;
 +				
 +			_double = GETARG(double);
 +			dtoaresult =
 +			    __hdtoa(_double, xdigs, prec,
 +				    &expt, &signflag, &dtoaend);
 +			if (dtoaresult == NULL)
 +				goto oomem;
 +			
 +			if (prec < 0) {
 +				_DIAGASSERT(__type_fit(int,
 +				    dtoaend - dtoaresult));
 +				prec = (int)(dtoaend - dtoaresult);
 +			}
 +			if (expt == INT_MAX)
 +				ox[1] = '\0';
 +			_DIAGASSERT(__type_fit(int, dtoaend - dtoaresult));
 +			ndig = (int)(dtoaend - dtoaresult);
 +			if (convbuf != NULL)
 +				free(convbuf);
 +#ifndef NARROW
 +			result = convbuf = __mbsconv(dtoaresult, -1, loc);
  #else
 +			/*XXX inefficient*/
 +			result = convbuf = strdup(dtoaresult);
 +#endif
 +			if (result == NULL)
 +				goto oomem;
 +			__freedtoa(dtoaresult);
 +
 +			if (signflag)
 +				sign = '-';
 +			if (expt == INT_MAX) {	/* inf or nan */
 +				if (*result == 'N') {
 +					result = (ch >= 'a') ? STRCONST("nan") :
 +					    STRCONST("NAN");
 +					sign = '\0';
 +				} else
 +					result = (ch >= 'a') ? STRCONST("inf") :
 +					    STRCONST("INF");
 +				size = 3;
 +				flags &= ~ZEROPAD;
 +				break;
 +			}
 +			goto fp_common_wrapup;
  		case 'e':
  		case 'E':
  		case 'f':
 @@ -1229,7 +1286,8 @@ fp_common:
  			__freedtoa(dtoaresult);
  			if (softsign)
  				sign = '-';
 -#endif
 +#endif /* WIDE_DOUBLE */
 +		fp_common_wrapup:
  			flags |= FPT;
  			if (ch == 'g' || ch == 'G') {
  				if (expt > -4 && expt <= prec) {