Re: lib/57250: dtoa mishandles infinite doubles on 32bit big endian machines

[Havard Eidnes <he%NetBSD.org@localhost>]

The following reply was made to PR lib/57250; it has been noted by GNATS.

From: Havard Eidnes <he%NetBSD.org@localhost>
To: gnats-bugs%netbsd.org@localhost, martin%duskware.de@localhost
Cc: lib-bug-people%netbsd.org@localhost, gnats-admin%netbsd.org@localhost,
 netbsd-bugs%netbsd.org@localhost, martin%NetBSD.org@localhost
Subject: Re: lib/57250: dtoa mishandles infinite doubles on 32bit big
 endian machines
Date: Tue, 04 Apr 2023 20:24:13 +0200 (CEST)

 Hi,
 
 a slightly simpler diff with less code duplication.
 
 - Havard
 
 ------------------------------
 
 Index: include/extern.h
 ===================================================================
 RCS file: /cvsroot/src/lib/libc/include/extern.h,v
 retrieving revision 1.26
 diff -u -p -r1.26 extern.h
 --- include/extern.h	15 May 2020 14:37:21 -0000	1.26
 +++ include/extern.h	4 Apr 2023 18:21:25 -0000
 @@ -49,11 +49,17 @@ struct sigaction;
  int __sigaction_sigtramp(int, const struct sigaction *,
      struct sigaction *, const void *, int);
  
 +/* is "long double" and "double" different? */
 +#if (__LDBL_MANT_DIG__ != __DBL_MANT_DIG__) || \
 +    (__LDBL_MAX_EXP__ != __DBL_MAX_EXP__)
 +#define WIDE_DOUBLE
 +#endif
 +
  #ifdef WIDE_DOUBLE
 -char *__hdtoa(double, const char *, int, int *, int *, char **);
  char *__hldtoa(long double, const char *, int, int *, int *,  char **);
  char *__ldtoa(long double *, int, int, int *, int *, char **);
  #endif
 +char *__hdtoa(double, const char *, int, int *, int *, char **);
  
  #ifndef __LIBC12_SOURCE__
  struct syslog_data;
 Index: stdio/Makefile.inc
 ===================================================================
 RCS file: /cvsroot/src/lib/libc/stdio/Makefile.inc,v
 retrieving revision 1.47
 diff -u -p -r1.47 Makefile.inc
 --- stdio/Makefile.inc	29 Dec 2015 17:55:23 -0000	1.47
 +++ stdio/Makefile.inc	4 Apr 2023 18:21:25 -0000
 @@ -4,8 +4,6 @@
  # stdio sources
  .PATH: ${.CURDIR}/stdio
  
 -CPPFLAGS+=-DWIDE_DOUBLE
 -
  SRCS+=	clrerr.c dprintf.c fclose.c fdopen.c feof.c ferror.c \
  	fflush.c fgetc.c fgetln.c fgetpos.c fgets.c fgetstr.c fgetwc.c \
  	fgetwln.c fgetws.c fileno.c findfp.c flags.c flockfile.c fopen.c \
 Index: stdio/vfwprintf.c
 ===================================================================
 RCS file: /cvsroot/src/lib/libc/stdio/vfwprintf.c,v
 retrieving revision 1.39
 diff -u -p -r1.39 vfwprintf.c
 --- stdio/vfwprintf.c	19 Apr 2022 20:32:16 -0000	1.39
 +++ stdio/vfwprintf.c	4 Apr 2023 18:21:25 -0000
 @@ -650,16 +650,16 @@ WDECL(__vf,printf_unlocked_l)(FILE *fp, 
  	 */
  	char *decimal_point;	/* locale specific decimal point */
  #ifdef WIDE_DOUBLE
 -	int signflag;		/* true if float is negative */
  	union {			/* floating point arguments %[aAeEfFgG] */
  		double dbl;
  		long double ldbl;
  	} fparg;
 -	char *dtoaend;		/* pointer to end of converted digits */
  #else
  	double _double;		/* double precision arguments %[eEfgG] */
 -	char softsign;		/* temporary negative sign for floats */
 +	char softsign = 0;	/* temporary negative sign for floats */
  #endif
 +	int signflag = 0;	/* true if float is negative */
 +	char *dtoaend;		/* pointer to end of converted digits */
  	char *dtoaresult;	/* buffer allocated by dtoa */
  	int expt;		/* integer value of exponent */
  	char expchar;		/* exponent character: [eEpP\0] */
 @@ -1058,7 +1058,6 @@ reswitch:	switch (ch) {
  			base = 10;
  			goto number;
  #ifndef NO_FLOATING_POINT
 -#ifdef WIDE_DOUBLE
  		case 'a':
  		case 'A':
  			if (ch == 'a') {
 @@ -1072,6 +1071,7 @@ reswitch:	switch (ch) {
  			}
  			if (prec >= 0)
  				prec++;
 +#ifdef WIDE_DOUBLE
  			if (flags & LONGDBL) {
  				fparg.ldbl = GETARG(long double);
  				dtoaresult =
 @@ -1083,6 +1083,12 @@ reswitch:	switch (ch) {
  				    __hdtoa(fparg.dbl, xdigs, prec,
  				        &expt, &signflag, &dtoaend);
  			}
 +#else /* !WIDE_DOUBLE */
 +			_double = GETARG(double);
 +			dtoaresult =
 +			    __hdtoa(_double, xdigs, prec,
 +				    &expt, &signflag, &dtoaend);
 +#endif /* WIDE_DOUBLE */
  			if (dtoaresult == NULL)
  				goto oomem;
  			
 @@ -1107,6 +1113,8 @@ reswitch:	switch (ch) {
  				goto oomem;
  			__freedtoa(dtoaresult);
  			goto fp_common;
 +#ifdef WIDE_DOUBLE
 +
  		case 'e':
  		case 'E':
  			expchar = ch;
 @@ -1170,7 +1178,7 @@ fp_common:
  				flags &= ~ZEROPAD;
  				break;
  			}
 -#else
 +#else /* !WIDE_DOUBLE */
  		case 'e':
  		case 'E':
  		case 'f':
 @@ -1227,9 +1235,11 @@ fp_common:
  			if (result == NULL)
  				goto oomem;
  			__freedtoa(dtoaresult);
 -			if (softsign)
 +fp_common:
 +			if (signflag || softsign)
  				sign = '-';
 -#endif
 +#endif /* WIDE_DOUBLE */
 +
  			flags |= FPT;
  			if (ch == 'g' || ch == 'G') {
  				if (expt > -4 && expt <= prec) {