Subject: mklocale and the new toolchain
To: None <tech-toolchain@netbsd.org>
From: James Chacon <jchacon@genuity.net>
List: tech-toolchain
Date: 10/17/2001 03:10:54
As some folks are aware mklocale is pretty heavily tied to -current so getting
it to compile with USE_NEW_TOOLCHAIN had been put off.

I took a look and this isn't actually very hard but I wanted to get some
feedback on this before committing (since it's understandbly ugly, but so
is mklocale...)

Anyways there are is one major include file that gets pulled in:

runetype.h (via rune.h)

There's a good section of this include file that only needs to be enabled if
you're building libc locale support directly. mklocale only needs the locale
struct and the file locale struct.

So what I'm proposing is a new define HOST_TOOLCHAIN that is used to null out
sections like this (and accordingly have the tools/Makefile's set it within
CPPFLAGS for all builds). Doing this was relatively simple on runetype.h.

This will become even more important if the idea of foreign builds ever wants
to succeed (try compiling something like mtree on anything but a NetBSD system
for instance...it's quite painful but most of the pain is in routines you
don't need if you're just needing an mtree for build purposes.)

The other thing is runetype.h depends on machine/int_types.h which wasn't
included until post 1.5-release. To get around that I just check the netbsd
version and use the typedef's from inttypes.h to typedef the __uint* 
definitions needed here.

Finally, the really ugly part...When you create a new locale the invalid rune
gets set to the INVALID_RUNE macro as the default. This macro though is just a
reference to _CurrentLocale which of course isn't a symbol present in anything
except -current. (Why the invalid rune for a new locale should default to 
different values depending on ones current locale while using mklocale is 
beyond me, but I'm not going there...)

The choices are 2:

Just set this value (via a netbsd version check) to -3 (which is what it 
defaults to in the end libc code)

or

just pull runetable.c into mklocale (with some sections ifdef'd out with
HOST_TOOLCHAIN from above) and just bring the default locale structs directly
in. Of course if the netbsd version is 1.5-current or newer this ends up just 
ifdef'ing the whole file out (since it breaks anything dealing with locale in
the current libraries otherwise).

I've got code that does the latter but neither solution is particulary pretty.
(especially since runetable.c won't compile with WARNS=2 so that has to be
turned down to 1 if building the tools)

Diff are attached. I've tested this on non NEW_TOOLCHAIN platforms (macppc)
and on both 1.5.2 and -current systems. Everything builds clean that I can
see and comparing the .o's from before/after shows no differences for the 
library builds.

(With this and the sets changes I'm gonna mail off in a sec a complete snapshot
can be built with USE_NEW_TOOLCHAIN again)

James

Index: lib/libc/locale/runetable.c
===================================================================
RCS file: /cvsroot/basesrc/lib/libc/locale/runetable.c,v
retrieving revision 1.6
diff -u -r1.6 runetable.c
--- lib/libc/locale/runetable.c 2001/03/26 19:55:42     1.6
+++ lib/libc/locale/runetable.c 2001/10/17 06:55:18
@@ -38,6 +38,16 @@
  * $FreeBSD: src/lib/libc/locale/table.c,v 1.13 2000/02/08 07:43:25 obrien Exp $
  */
 
+#include <sys/param.h>
+
+/* 
+ * Compile on 1.5-current or newer if HOST_TOOLCHAIN isn't set. (libc)
+ * Compile on 1.5-release or older if HOST_TOOLCHAIN is set. (tools/mklocale)
+ */
+
+#if ((__NetBSD_Version__ > 105009900) && !defined(HOST_TOOLCHAIN)) || \
+     (defined(HOST_TOOLCHAIN) && (__NetBSD_Version__ < 105009900))
+
 #include <sys/cdefs.h>
 #if defined(LIBC_SCCS) && !defined(lint)
 #if 0
@@ -52,6 +62,7 @@
 #include "rune.h"
 #include <stdlib.h>
 
+#ifndef HOST_TOOLCHAIN
 extern size_t  _none_mbrtowc __P((struct _RuneLocale *, rune_t *, const char *,
        size_t, void *));
 extern size_t  _none_wcrtomb __P((struct _RuneLocale *, char *, size_t,
@@ -63,6 +74,7 @@
        NULL,           /* packstate */
        NULL            /* unpackstate */
 };
+#endif
 
 _RuneLocale _DefaultRuneLocale = {
     _RUNE_MAGIC_1,
@@ -268,10 +280,12 @@
     { 0, NULL },
     { 0, NULL },
     NULL, 0, 
+#ifndef HOST_TOOLCHAIN
     (__rune_mbrtowc_t)_none_mbrtowc,
     (__rune_wcrtomb_t)_none_wcrtomb,
-    &_DefaultRuneState, 1,
-    NULL, "646"
+    &_DefaultRuneState,
+#endif
+    1, NULL, "646"
 };
 
 _RuneLocale *_CurrentRuneLocale = &_DefaultRuneLocale;
@@ -279,3 +293,4 @@
 void **_StreamStateTable = NULL;
 
 char   *_PathLocale;
+#endif /* __NetBSD_Version__ */
Index: lib/libc/locale/runetype.h
===================================================================
RCS file: /cvsroot/basesrc/lib/libc/locale/runetype.h,v
retrieving revision 1.5
diff -u -r1.5 runetype.h
--- lib/libc/locale/runetype.h  2001/03/26 19:55:43     1.5
+++ lib/libc/locale/runetype.h  2001/10/17 06:55:18
@@ -44,7 +44,18 @@
 #include <sys/cdefs.h>
 #include <machine/ansi.h>
 
+#include <sys/param.h>
+
+#if (__NetBSD_Version__ < 105009900)
+#include <inttypes.h>
+typedef int64_t                __int64_t;
+typedef uint64_t       __uint64_t;
+typedef int32_t                __int32_t;
+typedef uint32_t       __uint32_t;
+#else
 #include <machine/int_types.h>
+#endif
+
 typedef __uint64_t     __runepad_t;
 
 #ifdef _BSD_SIZE_T_
@@ -171,6 +182,7 @@
 } _RuneRange;
 
 
+#ifndef HOST_TOOLCHAIN
 struct _RuneLocale;
 typedef struct _RuneState {
        size_t          __sizestate;
@@ -186,6 +198,7 @@
        const char *, size_t, void *));
 typedef size_t (*__rune_wcrtomb_t) __P((struct _RuneLocale *, char *, size_t,
        const rune_t, void *));
+#endif
 
 typedef struct _RuneLocale {
        /*
@@ -208,6 +221,7 @@
        size_t          __variable_len; /* how long that data is */
        void *          __rune_variable;
 
+#ifndef HOST_TOOLCHAIN
        /*
         * the following portion is generated on the fly
         */
@@ -215,6 +229,7 @@
        __rune_wcrtomb_t __rune_wcrtomb;
 
        struct _RuneState  *__rune_RuneState;
+#endif
        size_t          __rune_mb_cur_max;
 
        void            *__rune_RuneModule;
Index: tools/Makefile
===================================================================
RCS file: /cvsroot/basesrc/tools/Makefile,v
retrieving revision 1.7
diff -u -r1.7 Makefile
--- tools/Makefile      2001/10/13 06:11:06     1.7
+++ tools/Makefile      2001/10/17 07:01:16
@@ -6,7 +6,7 @@
 SUBDIR=                binstall .WAIT gencat mkdep mtree rpcgen tsort \
                yacc .WAIT lex .WAIT \
                compile_et config lint lint1 lint2 lorder \
-               toolchain .WAIT texinfo crunchgen msgc menuc
+               toolchain .WAIT texinfo crunchgen msgc menuc mklocale
 
 TARGETS+=      build
 
Index: tools/Makefile.host
===================================================================
RCS file: /cvsroot/basesrc/tools/Makefile.host,v
retrieving revision 1.2
diff -u -r1.2 Makefile.host
--- tools/Makefile.host 2001/10/13 06:09:25     1.2
+++ tools/Makefile.host 2001/10/17 07:02:16
@@ -17,7 +17,7 @@
 HOSTPROGNAME?= ${HOSTPROG}
 HOST_BINDIR?=  ${TOOLDIR}/bin
 HOST_CPPFLAGS:=        -include ${_CURDIR}/../compat/compat_netbsd.h -Wall \
-               ${HOST_CPPFLAGS} ${CPPFLAGS}
+               -DHOST_TOOLCHAIN ${HOST_CPPFLAGS} ${CPPFLAGS}
 MKMAN=         no
 SRCS?=         ${PROG}.c
 SRCS+=         ${HOST_SRCS} nb_progname.c
Index: usr.bin/mklocale/Makefile
===================================================================
RCS file: /cvsroot/basesrc/usr.bin/mklocale/Makefile,v
retrieving revision 1.6
diff -u -r1.6 Makefile
--- usr.bin/mklocale/Makefile   2001/08/14 10:18:28     1.6
+++ usr.bin/mklocale/Makefile   2001/10/17 07:03:09
@@ -7,6 +7,12 @@
 SRCS=  yacc.y lex.l #y.tab.h
 CPPFLAGS+=-I${.OBJDIR} # for y.tab.h
 CPPFLAGS+=-I${.CURDIR} -I${.CURDIR}/../../lib/libc/locale
+.ifdef HOSTPROG
+.PATH: ${.CURDIR}/../../lib/libc/locale
+SRCS+= runetable.c
+# So runetable.c can be pulled in without warnings on the strings
+WARNS= 1
+.endif
 MAN1=  mklocale.1
 CLEANFILES+= y.tab.c y.tab.h
 YHEADER=       yes
Index: share/mk/bsd.own.mk
===================================================================
RCS file: /cvsroot/sharesrc/share/mk/bsd.own.mk,v
retrieving revision 1.187
diff -u -r1.187 bsd.own.mk
--- share/mk/bsd.own.mk 2001/10/13 06:11:23     1.187
+++ share/mk/bsd.own.mk 2001/10/17 07:04:48
@@ -58,7 +58,7 @@
 MAKEINFO=      ${TOOLDIR}/bin/makeinfo
 MENUC=         MENUDEF=${TOOLDIR}/share/misc ${TOOLDIR}/bin/menuc
 MKDEP=         CC=${CC} ${TOOLDIR}/bin/mkdep
-#MKLOCALE=     ${TOOLDIR}/bin/mklocale
+MKLOCALE=      ${TOOLDIR}/bin/mklocale
 MSGC=          MSGDEF=${TOOLDIR}/share/misc ${TOOLDIR}/bin/msgc
 MTREE=         ${TOOLDIR}/bin/mtree
 NM=            ${TOOLDIR}/bin/${MACHINE_GNU_PLATFORM}-nm
Index: share/locale/Makefile
===================================================================
RCS file: /cvsroot/sharesrc/share/locale/Makefile,v
retrieving revision 1.2
diff -u -r1.2 Makefile
--- share/locale/Makefile       2001/08/14 11:40:40     1.2
+++ share/locale/Makefile       2001/10/17 07:05:37
@@ -2,14 +2,6 @@
 
 .include <bsd.own.mk>
 
-# XXX - Temporarily disable building locale files on USE_NEW_TOOLCHAIN. - XXX
-#
-# The /usr/bin/mklocale program is very tied to NetBSD-current and is useless
-# for cross builds (as is one of the main purposes of the new toolchain).
-# Disable building the ctype files until mklocale is in a cross-usable state.
-
-.ifndef USE_NEW_TOOLCHAIN
 SUBDIR=        ctype
-.endif
 
 .include <bsd.subdir.mk>
--- /dev/null   Wed Oct 17 04:44:40 2001
+++ tools/mklocale/Makefile     Mon Oct 15 07:35:55 2001
@@ -0,0 +1,7 @@
+#      $NetBSD$
+
+HOST_SRCDIR=   usr.bin/mklocale
+
+.include "${.CURDIR}/../Makefile.host"
+
+${TIMESTAMP}: ${SRCS}