Subject: Parallelizing set creation
To: None <tech-toolchain@netbsd.org>
From: James Chacon <jmc@netbsd.org>
List: tech-toolchain
Date: 12/18/2003 01:31:54
Working through the toolchain PR's I found there are 2 open change-requests
related to making set and sum creation parallel capable. The attached patches 
do this (and take about half the time to build sets as before when I use -j4).

Any objections?

James

Index: Makefile
===================================================================
RCS file: /cvsroot/src/distrib/sets/Makefile,v
retrieving revision 1.32
diff -u -r1.32 Makefile
--- Makefile	30 Sep 2003 06:26:37 -0000	1.32
+++ Makefile	18 Dec 2003 07:29:36 -0000
@@ -13,6 +13,10 @@
 		CKSUM=${TOOL_CKSUM:Q} MAKE=${MAKE:Q} MTREE=${TOOL_MTREE:Q} \
 		MKTEMP=${TOOL_MKTEMP:Q} PAX=${TOOL_PAX:Q} 
 
+.if !defined(MAKETARSETS)
+MAKETARSETS!=	${SETSENV} ${HOST_SH} -c '. sets.defaults; echo $${lists}'
+.endif
+
 print_have_gcc3:
 	@echo ${HAVE_GCC3}
 
@@ -65,14 +69,21 @@
 	${SETSENV} ${HOST_SH} ${.CURDIR}/checkflist -x ${CHECKFLIST_FLAGS}
 
 .PRECIOUS: maketars
-maketars: check_DESTDIR check_RELEASEDIR
+maketars: check_DESTDIR check_RELEASEDIR .WAIT maketarsetup .WAIT ${MAKETARSETS:@.TARS.@${RELEASEDIR}/${MACHINE}/binary/sets/${.TARS.}.tgz@}
+	@true
+
+maketarsetup: .PHONY
 .if defined(DESTDIR) && ${DESTDIR} != ""
 	${MAKE} checkflist
 .endif
 	mkdir -p ${RELEASEDIR}/${MACHINE}/binary/sets
+
+.for tar in ${MAKETARSETS}
+${RELEASEDIR}/${MACHINE}/binary/sets/${tar}.tgz:
 	${SETSENV} ${HOST_SH} ${.CURDIR}/maketars -d ${DESTDIR:S,^$,/,} \
 	    ${METALOG.unpriv} -N ${NETBSDSRCDIR}/etc \
-	    -t ${RELEASEDIR}/${MACHINE}/binary/sets ${MAKETARSETS}
+	    -t ${RELEASEDIR}/${MACHINE}/binary/sets ${tar}
+.endfor
 
 .PRECIOUS: makesrctars
 makesrctars: check_RELEASEDIR
@@ -81,8 +92,13 @@
 	    ${NETBSDSRCDIR} ${RELEASEDIR}/source/sets
 
 .PRECIOUS: makesums
-makesums: check_RELEASEDIR
-	${SETSENV} ${MAKESUMS} -t ${RELEASEDIR}/${MACHINE}/binary/sets
+makesums: check_RELEASEDIR .WAIT ${MAKETARSETS:@.TARS.@do-sum-${.TARS.}@}
+	@true
+
+.for tar in ${MAKETARSETS}
+do-sum-${tar}: .PHONY ${RELEASEDIR}/${MACHINE}/binary/sets/${tar}.tgz
+	${SETSENV} ${MAKESUMS} -t ${RELEASEDIR}/${MACHINE}/binary/sets ${tar}.tgz
+.endfor
 
 .PRECIOUS: installsets
 installsets: check_DESTDIR
Index: makeflist
===================================================================
RCS file: /cvsroot/src/distrib/sets/makeflist,v
retrieving revision 1.65
diff -u -r1.65 makeflist
--- makeflist	25 Nov 2003 07:19:46 -0000	1.65
+++ makeflist	18 Dec 2003 07:29:36 -0000
@@ -10,9 +10,8 @@
 . ./sets.defaults
 
 setd=`pwd`
-nlists="base comp etc games man misc text"
+nlists=$lists
 xlists="xbase xcomp xcontrib xfont xserver xmisc"
-lists=$nlists
 
 . ./sets.subr
 
Index: makeobsolete
===================================================================
RCS file: /cvsroot/src/distrib/sets/makeobsolete,v
retrieving revision 1.20
diff -u -r1.20 makeobsolete
--- makeobsolete	9 Dec 2003 17:57:42 -0000	1.20
+++ makeobsolete	18 Dec 2003 07:29:36 -0000
@@ -10,9 +10,8 @@
 # set defaults
 . $(dirname $0)/sets.defaults
 setd=`pwd`
-nlists="base comp etc games man misc text"
+nlists=$lists
 xlists="xbase xcomp xcontrib xfont xserver xmisc"
-lists=$nlists
 target=./dist
 
 # handle args
Index: maketars
===================================================================
RCS file: /cvsroot/src/distrib/sets/maketars,v
retrieving revision 1.45
diff -u -r1.45 maketars
--- maketars	25 Nov 2003 07:19:46 -0000	1.45
+++ maketars	18 Dec 2003 07:29:36 -0000
@@ -23,9 +23,8 @@
 . $(dirname $0)/sets.defaults
 
 setd=$(pwd)
-nlists="base comp etc games man misc text"
+nlists=$lists
 xlists="xbase xcomp xcontrib xfont xserver xmisc"
-lists=$nlists
 tars=$RELEASEDIR
 dest=$DESTDIR
 metalog=
Index: sets.defaults
===================================================================
RCS file: /cvsroot/src/distrib/sets/sets.defaults,v
retrieving revision 1.4
diff -u -r1.4 sets.defaults
--- sets.defaults	10 Dec 2003 16:06:09 -0000	1.4
+++ sets.defaults	18 Dec 2003 07:29:36 -0000
@@ -1,5 +1,8 @@
 #!/bin/sh
 
+oIFS=$IFS
+IFS="
+"
 for x in $(
 ${MAKE:-make} -B -f- all <<EOF
 .include <bsd.own.mk>
@@ -13,6 +16,7 @@
 	@echo toolchain_missing=\${TOOLCHAIN_MISSING}
 	@echo use_tools_toolchain=\${USE_TOOLS_TOOLCHAIN}
 	@echo have_gcc3=\${HAVE_GCC3}
+	@echo lists=\'base comp etc games man misc text\'
 .if defined(USE_XF86_4) && (\${USE_XF86_4} != no)
 	@echo x11_version=4
 .else
@@ -23,3 +27,4 @@
 ); do
 	eval $x
 done
+IFS=$oIFS