Subject: Parallelizing set creation
To: None <tech-toolchain@netbsd.org>
From: James Chacon <jmc@netbsd.org>
List: tech-toolchain
Date: 12/18/2003 01:31:54
Working through the toolchain PR's I found there are 2 open change-requests
related to making set and sum creation parallel capable. The attached patches
do this (and take about half the time to build sets as before when I use -j4).
Any objections?
James
Index: Makefile
===================================================================
RCS file: /cvsroot/src/distrib/sets/Makefile,v
retrieving revision 1.32
diff -u -r1.32 Makefile
--- Makefile 30 Sep 2003 06:26:37 -0000 1.32
+++ Makefile 18 Dec 2003 07:29:36 -0000
@@ -13,6 +13,10 @@
CKSUM=${TOOL_CKSUM:Q} MAKE=${MAKE:Q} MTREE=${TOOL_MTREE:Q} \
MKTEMP=${TOOL_MKTEMP:Q} PAX=${TOOL_PAX:Q}
+.if !defined(MAKETARSETS)
+MAKETARSETS!= ${SETSENV} ${HOST_SH} -c '. sets.defaults; echo $${lists}'
+.endif
+
print_have_gcc3:
@echo ${HAVE_GCC3}
@@ -65,14 +69,21 @@
${SETSENV} ${HOST_SH} ${.CURDIR}/checkflist -x ${CHECKFLIST_FLAGS}
.PRECIOUS: maketars
-maketars: check_DESTDIR check_RELEASEDIR
+maketars: check_DESTDIR check_RELEASEDIR .WAIT maketarsetup .WAIT ${MAKETARSETS:@.TARS.@${RELEASEDIR}/${MACHINE}/binary/sets/${.TARS.}.tgz@}
+ @true
+
+maketarsetup: .PHONY
.if defined(DESTDIR) && ${DESTDIR} != ""
${MAKE} checkflist
.endif
mkdir -p ${RELEASEDIR}/${MACHINE}/binary/sets
+
+.for tar in ${MAKETARSETS}
+${RELEASEDIR}/${MACHINE}/binary/sets/${tar}.tgz:
${SETSENV} ${HOST_SH} ${.CURDIR}/maketars -d ${DESTDIR:S,^$,/,} \
${METALOG.unpriv} -N ${NETBSDSRCDIR}/etc \
- -t ${RELEASEDIR}/${MACHINE}/binary/sets ${MAKETARSETS}
+ -t ${RELEASEDIR}/${MACHINE}/binary/sets ${tar}
+.endfor
.PRECIOUS: makesrctars
makesrctars: check_RELEASEDIR
@@ -81,8 +92,13 @@
${NETBSDSRCDIR} ${RELEASEDIR}/source/sets
.PRECIOUS: makesums
-makesums: check_RELEASEDIR
- ${SETSENV} ${MAKESUMS} -t ${RELEASEDIR}/${MACHINE}/binary/sets
+makesums: check_RELEASEDIR .WAIT ${MAKETARSETS:@.TARS.@do-sum-${.TARS.}@}
+ @true
+
+.for tar in ${MAKETARSETS}
+do-sum-${tar}: .PHONY ${RELEASEDIR}/${MACHINE}/binary/sets/${tar}.tgz
+ ${SETSENV} ${MAKESUMS} -t ${RELEASEDIR}/${MACHINE}/binary/sets ${tar}.tgz
+.endfor
.PRECIOUS: installsets
installsets: check_DESTDIR
Index: makeflist
===================================================================
RCS file: /cvsroot/src/distrib/sets/makeflist,v
retrieving revision 1.65
diff -u -r1.65 makeflist
--- makeflist 25 Nov 2003 07:19:46 -0000 1.65
+++ makeflist 18 Dec 2003 07:29:36 -0000
@@ -10,9 +10,8 @@
. ./sets.defaults
setd=`pwd`
-nlists="base comp etc games man misc text"
+nlists=$lists
xlists="xbase xcomp xcontrib xfont xserver xmisc"
-lists=$nlists
. ./sets.subr
Index: makeobsolete
===================================================================
RCS file: /cvsroot/src/distrib/sets/makeobsolete,v
retrieving revision 1.20
diff -u -r1.20 makeobsolete
--- makeobsolete 9 Dec 2003 17:57:42 -0000 1.20
+++ makeobsolete 18 Dec 2003 07:29:36 -0000
@@ -10,9 +10,8 @@
# set defaults
. $(dirname $0)/sets.defaults
setd=`pwd`
-nlists="base comp etc games man misc text"
+nlists=$lists
xlists="xbase xcomp xcontrib xfont xserver xmisc"
-lists=$nlists
target=./dist
# handle args
Index: maketars
===================================================================
RCS file: /cvsroot/src/distrib/sets/maketars,v
retrieving revision 1.45
diff -u -r1.45 maketars
--- maketars 25 Nov 2003 07:19:46 -0000 1.45
+++ maketars 18 Dec 2003 07:29:36 -0000
@@ -23,9 +23,8 @@
. $(dirname $0)/sets.defaults
setd=$(pwd)
-nlists="base comp etc games man misc text"
+nlists=$lists
xlists="xbase xcomp xcontrib xfont xserver xmisc"
-lists=$nlists
tars=$RELEASEDIR
dest=$DESTDIR
metalog=
Index: sets.defaults
===================================================================
RCS file: /cvsroot/src/distrib/sets/sets.defaults,v
retrieving revision 1.4
diff -u -r1.4 sets.defaults
--- sets.defaults 10 Dec 2003 16:06:09 -0000 1.4
+++ sets.defaults 18 Dec 2003 07:29:36 -0000
@@ -1,5 +1,8 @@
#!/bin/sh
+oIFS=$IFS
+IFS="
+"
for x in $(
${MAKE:-make} -B -f- all <<EOF
.include <bsd.own.mk>
@@ -13,6 +16,7 @@
@echo toolchain_missing=\${TOOLCHAIN_MISSING}
@echo use_tools_toolchain=\${USE_TOOLS_TOOLCHAIN}
@echo have_gcc3=\${HAVE_GCC3}
+ @echo lists=\'base comp etc games man misc text\'
.if defined(USE_XF86_4) && (\${USE_XF86_4} != no)
@echo x11_version=4
.else
@@ -23,3 +27,4 @@
); do
eval $x
done
+IFS=$oIFS