Subject: non-.WAIT bsd.subdir.mk (was Re: patches to fix make -j)
To: James Chacon <jmc@NetBSD.org>
From: Todd Vierling <tv@duh.org>
List: tech-toolchain
Date: 12/18/2003 11:25:56
On Thu, 18 Dec 2003, Todd Vierling wrote:

: I had, at one time, a patch that let you do something like
:
: SUBDIRS=	a b:a c:a d e:b
:
: which was indeed parallelizable, and guaranteed that b/c came after a, and e
: came after b for the operations "all", "depend", "dependall", and
: "includes".  This was quite useful in such directories as
: src/gnu/usr.bin/gcc, src/gnu/usr.bin/binutils, src/gnu/libexec/uucp, etc.
: It also demonstrated well how these things can be expressed as make
: dependencies rather than the hacky .WAIT turnstile.

And I found the diff, and updated it to -current; see below.  In more
detail, you can specify something like

SUBDIR=		a:d b:c c d:e:b e

and you will get a guarantee that c builds before b, d builds before a, and
e and b both build before d, regardless of -j setting.  With parallelism,
the above will indeed build c and e in parallel before proceeding with their
dependents.

A real-life example would be to use this change to update
src/gnu/usr.bin/gcc3/Makefile's SUBDIR definition.

SUBDIR=		backend:libiberty:host-libiberty \
		cc1:backend:libcpp \
		cc1obj:backend:libcpp \
		cc1plus:backend:libcpp \
		cpp:frontend \
		f771:backend \
		frontend:libiberty \
		g++:frontend \
		g77:frontend \
		gcc:frontend \
		gcov:frontend \
		host-libiberty \
		include \
		libcpp \
		libiberty \
		protoize:frontend \
		unprotoize:frontend

With the above, it's possible to do "make all-cc1plus" from a clean tree,
and it will work.  You'll also get higher parallelization with a -j build.

There's also a variable DEPTARGETS where additional targets that should get
this treatment can be specified.

=====

--- bsd.subdir.mk	Fri Nov  2 00:21:51 2001
+++ bsd.subdir.mk	Thu Dec 18 11:07:39 2003
@@ -4,12 +4,16 @@
 .include <bsd.init.mk>

 .for dir in ${SUBDIR}
-.if exists(${dir}.${MACHINE})
-__REALSUBDIR+=${dir}.${MACHINE}
+__dir:=${dir:C/:.*$//}
+.if exists(${__dir}.${MACHINE})
+__REALSUBDIR.${__dir}:=${__dir}.${MACHINE}
 .else
-__REALSUBDIR+=${dir}
+__REALSUBDIR.${__dir}:=${__dir}
 .endif
+__REALSUBDIR:=${__REALSUBDIR} ${__REALSUBDIR.${__dir}}
+__deps.${__REALSUBDIR.${__dir}}:=${dir:S/:/ /g:C/.*//1}
 .endfor
+.undef __dir

 __recurse: .USE
 	@targ=${.TARGET:C/-.*$//};dir=${.TARGET:C/^[^-]*-//};		\
@@ -32,6 +36,8 @@
 __RECURSETARG=	${TARGETS}
 .endif

+DEPTARGETS+=	all depend dependall install
+
 # for obscure reasons, we can't do a simple .if ${dir} == ".WAIT"
 # but have to assign to __TARGDIR first.
 .for targ in ${__RECURSETARG}
@@ -43,6 +49,11 @@
 .PHONY: ${targ}-${dir}
 ${targ}-${dir}: .MAKE __recurse
 SUBDIR_${targ} += ${targ}-${dir}
+.if !empty(DEPTARGETS:M${targ})
+.for dep in ${__deps.${dir}}
+${targ}-${dir}: ${targ}-${__REALSUBDIR.${dep:C/:.*$//}}
+.endfor
+.endif
 .endif
 .endfor
 .if defined(__REALSUBDIR)
@@ -51,5 +62,12 @@
 ${targ}: subdir-${targ}
 .endif
 .endfor
+
+.for dir in ${SUBDIR}
+.undef __REALSUBDIR.${dir:C/:.*$//}
+.endfor
+.undef __REALSUBDIR
+.undef __RECURSETARG
+.undef __TARGDIR

 ${TARGETS}:	# ensure existence

-- 
-- Todd Vierling <tv@duh.org> <tv@pobox.com>