On Tue, 1 Jan 2002, Thomas Klausner wrote:

> > I define a new class of variables -- ${SITES_foo.tar.gz} &c -- which
> > aren't required, however, to be defined in the package Makefile
> > (though you could do so), so no changes to the body of pkgsrc are
> > required. If ${MASTER_SITES_foo} and ${PATCH_SITES_foo" were to be
> > swept from pkgsrc, the assignments to ${SITES_foo} could be pruned,
> > leading to a further reduction in line count.
>
> Please do so. AFAIK (and grep doesn't tell me otherwise) the only two
> Makefiles to "sweep" through would be
> print/ghostscript-nox11/Makefile.common and x11/kterm/Makefile.

That's cool. It's still no big deal to do this in two stages.

> These two make good test cases, too :)

I want to play with this a bit more, because it is such a big change.
The "fetch-list" target, especially, is and has always been abominably
slow for any package with a great number of depends, so I haven't been
able to test that extensively. Here's the latest:

1) I added a "does nothing" command to the _FETCH_ALLFILES macro in
the no files case, to keep it from blowing up for digest, gnome, &c.

2) Put MASTER_SITE_BACKUP back into the mix for MASTER_SORT, and
MASTER_SORT_REGEX, as I seem to remember now that that was a requested
feature. Added a minor optimization, to not call "awk" to sort at all,
in the case where MASTER_SORT and MASTER_SORT_REGEX are not defined.

3) Moved the call to "fetch-list-one-pkg" to the end of
"fetch-list-recursive", so that the depends are fetched first.

4) Cleaned up the comments in that area.

I'd appreciate it if someone else would actually test this, and let me
know if there are any problems! [I've set Reply-To to me.] I'll commit
this weekend, unless I find I've overlooked some problem before then.

Frederick

Index: bsd.pkg.mk
===================================================================
RCS file: /cvsroot/pkgsrc/mk/bsd.pkg.mk,v
retrieving revision 1.886
diff -u -r1.886 bsd.pkg.mk
--- bsd.pkg.mk	2001/12/23 19:32:15	1.886
+++ bsd.pkg.mk	2002/01/01 19:35:16
@@ -791,10 +791,6 @@
 	http://ftp1.sourceforge.net/ \
 	ftp://ftp.tuwien.ac.at/opsys/linux/sourceforge/

-# Empty declaration to avoid "variable MASTER_SITES recursive" error
-MASTER_SITES?=
-PATCH_SITES?=
-
 # The primary backup site.
 MASTER_SITE_BACKUP?=	\
 	ftp://ftp.netbsd.org/pub/NetBSD/packages/distfiles/ \
@@ -815,16 +811,6 @@
 MASTER_SITE_LOCAL?= \
 	${MASTER_SITE_BACKUP:=LOCAL_PORTS/}

-# I guess we're in the master distribution business! :)  As we gain mirror
-# sites for distfiles, add them to this list.
-.if !defined(MASTER_SITE_OVERRIDE)
-MASTER_SITES+=	${_MASTER_SITE_BACKUP}
-PATCH_SITES+=	${_MASTER_SITE_BACKUP}
-.else
-MASTER_SITES:=	${_MASTER_SITE_OVERRIDE} ${MASTER_SITES}
-PATCH_SITES:=	${_MASTER_SITE_OVERRIDE} ${PATCH_SITES}
-.endif
-
 # Derived names so that they're easily overridable.
 DISTFILES?=		${DISTNAME}${EXTRACT_SUFX}
 .if defined(PKGREVISION) && ${PKGREVISION} != "" && ${PKGREVISION} != "0"
@@ -860,6 +846,7 @@
 _IGNOREFILES?=	${IGNOREFILES}
 _PATCHFILES?=	${PATCHFILES}
 .endif
+_ALLFILES?=	${_DISTFILES} ${_PATCHFILES}

 # This is what is actually going to be extracted, and is overridable
 #  by user.
@@ -1177,7 +1164,9 @@
 # adding pre-* or post-* targets/scripts, override these.
 ################################################################

-# Fetch
+#
+# Define the elementary fetch macros.
+#
 _FETCH_FILE=								\
 	if [ ! -f $$file -a ! -f $$bfile -a ! -h $$bfile ]; then	\
 		${ECHO_MSG} "=> $$bfile doesn't seem to exist on this system."; \
@@ -1228,8 +1217,9 @@
 	fi

 #
-# Sort the master site list according to the patterns in MASTER_SORT
+# Sort the master site list according to the patterns in MASTER_SORT.
 #
+.if defined(MASTER_SORT) || defined(MASTER_SORT_REGEX)
 MASTER_SORT?=
 MASTER_SORT_REGEX?=
 MASTER_SORT_REGEX+= ${MASTER_SORT:S/./\\./g:C/.*/:\/\/[^\/]*&\//}
@@ -1239,33 +1229,54 @@
 MASTER_SORT_AWK+= /${srt:C/\//\\\//g}/ { good["${srt}"] = good["${srt}"] " " $$0 ; next; }
 .endfor
 MASTER_SORT_AWK+= { rest = rest " " $$0; } END { n=split(gl, gla); for(i=1;i<=n;i++) { print good[gla[i]]; } print rest; }
-SORTED_MASTER_SITES_CMD= ${ECHO} '${MASTER_SITES}' | ${AWK} '${MASTER_SORT_AWK}'
-SORTED_PATCH_SITES_CMD= ${ECHO} '${PATCH_SITES}' | ${AWK} '${MASTER_SORT_AWK}'

+SORT_SITES_CMD= ${ECHO} $$unsorted_sites | ${AWK} '${MASTER_SORT_AWK}'
+ORDERED_SITES= ${MASTER_SITE_OVERRIDE} `${SORT_SITES_CMD}`
+.else
+ORDERED_SITES= ${MASTER_SITE_OVERRIDE} $$unsorted_sites
+.endif

-.if !target(do-fetch)
-do-fetch:
-	${_PKG_SILENT}${_PKG_DEBUG}${MKDIR} ${_DISTDIR}
-.for fetchfile in ${_DISTFILES}
-	${_PKG_SILENT}${_PKG_DEBUG}cd ${_DISTDIR};			\
-	sortedsites=`${SORTED_MASTER_SITES_CMD}`;			\
-	sites="${MASTER_SITES_${fetchfile:T}} $$sortedsites";		\
-	file="${fetchfile}";						\
-	bfile="${fetchfile:T}";						\
-	${_CHECK_DIST_PATH};						\
-	${_FETCH_FILE}
-.endfor
-.  if defined(_PATCHFILES)
-.for fetchfile in ${_PATCHFILES}
-	${_PKG_SILENT}${_PKG_DEBUG}cd ${_DISTDIR};			\
-	sortedsites=`${SORTED_PATCH_SITES_CMD}`;			\
-	sites="${PATCH_SITES_${fetchfile:T}} $$sortedsites";		\
+#
+# Complete macro for the do-fetch and fetch-list-one-package targets.
+#
+.if !defined(_FETCH_ALLFILES)
+.  if !empty(_DISTFILES)
+.    for fetchfile in ${_DISTFILES}
+.      if defined(MASTER_SITES_${fetchfile:T})
+SITES_${fetchfile:T}?= ${MASTER_SITES_${fetchfile:T}}
+.      else
+SITES_${fetchfile:T}?= ${MASTER_SITES}
+.      endif
+.    endfor
+.  endif
+.  if !empty(_PATCHFILES)
+.    for fetchfile in ${_PATCHFILES}
+.      if defined(PATCH_SITES_${fetchfile:T})
+SITES_${fetchfile:T}?= ${PATCH_SITES_${fetchfile:T}}
+.      else
+SITES_${fetchfile:T}?= ${PATCH_SITES}
+.      endif
+.    endfor
+.  endif
+.  if !empty(_ALLFILES)
+_FETCH_ALLFILES= ${MKDIR} ${_DISTDIR};
+_FETCH_ALLFILES+= cd ${_DISTDIR};
+.    for fetchfile in ${_ALLFILES}
+_FETCH_ALLFILES+= 							\
+	unsorted_sites="${SITES_${fetchfile:T}} ${MASTER_SITE_BACKUP}";	\
+	sites="${ORDERED_SITES}";					\
 	file="${fetchfile}";						\
 	bfile="${fetchfile:T}";						\
 	${_CHECK_DIST_PATH};						\
-	${_FETCH_FILE}
-.endfor
+	${_FETCH_FILE};
+.    endfor
 .  endif
+_FETCH_ALLFILES?= ${DO_NADA}
+.endif
+
+.if !target(do-fetch)
+do-fetch:
+	${_PKG_SILENT}${_PKG_DEBUG}${_FETCH_ALLFILES}
 .endif

 # show both build and run depends directories (non-recursively)
@@ -2585,7 +2596,6 @@

 .if !target(fetch-list-recursive)
 fetch-list-recursive:
-	@${MAKE} ${MAKEFLAGS} fetch-list-one-pkg
 .  if ${RECURSIVE_FETCH_LIST} != "NO"
 	${_PKG_SILENT}${_PKG_DEBUG}					\
 	for dir in `${ECHO} ${BUILD_DEPENDS:C/^[^:]*://:C/:.*//}	\
@@ -2595,37 +2605,12 @@
 		${MAKE} ${MAKEFLAGS} fetch-list-recursive;		\
 	done
 .  endif # ${RECURSIVE_FETCH_LIST} != "NO"
+	@${MAKE} ${MAKEFLAGS} fetch-list-one-pkg
 .endif # !target(fetch-list-recursive)

 .if !target(fetch-list-one-pkg)
 fetch-list-one-pkg:
-	@${MKDIR} ${_DISTDIR}
-	@[ -z "${_DISTDIR}" ] || ${ECHO} "${MKDIR} ${_DISTDIR}"
-.  if defined(DISTFILES)
-.for fetchfile in ${DISTFILES}
-	@(cd ${_DISTDIR};						\
-	if [ ! -f ${fetchfile} -a ! -f ${fetchfile:T} ]; then		\
-		${ECHO} -n "cd ${_DISTDIR} && [ -f ${fetchfile} -o -f ${fetchfile:T} ] || "; \
-		for site in "" ${MASTER_SITES_${fetchfile:T}} `${SORTED_MASTER_SITES_CMD}`; do	\
-			if [ "X$$site" = X"" ]; then continue; fi; 	\
-			${ECHO} -n ${FETCH_CMD} ${FETCH_BEFORE_ARGS} "'"$${site}${fetchfile}"'" "${FETCH_AFTER_ARGS}" '|| '; \
-		done;							\
-		${ECHO} "${ECHO} ${fetchfile} not fetched";		\
-	fi)
-.endfor
-.  endif # DISTFILES
-.  if defined(PATCHFILES)
-.for fetchfile in ${PATCHFILE}
-	@(cd ${_DISTDIR};						\
-	if [ ! -f ${fetchfile} -a ! -f ${fetchfile:T} ]; then		\
-		${ECHO} -n "cd ${_DISTDIR} && [ -f ${fetchfile} -o -f ${fetchfile:T} ] || "; \
-		for site in ${PATCH_SITES_${fetchfile:T}} `${SORTED_PATCH_SITES_CMD}`; do		\
-			${ECHO} -n ${FETCH_CMD} ${FETCH_BEFORE_ARGS} $${site}$${fetchfile} "${FETCH_AFTER_ARGS}" '|| '; \
-		done;							\
-		${ECHO} "${ECHO} $${fetchfile} not fetched";		\
-	fi)
-.endfor
-.  endif # defined(PATCHFILES)
+	@${ECHO} ${_FETCH_ALLFILES:Q}
 .endif # !target(fetch-list-one-pkg)

 # Checksumming utilities