Subject: Hitting the correct site for each distfile
To: David Brownlee <abs@netbsd.org>
From: Frederick Bruckman <fredb@immanent.net>
List: tech-pkg
Date: 12/31/2001 11:27:01
Here's a first stab at it. I couldn't get the for loop to expand
correctly within the do-fetch target, so I added another macro,
instead. One unexpected benefit is that "make do-fetch" and "make
fetch-list-one-pkg | sh" will now do exactly the same thing.

The patch also consolidates the MASTER_SITES* and PATCH_SITES*
handling (for the fetch targets), eliminates some mumbo-jumbo over
MASTER_SITE_OVERRIDE (rather, handling it as documented), and on the
whole, leads to a net reduction in line count for bsd.pkg.mk.

I define a new class of variables -- ${SITES_foo.tar.gz} &c -- which
aren't required, however, to be defined in the package Makefile
(though you could do so), so no changes to the body of pkgsrc are
required. If ${MASTER_SITES_foo} and ${PATCH_SITES_foo" were to be
swept from pkgsrc, the assignments to ${SITES_foo} could be pruned,
leading to a further reduction in line count.

The only behavioral change should be, distfiles whose sites are
already specified in ${MASTER_SITES_foo} or ${PATCH_SITES_foo} won't
ever cause bogus hits to ${MASTER_SITES} or ${PATCH_SITES}. Also,
patches won't ever cause hits to ${MASTER_SITES}, and distfiles won't
ever cause hits to ${PATCH_SITES}.

So, what's supposed to happen now, any sites specified by the user in
${MASTER_SITE_OVERRIDE} are always checked first, then the sorted
sites specified in the package's Makefile, then sites in
${MASTER_SITE_BACKUP}, also in the order specified by the user.
There's no need to apply the sort to the override and backup sites, as
they can be easily redefined. OK, so that's a behavioral change -- it
was possible before, to use sort to prefer the backup site to the
package's master site, but I don't see why you'd need that (since you
can always set ${MASTER_SITE_OVERRIDE} to always be hit first). If I'm
wrong about that, it would be fairly trivial to fix. I'm pretty sure
it makes no sense at all to sort ${MASTER_SITE_OVERRIDE}.

Frederick

Index: bsd.pkg.mk
===================================================================
RCS file: /cvsroot/pkgsrc/mk/bsd.pkg.mk,v
retrieving revision 1.886
diff -u -u -r1.886 bsd.pkg.mk
--- bsd.pkg.mk	2001/12/23 19:32:15	1.886
+++ bsd.pkg.mk	2001/12/31 17:24:12
@@ -815,16 +815,6 @@
 MASTER_SITE_LOCAL?= \
 	${MASTER_SITE_BACKUP:=LOCAL_PORTS/}

-# I guess we're in the master distribution business! :)  As we gain mirror
-# sites for distfiles, add them to this list.
-.if !defined(MASTER_SITE_OVERRIDE)
-MASTER_SITES+=	${_MASTER_SITE_BACKUP}
-PATCH_SITES+=	${_MASTER_SITE_BACKUP}
-.else
-MASTER_SITES:=	${_MASTER_SITE_OVERRIDE} ${MASTER_SITES}
-PATCH_SITES:=	${_MASTER_SITE_OVERRIDE} ${PATCH_SITES}
-.endif
-
 # Derived names so that they're easily overridable.
 DISTFILES?=		${DISTNAME}${EXTRACT_SUFX}
 .if defined(PKGREVISION) && ${PKGREVISION} != "" && ${PKGREVISION} != "0"
@@ -860,6 +850,7 @@
 _IGNOREFILES?=	${IGNOREFILES}
 _PATCHFILES?=	${PATCHFILES}
 .endif
+_ALLFILES?=	${_DISTFILES} ${_PATCHFILES}

 # This is what is actually going to be extracted, and is overridable
 #  by user.
@@ -1239,33 +1230,48 @@
 MASTER_SORT_AWK+= /${srt:C/\//\\\//g}/ { good["${srt}"] = good["${srt}"] " " $$0 ; next; }
 .endfor
 MASTER_SORT_AWK+= { rest = rest " " $$0; } END { n=split(gl, gla); for(i=1;i<=n;i++) { print good[gla[i]]; } print rest; }
-SORTED_MASTER_SITES_CMD= ${ECHO} '${MASTER_SITES}' | ${AWK} '${MASTER_SORT_AWK}'
-SORTED_PATCH_SITES_CMD= ${ECHO} '${PATCH_SITES}' | ${AWK} '${MASTER_SORT_AWK}'

+SORT_SITES_CMD= ${ECHO} $$unsorted_sites | ${AWK} '${MASTER_SORT_AWK}'
+ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} `${SORT_SITES_CMD}` ${_MASTER_SITE_BACKUP}

+# Set-up for the do-fetch and fetch-list-one-package targets.
+.if !defined(_FETCH_ALLFILES)
+.  if !empty(_DISTFILES)
+.    for fetchfile in ${_DISTFILES}
+.      if defined(MASTER_SITES_${fetchfile:T})
+SITES_${fetchfile:T}?= ${MASTER_SITES_${fetchfile:T}}
+.      else
+SITES_${fetchfile:T}?= ${MASTER_SITES}
+.      endif
+.    endfor
+.  endif
+.  if !empty(_PATCHFILES)
+.    for fetchfile in ${_PATCHFILES}
+.      if defined(PATCH_SITES_${fetchfile:T})
+SITES_${fetchfile:T}?= ${PATCH_SITES_${fetchfile:T}}
+.      else
+SITES_${fetchfile:T}?= ${PATCH_SITES}
+.      endif
+.    endfor
+.  endif
+.  if !empty(_ALLFILES)
+_FETCH_ALLFILES= ${MKDIR} ${_DISTDIR};
+_FETCH_ALLFILES+= cd ${_DISTDIR};
+.    for fetchfile in ${_ALLFILES}
+_FETCH_ALLFILES+= 					\
+	unsorted_sites="${SITES_${fetchfile:T}}";	\
+	sites="${ORDERED_SITES}";			\
+	file="${fetchfile}";				\
+	bfile="${fetchfile:T}";				\
+	${_CHECK_DIST_PATH};				\
+	${_FETCH_FILE};
+.    endfor
+.  endif
+.endif
+
 .if !target(do-fetch)
 do-fetch:
-	${_PKG_SILENT}${_PKG_DEBUG}${MKDIR} ${_DISTDIR}
-.for fetchfile in ${_DISTFILES}
-	${_PKG_SILENT}${_PKG_DEBUG}cd ${_DISTDIR};			\
-	sortedsites=`${SORTED_MASTER_SITES_CMD}`;			\
-	sites="${MASTER_SITES_${fetchfile:T}} $$sortedsites";		\
-	file="${fetchfile}";						\
-	bfile="${fetchfile:T}";						\
-	${_CHECK_DIST_PATH};						\
-	${_FETCH_FILE}
-.endfor
-.  if defined(_PATCHFILES)
-.for fetchfile in ${_PATCHFILES}
-	${_PKG_SILENT}${_PKG_DEBUG}cd ${_DISTDIR};			\
-	sortedsites=`${SORTED_PATCH_SITES_CMD}`;			\
-	sites="${PATCH_SITES_${fetchfile:T}} $$sortedsites";		\
-	file="${fetchfile}";						\
-	bfile="${fetchfile:T}";						\
-	${_CHECK_DIST_PATH};						\
-	${_FETCH_FILE}
-.endfor
-.  endif
+	${_PKG_SILENT}${_PKG_DEBUG}${_FETCH_ALLFILES}
 .endif

 # show both build and run depends directories (non-recursively)
@@ -2599,33 +2605,7 @@

 .if !target(fetch-list-one-pkg)
 fetch-list-one-pkg:
-	@${MKDIR} ${_DISTDIR}
-	@[ -z "${_DISTDIR}" ] || ${ECHO} "${MKDIR} ${_DISTDIR}"
-.  if defined(DISTFILES)
-.for fetchfile in ${DISTFILES}
-	@(cd ${_DISTDIR};						\
-	if [ ! -f ${fetchfile} -a ! -f ${fetchfile:T} ]; then		\
-		${ECHO} -n "cd ${_DISTDIR} && [ -f ${fetchfile} -o -f ${fetchfile:T} ] || "; \
-		for site in "" ${MASTER_SITES_${fetchfile:T}} `${SORTED_MASTER_SITES_CMD}`; do	\
-			if [ "X$$site" = X"" ]; then continue; fi; 	\
-			${ECHO} -n ${FETCH_CMD} ${FETCH_BEFORE_ARGS} "'"$${site}${fetchfile}"'" "${FETCH_AFTER_ARGS}" '|| '; \
-		done;							\
-		${ECHO} "${ECHO} ${fetchfile} not fetched";		\
-	fi)
-.endfor
-.  endif # DISTFILES
-.  if defined(PATCHFILES)
-.for fetchfile in ${PATCHFILE}
-	@(cd ${_DISTDIR};						\
-	if [ ! -f ${fetchfile} -a ! -f ${fetchfile:T} ]; then		\
-		${ECHO} -n "cd ${_DISTDIR} && [ -f ${fetchfile} -o -f ${fetchfile:T} ] || "; \
-		for site in ${PATCH_SITES_${fetchfile:T}} `${SORTED_PATCH_SITES_CMD}`; do		\
-			${ECHO} -n ${FETCH_CMD} ${FETCH_BEFORE_ARGS} $${site}$${fetchfile} "${FETCH_AFTER_ARGS}" '|| '; \
-		done;							\
-		${ECHO} "${ECHO} $${fetchfile} not fetched";		\
-	fi)
-.endfor
-.  endif # defined(PATCHFILES)
+	@${ECHO} ${_FETCH_ALLFILES:Q}
 .endif # !target(fetch-list-one-pkg)

 # Checksumming utilities