Subject: Hitting the correct site for each distfile
To: David Brownlee <abs@netbsd.org>
From: Frederick Bruckman <fredb@immanent.net>
List: tech-pkg
Date: 12/31/2001 11:27:01
Here's a first stab at it. I couldn't get the for loop to expand
correctly within the do-fetch target, so I added another macro,
instead. One unexpected benefit is that "make do-fetch" and "make
fetch-list-one-pkg | sh" will now do exactly the same thing.
The patch also consolidates the MASTER_SITES* and PATCH_SITES*
handling (for the fetch targets), eliminates some mumbo-jumbo over
MASTER_SITE_OVERRIDE (rather, handling it as documented), and on the
whole, leads to a net reduction in line count for bsd.pkg.mk.
I define a new class of variables -- ${SITES_foo.tar.gz} &c -- which
aren't required, however, to be defined in the package Makefile
(though you could do so), so no changes to the body of pkgsrc are
required. If ${MASTER_SITES_foo} and ${PATCH_SITES_foo" were to be
swept from pkgsrc, the assignments to ${SITES_foo} could be pruned,
leading to a further reduction in line count.
The only behavioral change should be, distfiles whose sites are
already specified in ${MASTER_SITES_foo} or ${PATCH_SITES_foo} won't
ever cause bogus hits to ${MASTER_SITES} or ${PATCH_SITES}. Also,
patches won't ever cause hits to ${MASTER_SITES}, and distfiles won't
ever cause hits to ${PATCH_SITES}.
So, what's supposed to happen now, any sites specified by the user in
${MASTER_SITE_OVERRIDE} are always checked first, then the sorted
sites specified in the package's Makefile, then sites in
${MASTER_SITE_BACKUP}, also in the order specified by the user.
There's no need to apply the sort to the override and backup sites, as
they can be easily redefined. OK, so that's a behavioral change -- it
was possible before, to use sort to prefer the backup site to the
package's master site, but I don't see why you'd need that (since you
can always set ${MASTER_SITE_OVERRIDE} to always be hit first). If I'm
wrong about that, it would be fairly trivial to fix. I'm pretty sure
it makes no sense at all to sort ${MASTER_SITE_OVERRIDE}.
Frederick
Index: bsd.pkg.mk
===================================================================
RCS file: /cvsroot/pkgsrc/mk/bsd.pkg.mk,v
retrieving revision 1.886
diff -u -u -r1.886 bsd.pkg.mk
--- bsd.pkg.mk 2001/12/23 19:32:15 1.886
+++ bsd.pkg.mk 2001/12/31 17:24:12
@@ -815,16 +815,6 @@
MASTER_SITE_LOCAL?= \
${MASTER_SITE_BACKUP:=LOCAL_PORTS/}
-# I guess we're in the master distribution business! :) As we gain mirror
-# sites for distfiles, add them to this list.
-.if !defined(MASTER_SITE_OVERRIDE)
-MASTER_SITES+= ${_MASTER_SITE_BACKUP}
-PATCH_SITES+= ${_MASTER_SITE_BACKUP}
-.else
-MASTER_SITES:= ${_MASTER_SITE_OVERRIDE} ${MASTER_SITES}
-PATCH_SITES:= ${_MASTER_SITE_OVERRIDE} ${PATCH_SITES}
-.endif
-
# Derived names so that they're easily overridable.
DISTFILES?= ${DISTNAME}${EXTRACT_SUFX}
.if defined(PKGREVISION) && ${PKGREVISION} != "" && ${PKGREVISION} != "0"
@@ -860,6 +850,7 @@
_IGNOREFILES?= ${IGNOREFILES}
_PATCHFILES?= ${PATCHFILES}
.endif
+_ALLFILES?= ${_DISTFILES} ${_PATCHFILES}
# This is what is actually going to be extracted, and is overridable
# by user.
@@ -1239,33 +1230,48 @@
MASTER_SORT_AWK+= /${srt:C/\//\\\//g}/ { good["${srt}"] = good["${srt}"] " " $$0 ; next; }
.endfor
MASTER_SORT_AWK+= { rest = rest " " $$0; } END { n=split(gl, gla); for(i=1;i<=n;i++) { print good[gla[i]]; } print rest; }
-SORTED_MASTER_SITES_CMD= ${ECHO} '${MASTER_SITES}' | ${AWK} '${MASTER_SORT_AWK}'
-SORTED_PATCH_SITES_CMD= ${ECHO} '${PATCH_SITES}' | ${AWK} '${MASTER_SORT_AWK}'
+SORT_SITES_CMD= ${ECHO} $$unsorted_sites | ${AWK} '${MASTER_SORT_AWK}'
+ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} `${SORT_SITES_CMD}` ${_MASTER_SITE_BACKUP}
+# Set-up for the do-fetch and fetch-list-one-package targets.
+.if !defined(_FETCH_ALLFILES)
+. if !empty(_DISTFILES)
+. for fetchfile in ${_DISTFILES}
+. if defined(MASTER_SITES_${fetchfile:T})
+SITES_${fetchfile:T}?= ${MASTER_SITES_${fetchfile:T}}
+. else
+SITES_${fetchfile:T}?= ${MASTER_SITES}
+. endif
+. endfor
+. endif
+. if !empty(_PATCHFILES)
+. for fetchfile in ${_PATCHFILES}
+. if defined(PATCH_SITES_${fetchfile:T})
+SITES_${fetchfile:T}?= ${PATCH_SITES_${fetchfile:T}}
+. else
+SITES_${fetchfile:T}?= ${PATCH_SITES}
+. endif
+. endfor
+. endif
+. if !empty(_ALLFILES)
+_FETCH_ALLFILES= ${MKDIR} ${_DISTDIR};
+_FETCH_ALLFILES+= cd ${_DISTDIR};
+. for fetchfile in ${_ALLFILES}
+_FETCH_ALLFILES+= \
+ unsorted_sites="${SITES_${fetchfile:T}}"; \
+ sites="${ORDERED_SITES}"; \
+ file="${fetchfile}"; \
+ bfile="${fetchfile:T}"; \
+ ${_CHECK_DIST_PATH}; \
+ ${_FETCH_FILE};
+. endfor
+. endif
+.endif
+
.if !target(do-fetch)
do-fetch:
- ${_PKG_SILENT}${_PKG_DEBUG}${MKDIR} ${_DISTDIR}
-.for fetchfile in ${_DISTFILES}
- ${_PKG_SILENT}${_PKG_DEBUG}cd ${_DISTDIR}; \
- sortedsites=`${SORTED_MASTER_SITES_CMD}`; \
- sites="${MASTER_SITES_${fetchfile:T}} $$sortedsites"; \
- file="${fetchfile}"; \
- bfile="${fetchfile:T}"; \
- ${_CHECK_DIST_PATH}; \
- ${_FETCH_FILE}
-.endfor
-. if defined(_PATCHFILES)
-.for fetchfile in ${_PATCHFILES}
- ${_PKG_SILENT}${_PKG_DEBUG}cd ${_DISTDIR}; \
- sortedsites=`${SORTED_PATCH_SITES_CMD}`; \
- sites="${PATCH_SITES_${fetchfile:T}} $$sortedsites"; \
- file="${fetchfile}"; \
- bfile="${fetchfile:T}"; \
- ${_CHECK_DIST_PATH}; \
- ${_FETCH_FILE}
-.endfor
-. endif
+ ${_PKG_SILENT}${_PKG_DEBUG}${_FETCH_ALLFILES}
.endif
# show both build and run depends directories (non-recursively)
@@ -2599,33 +2605,7 @@
.if !target(fetch-list-one-pkg)
fetch-list-one-pkg:
- @${MKDIR} ${_DISTDIR}
- @[ -z "${_DISTDIR}" ] || ${ECHO} "${MKDIR} ${_DISTDIR}"
-. if defined(DISTFILES)
-.for fetchfile in ${DISTFILES}
- @(cd ${_DISTDIR}; \
- if [ ! -f ${fetchfile} -a ! -f ${fetchfile:T} ]; then \
- ${ECHO} -n "cd ${_DISTDIR} && [ -f ${fetchfile} -o -f ${fetchfile:T} ] || "; \
- for site in "" ${MASTER_SITES_${fetchfile:T}} `${SORTED_MASTER_SITES_CMD}`; do \
- if [ "X$$site" = X"" ]; then continue; fi; \
- ${ECHO} -n ${FETCH_CMD} ${FETCH_BEFORE_ARGS} "'"$${site}${fetchfile}"'" "${FETCH_AFTER_ARGS}" '|| '; \
- done; \
- ${ECHO} "${ECHO} ${fetchfile} not fetched"; \
- fi)
-.endfor
-. endif # DISTFILES
-. if defined(PATCHFILES)
-.for fetchfile in ${PATCHFILE}
- @(cd ${_DISTDIR}; \
- if [ ! -f ${fetchfile} -a ! -f ${fetchfile:T} ]; then \
- ${ECHO} -n "cd ${_DISTDIR} && [ -f ${fetchfile} -o -f ${fetchfile:T} ] || "; \
- for site in ${PATCH_SITES_${fetchfile:T}} `${SORTED_PATCH_SITES_CMD}`; do \
- ${ECHO} -n ${FETCH_CMD} ${FETCH_BEFORE_ARGS} $${site}$${fetchfile} "${FETCH_AFTER_ARGS}" '|| '; \
- done; \
- ${ECHO} "${ECHO} $${fetchfile} not fetched"; \
- fi)
-.endfor
-. endif # defined(PATCHFILES)
+ @${ECHO} ${_FETCH_ALLFILES:Q}
.endif # !target(fetch-list-one-pkg)
# Checksumming utilities