pkgsrc-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[pkgsrc/trunk]: pkgsrc/mk Overhaul fetch module:
details: https://anonhg.NetBSD.org/pkgsrc/rev/c661ccdfbfcc
branches: trunk
changeset: 516315:c661ccdfbfcc
user: jlam <jlam%pkgsrc.org@localhost>
date: Tue Jul 18 22:41:06 2006 +0000
description:
Overhaul fetch module:
* All the smarts is now encapsulated in the "fetch" script. The fetch
script understands how to use the distinfo file (if specified) to
look up the size and checksums of the file to fetch and will use
that information to verify checksums of the fetched files or resume
transfers of interrupted fetches.
* Move the default settings for FETCH_RESUME_ARGS and FETCH_OUTPUT_ARGS
for "ftp" from mk/defaults/mk.conf into mk/fetch/fetch.mk. We rewrite
it to avoid needing conditional statements.
* Avoid spawning a new make(1) process just to mirror a distfile.
* Split out fetch-list targets into a separate file fetch-list.mk.
These targets should probably be moved into a standalone script.
* Fix distclean target to properly remove partial downloads.
diffstat:
mk/defaults/mk.conf | 22 +-
mk/fetch/bsd.fetch-vars.mk | 9 +-
mk/fetch/bsd.fetch.mk | 3 +-
mk/fetch/distclean.mk | 15 +-
mk/fetch/fetch | 180 +++++++++++--
mk/fetch/fetch-list.mk | 99 ++++++++
mk/fetch/fetch.mk | 546 ++++++++++++++++----------------------------
7 files changed, 465 insertions(+), 409 deletions(-)
diffs (truncated from 1056 to 300 lines):
diff -r 7085f0a6c7b2 -r c661ccdfbfcc mk/defaults/mk.conf
--- a/mk/defaults/mk.conf Tue Jul 18 22:38:18 2006 +0000
+++ b/mk/defaults/mk.conf Tue Jul 18 22:41:06 2006 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: mk.conf,v 1.124 2006/07/10 17:11:32 wiz Exp $
+# $NetBSD: mk.conf,v 1.125 2006/07/18 22:41:06 jlam Exp $
#
# This file provides default values for variables that may be overridden
@@ -273,26 +273,6 @@
# Default: NetBSD's ftp(1).
# Possible: any.
-.if ${FETCH_CMD:T} == "ftp"
-FETCH_RESUME_ARGS?= -R
-.else
-FETCH_RESUME_ARGS?= # empty
-.endif
-# Used when PKG_RESUME_TRANSFERS is enabled, to specify default argument
-# to resume transfers.
-# Default: -R
-# Possible: depends on your FETCH_CMD value.
-
-.if ${FETCH_CMD:T} == "ftp"
-FETCH_OUTPUT_ARGS?= -o
-.else
-FETCH_OUTPUT_ARGS?= # empty
-.endif
-# Used when PKG_RESUME_TRANSFERS is enabled, to specify default argument
-# in FETCH_CMD to fetch the file to a temporary name.
-# Default: -o
-# Possible: depends on your FETCH_CMD value.
-
LIBTOOLIZE_PLIST?= yes
# This determines whether to expand libtool archives (.la files) in PLISTs
# into the represented library names.
diff -r 7085f0a6c7b2 -r c661ccdfbfcc mk/fetch/bsd.fetch-vars.mk
--- a/mk/fetch/bsd.fetch-vars.mk Tue Jul 18 22:38:18 2006 +0000
+++ b/mk/fetch/bsd.fetch-vars.mk Tue Jul 18 22:41:06 2006 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.fetch-vars.mk,v 1.3 2006/07/13 14:02:34 jlam Exp $
+# $NetBSD: bsd.fetch-vars.mk,v 1.4 2006/07/18 22:41:06 jlam Exp $
#
# This Makefile fragment is included separately by bsd.pkg.mk and
# defines some variables which must be defined earlier than where
@@ -27,3 +27,10 @@
# verify any fetched files.
#
USE_TOOLS+= ${FAILOVER_FETCH:Ddigest\:bootstrap}
+
+# When mirroring distfiles which others may fetch, only fetch the
+# distfiles if it is allowed to be re-distributed freely.
+#
+.if make(mirror-distfiles)
+NO_SKIP= # defined
+.endif
diff -r 7085f0a6c7b2 -r c661ccdfbfcc mk/fetch/bsd.fetch.mk
--- a/mk/fetch/bsd.fetch.mk Tue Jul 18 22:38:18 2006 +0000
+++ b/mk/fetch/bsd.fetch.mk Tue Jul 18 22:41:06 2006 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.fetch.mk,v 1.2 2006/06/06 03:35:42 jlam Exp $
+# $NetBSD: bsd.fetch.mk,v 1.3 2006/07/18 22:41:06 jlam Exp $
#
# This Makefile fragment is included by bsd.pkg.mk and defines the
# relevant variables and targets for the "fetch" step.
@@ -14,4 +14,5 @@
.include "${PKGSRCDIR}/mk/fetch/sites.mk"
.include "${PKGSRCDIR}/mk/fetch/fetch.mk"
+.include "${PKGSRCDIR}/mk/fetch/fetch-list.mk"
.include "${PKGSRCDIR}/mk/fetch/distclean.mk"
diff -r 7085f0a6c7b2 -r c661ccdfbfcc mk/fetch/distclean.mk
--- a/mk/fetch/distclean.mk Tue Jul 18 22:38:18 2006 +0000
+++ b/mk/fetch/distclean.mk Tue Jul 18 22:41:06 2006 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: distclean.mk,v 1.2 2006/07/12 12:34:13 rillig Exp $
+# $NetBSD: distclean.mk,v 1.3 2006/07/18 22:41:06 jlam Exp $
.PHONY: pre-distclean
.if !target(pre-distclean)
@@ -10,14 +10,11 @@
.if !target(distclean)
distclean: pre-distclean clean
@${PHASE_MSG} "Dist cleaning for ${PKGNAME}"
- ${_PKG_SILENT}${_PKG_DEBUG}if [ -d ${_DISTDIR} ]; then \
- cd ${_DISTDIR} && \
- ${TEST} -z "${DISTFILES}" || ${RM} -f ${DISTFILES}; \
- if [ "${PKG_RESUME_TRANSFERS:M[Yy][Ee][Ss]}" ]; then \
- ${TEST} -z "${DISTFILES}.temp" || ${RM} -f ${DISTFILES}.temp; \
- fi; \
- ${TEST} -z "${PATCHFILES}" || ${RM} -f ${PATCHFILES}; \
- fi
+ ${_PKG_SILENT}${_PKG_DEBUG} \
+ ${TEST} -d ${_DISTDIR} || exit 0; \
+ cd ${_DISTDIR}; \
+ ${RM} -f ${ALLFILES}; \
+ ${RM} -f ${ALLFILES:S/$/.pkgsrc.resume/}
. if defined(DIST_SUBDIR)
${_PKG_SILENT}${_PKG_DEBUG} \
${TEST} ! -d ${_DISTDIR} \
diff -r 7085f0a6c7b2 -r c661ccdfbfcc mk/fetch/fetch
--- a/mk/fetch/fetch Tue Jul 18 22:38:18 2006 +0000
+++ b/mk/fetch/fetch Tue Jul 18 22:41:06 2006 +0000
@@ -1,6 +1,6 @@
#!/bin/sh
#
-# $NetBSD: fetch,v 1.1 2006/07/17 15:34:22 jlam Exp $
+# $NetBSD: fetch,v 1.2 2006/07/18 22:41:06 jlam Exp $
#
# Copyright (c) 2006 The NetBSD Foundation, Inc.
# All rights reserved.
@@ -43,49 +43,94 @@
# fetch -- fetch files via URLs
#
# SYNOPSIS
-# fetch [-c] [-f distinfo] file site ..."
+# fetch [-c] [-d dir] [-f distinfo] file site ..."
#
# DESCRIPTION
# fetch will attempt to fetch the file from the list of specified
# sites in the order given. The complete URL to the file on each
# site should be the concatenation of the specified site and file.
-# If the file cannot be fetched successfully, then try the next
+# If the file cannot be fetched successfully, then we try the next
# listed site.
#
+# If the file already exists on the disk and is verified, then
+# no fetch action is taken.
+#
# OPTIONS
# -c Verify the checksum for the file. If the checksum
# does not match, then the fetch is determined to be
# not sucessful.
#
+# -d dir Fetch the files into the specified directory.
+#
# -f distinfo
# The path to the distinfo file containing the checksums
# for the file. The file format should match what is
# needed by the pkgsrc/mk/checksum/checksum script.
#
+# -r Resume a previous fetch for the file. In this case,
+# the file is first saved to a ".pkgsrc.resume" file,
+# and is later renamed to the final file name if the
+# complete file has been sucessfully fetched.
+#
+# ENVIRONMENT
+# PKGSRCDIR This is a hint to help locate the default
+# checksum script.
+#
+# CHECKSUM This is the path to the checksum script used
+# when "-c" is specified.
+#
+# FETCH_CMD This is the actual command used for transferring
+# files from the various sites.
+#
+# The following are lists of options to pass to ${FETCH_CMD}:
+#
+# FETCH_BEFORE_ARGS
+# These options appear before all other options.
+#
+# FETCH_AFTER_ARGS
+# These options appear after all other options.
+#
+# FETCH_RESUME_ARGS
+# These options appear just after FETCH_BEFORE_ARGS
+# options and cause ${FETCH_CMD} to resume a
+# previous file transfer.
+#
+# FETCH_OUTPUT_ARGS
+# These options specify the name of the local file
+# that will hold the contents of the fetched file.
+#
######################################################################
set -e # exit on errors
-: ${CHECKSUM:=checksum}
+: ${PKGSRCDIR:=/usr/pkgsrc}
+: ${CHECKSUM:=${PKGSRCDIR}/mk/checksum/checksum}
+: ${CP:=cp}
: ${ECHO:=echo}
: ${FETCH_CMD:=ftp}
-: ${RM:=rm}
+: ${MKDIR:=mkdir}
+: ${MV:=mv}
: ${TEST:=test}
+: ${TOUCH:=touch}
: ${WC:=wc}
self="${0##*/}"
usage() {
- ${ECHO} 1>&2 "usage: $self [-c] [-f distinfo] file site ..."
+ ${ECHO} 1>&2 "usage: $self [-c] [-f distinfo] [-r] file site ..."
}
# Process optional arguments
checksum=
distinfo=
+fetchdir=.
+resume=
while ${TEST} $# -gt 0; do
case "$1" in
-c) checksum=yes; shift ;;
+ -d) fetchdir="$2"; shift 2 ;;
-f) distinfo="$2"; shift 2 ;;
+ -r) resume=yes; shift ;;
--) shift; break ;;
-*) ${ECHO} 1>&2 "$self: unknown option -- ${1#-}"
usage
@@ -95,12 +140,19 @@
esac
done
if ${TEST} -n "$checksum" -a -z "$distinfo"; then
- ${ECHO} 1>&2 "$self: \`\`-c'' requires \``-f distinfo''"
+ ${ECHO} 1>&2 "$self: \`\`-c'' requires \`\`-f distinfo''."
exit 1
fi
-if ${TEST} -n "$resume" -a ! -z "$distinfo"; then
- ${ECHO} 1>&2 "$self: \`\`-r'' requires \``-f distinfo''"
- exit 1
+if ${TEST} -n "$resume"; then
+ if ${TEST} -z "$distinfo"; then
+ ${ECHO} 1>&2 "$self: \`\`-r'' requires \`\`-f distinfo''."
+ resume=
+ elif ${TEST} "x${FETCH_RESUME_ARGS}" = "x"; then
+ ${ECHO} 1>&2 "$self: \`\`-r'' requires FETCH_RESUME_ARGS to be non-empty."
+ resume=
+ fi
+ ${TEST} -n "$resume" ||
+ ${ECHO} 1>&2 "$self: Falling back to non-resume fetch."
fi
# Process required arguments
@@ -109,16 +161,16 @@
exit 1
fi
file="$1"; shift
+path="$fetchdir/$file"
if ${TEST} -n "$distinfo" && ${TEST} ! -f "$distinfo"; then
${ECHO} 1>&2 "$self: distinfo file missing: $distinfo"
exit 1
fi
-dir="${file%/*}"
-${TEST} "$dir" != "$file" || dir=.
-if ${TEST} ! -w "$dir/."; then
- ${ECHO} 1>&2: "$self: Cannot write to $dir"
+${TEST} -d $fetchdir || ${MKDIR} -p $fetchdir 2>/dev/null
+if ${TEST} ! -w $fetchdir; then
+ ${ECHO} 1>&2 "$self: Cannot write to $fetchdir"
exit 1
fi
@@ -131,42 +183,100 @@
Size) ;; # only handle "Size" lines
*) continue ;;
esac
- ${TEST} "d_file" != "($file)" || continue
+ case "$fetchdir" in
+ ".") ${TEST} "$d_file" = "($file)" || continue ;;
+ *) ${TEST} "$d_file" = "($path)" || continue ;;
+ esac
distsize="$d_size"; distunits="$d_units"
break
done < $distinfo
fi
-checksum_file() {
- _file="$1"
- if ${TEST} -z "$checksum" || ${CHECKSUM} $distinfo $_file; then
- return 0;
+# verify_file $file
+# If we can checksum the file, then see if it matches the listed
+# checksums in the distinfo file. If we can check the size, then
+# check that instead. We strip off ".pkgsrc.resume" from the
+# filename so that we can verify the checksum for the temporary
+# fetch file as well.
+#
+verify_file() {
+ _file="${1#./}"
+ ${TEST} -f $_file || return 1
+ if ${TEST} -n "$checksum"; then
+ ${CHECKSUM} -s ".pkgsrc.resume" $distinfo ${_file} || return 1
+ return 0
+ elif ${TEST} -n "$distsize"; then
+ _size=`${WC} -c < $_file`
+ ${TEST} "$_size" -eq "$distsize" || return 1
+ return 0
fi
- return 1
+ return 0;
}
-bfile="${file##*/}"
+# If the file already exists and it verifies, then we don't need to fetch
+# it again.
+#
Home |
Main Index |
Thread Index |
Old Index