[pkgsrc/trunk]: pkgsrc/mk pkg/34695: Static list of package master sites may ...

[mishka <mishka%pkgsrc.org@localhost>]

details:   https://anonhg.NetBSD.org/pkgsrc/rev/c69a834b98a5
branches:  trunk
changeset: 519706:c69a834b98a5
user:      mishka <mishka%pkgsrc.org@localhost>
date:      Sun Oct 08 11:37:38 2006 +0000

description:
pkg/34695: Static list of package master sites may (and often will)
lead to overloads of very first distribution site. Moreover, if first
site in the list is not available (often seen for sourceforge mirrors)
you have to wait for timeout each time. To distribute load on master
distribution sites and to make second problem not so annoying randomly
intermix list of MASTER_SITES with MASTER_SORT_RANDOM feature. Any of
MASTER_SORT and MASTER_SORT_REGEX can be applied later.

The feature is turned ON by default and is disabled for PKG_DEVELOPERs
or if MASTER_SORT_RANDOM=no.

diffstat:

 mk/defaults/mk.conf |  12 +++++++++++-
 mk/fetch/fetch.mk   |  28 +++++++++++++++++++++-------
 2 files changed, 32 insertions(+), 8 deletions(-)

diffs (85 lines):

diff -r 6bb9164316cf -r c69a834b98a5 mk/defaults/mk.conf
--- a/mk/defaults/mk.conf       Sun Oct 08 08:50:39 2006 +0000
+++ b/mk/defaults/mk.conf       Sun Oct 08 11:37:38 2006 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: mk.conf,v 1.132 2006/10/05 23:35:23 reed Exp $
+# $NetBSD: mk.conf,v 1.133 2006/10/08 11:37:38 mishka Exp $
 #
 
 # This file provides default values for variables that may be overridden
@@ -405,6 +405,16 @@
 # Possible: Regexps as in awk(1)
 # Default: none
 
+.if defined(PKG_DEVELOPER)
+MASTER_SORT_RANDOM?=   NO
+.else
+MASTER_SORT_RANDOM?=   YES
+.endif
+# If set to YES or yes, a list of master sites will be randomly intermixed.
+# Also, both MASTER_SORT and MASTER_SORT_REGEX may be applied later.
+# Possible: yes, no / not defined
+# Default: NO if PKG_DEVELOPER is defined, YES otherwise
+
 #PATCH_DEBUG=
 # Used to debug patches as they are applied
 # Possible: defined, not defined
diff -r 6bb9164316cf -r c69a834b98a5 mk/fetch/fetch.mk
--- a/mk/fetch/fetch.mk Sun Oct 08 08:50:39 2006 +0000
+++ b/mk/fetch/fetch.mk Sun Oct 08 11:37:38 2006 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: fetch.mk,v 1.21 2006/10/07 12:22:06 rillig Exp $
+# $NetBSD: fetch.mk,v 1.22 2006/10/08 11:37:38 mishka Exp $
 
 _MASTER_SITE_BACKUP=   ${MASTER_SITE_BACKUP:=${DIST_SUBDIR}${DIST_SUBDIR:D/}}
 _MASTER_SITE_OVERRIDE= ${MASTER_SITE_OVERRIDE:=${DIST_SUBDIR}${DIST_SUBDIR:D/}}
@@ -30,10 +30,21 @@
 _BUILD_DEFS+=  _DISTFILES _PATCHFILES
 
 # Set up _ORDERED_SITES to work out the exact list of sites for every file,
-# using the dynamic sites script, or sorting according to the master site
-# list or the patterns in MASTER_SORT or MASTER_SORT_REGEX as appropriate.
+# using the dynamic sites script, or ordering according to the master site
+# list, MASTER_SORT_RANDOM randomization feature, or the patterns in
+# MASTER_SORT or MASTER_SORT_REGEX as appropriate.
 # No actual sorting is done until _ORDERED_SITES is expanded.
 #
+.if defined(MASTER_SORT_RANDOM) && !empty(MASTER_SORT_RANDOM:M[yY][eE][sS])
+_MASTER_RAND_AWK= BEGIN { srand(seed) } {                              \
+               n = split($$0, site);                                   \
+               for (i = n; i > 0; i--) {                               \
+                       ir = int(rand() * i + 1);                       \
+                       t = site[i]; site[i] = site[ir]; site[ir] = t;  \
+                       print site[i]; } }
+_RAND_SITES_CMD= | ${AWK} -v seed=$$$$ '${_MASTER_RAND_AWK}'
+.endif
+
 .if defined(MASTER_SORT) || defined(MASTER_SORT_REGEX)
 MASTER_SORT?=
 MASTER_SORT_REGEX?=
@@ -44,9 +55,12 @@
 _MASTER_SORT_AWK+= /${srt:C/\//\\\//g}/ { good["${srt:S/\\/\\\\/g}"] = good["${srt:S/\\/\\\\/g}"] " " $$0 ; next; }
 .  endfor
 _MASTER_SORT_AWK+= { rest = rest " " $$0; } END { n=split(gl, gla); for(i=1;i<=n;i++) { print good[gla[i]]; } print rest; }
+_SORT_SITES_CMD+= | ${AWK} '${_MASTER_SORT_AWK}'
+.endif
 
-_SORT_SITES_CMD= ${ECHO} $$unsorted_sites | ${AWK} '${_MASTER_SORT_AWK}'
-_ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} `${_SORT_SITES_CMD:S/\\/\\\\/g:C/"/\"/g}`
+.if defined(_RAND_SITES_CMD) || defined(_SORT_SITES_CMD)
+_SORT_SITES_FULL_CMD= ${ECHO} $$unsorted_sites ${_RAND_SITES_CMD} ${_SORT_SITES_CMD}
+_ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} `${_SORT_SITES_FULL_CMD:S/\\/\\\\/g:C/"/\"/g}`
 .else
 _ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} $$unsorted_sites
 .endif
@@ -245,8 +259,8 @@
                fi;                                                     \
        done
        ${_PKG_SILENT}${_PKG_DEBUG}set -e;                              \
-       unsorted_sites="${SITES.${.TARGET:T:S/=/--/}} ${_MASTER_SITE_BACKUP}"; \
-       sites="${_ORDERED_SITES}";                                      \
+       unsorted_sites="${SITES.${.TARGET:T:S/=/--/}}";                 \
+       sites="${_ORDERED_SITES} ${_MASTER_SITE_BACKUP}";               \
        cd ${.TARGET:H:S/\/${DIST_SUBDIR}$//} &&                        \
        ${_FETCH_CMD} ${_FETCH_ARGS} ${.TARGET:T} $$sites
        ${_PKG_SILENT}${_PKG_DEBUG}                                     \