tech-pkg archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
checksum argmax fix
Ok I've attached a patch that appears to work for me, at least now
I'm able to 'bmake checksum' in wip/grafana, though it still doesn't
help systems that have older bmake without the temporary file fix.
Note for anyone testing mktool I've also added support for checksums,
and the latest version (0.1.8) also now supports -I to match this patch.
On my NetBSD VM with the patch applied:
$ cd wip/grafana
$ time bmake checksum | md5
5c186d75c658147ade1d6075558150ab
real 0m23.549s
user 0m17.799s
sys 0m6.153s
$ time bmake TOOLS_PLATFORM.mktool=mktool checksum | md5
5c186d75c658147ade1d6075558150ab
real 0m5.538s
user 0m17.401s
sys 0m19.618s
Testing appreciated! Also if anyone knows of any systems where we're
likely to even have troubles with argmax of 64K then let me know and I
can reduce the batch size even further.
Cheers,
--
Jonathan Perkin - mnx.io - pkgsrc.smartos.org
Open Source Complete Cloud www.tritondatacenter.com
diff --git a/mk/checksum/checksum.awk b/mk/checksum/checksum.awk
index 7bdd141af47c..b1dfe1dcc8c5 100755
--- a/mk/checksum/checksum.awk
+++ b/mk/checksum/checksum.awk
@@ -47,9 +47,11 @@ BEGIN {
# Retain output compatible with previous "checksum" shell script
progname = "checksum"
+ arg_count = 0
only_alg = ""
distinfo = ""
exitcode = 0
+ input = ""
patch = 0
suffix = ""
@@ -57,6 +59,11 @@ BEGIN {
opt = ARGV[arg]
if (opt == "-a") {
only_alg = ARGV[++arg]
+ } else if (opt == "-I") {
+ infile = ARGV[++arg]
+ while (getline < infile) {
+ arg_list[arg_count++] = $0
+ }
} else if (opt == "-p") {
patch = 1
} else if (opt == "-s") {
@@ -93,7 +100,11 @@ BEGIN {
# in patch mode (-p).
#
while (arg < ARGC) {
- distfile = ARGV[arg++]
+ arg_list[arg_count++] = ARGV[arg++]
+ }
+ i = 0
+ while (i < arg_count) {
+ distfile = arg_list[i++]
sfile = distfile
if (suffix) {
sfile = strip_suffix(sfile)
@@ -167,12 +178,30 @@ BEGIN {
continue
}
+ #
+ # Due to command line argument limits we sometimes need to
+ # split commands up. Store a count for each algorithm, which
+ # also serves as a way to iterate over all of the algorithms
+ # we've encountered.
+ #
+ if (!batch[algorithm]) {
+ batch[algorithm] = 0
+ }
+
#
# If not a patch file, then we're handling a distfile, where we
# want to build a list of input files to digest(1) so they can
# all be calculated in one go.
#
- distsums[algorithm] = sprintf("%s %s", distsums[algorithm],
+ # Increase the batch number if over 64K. This is well below the
+ # limits seen in the wild (e.g. NetBSD at 256K), but there are
+ # very minimal improvements above this threshold in testing.
+ #
+ b = batch[algorithm]
+ if (length(distsums[algorithm,b]) > 65536) {
+ batch[algorithm] = ++b
+ }
+ distsums[algorithm,b] = sprintf("%s %s", distsums[algorithm,b],
distfiles[distfile])
}
close(distinfo)
@@ -182,23 +211,26 @@ BEGIN {
# pass them all to a single digest(1) command and parse the checksums
# to be compared against distinfo.
#
- for (algorithm in distsums) {
- cmd = sprintf("%s %s %s", DIGEST, algorithm,
- distsums[algorithm])
- while ((cmd | getline) > 0) {
- # Should be unnecessary, but just in case. If we want
- # to be really paranoid then test that $1 == algorithm.
- if (NF != 4) {
- continue
- }
- # strip "(filename)" -> "filename"
- distfile = substr($2, 2, length($2) - 2)
- if (suffix) {
- distfile = strip_suffix(distfile)
+ for (algorithm in batch) {
+ for (b = 0; b <= batch[algorithm]; b++) {
+ cmd = sprintf("%s %s %s", DIGEST, algorithm,
+ distsums[algorithm,b])
+ while ((cmd | getline) > 0) {
+ # Should be unnecessary, but just in case. If
+ # we want to be really paranoid then test that
+ # $1 == algorithm.
+ if (NF != 4) {
+ continue
+ }
+ # strip "(filename)" -> "filename"
+ distfile = substr($2, 2, length($2) - 2)
+ if (suffix) {
+ distfile = strip_suffix(distfile)
+ }
+ checksums[$1, distfile] = $4
}
- checksums[$1, distfile] = $4
+ close(cmd)
}
- close(cmd)
}
#
diff --git a/mk/checksum/checksum.mk b/mk/checksum/checksum.mk
index 7cafe84dac68..323ccf306bfd 100644
--- a/mk/checksum/checksum.mk
+++ b/mk/checksum/checksum.mk
@@ -25,11 +25,14 @@ _PATCH_DIGEST_ALGORITHMS?= SHA1
#
_COOKIE.checksum= ${_COOKIE.extract}
+.if defined(TOOLS_PLATFORM.mktool)
+_CHECKSUM_CMD= ${TOOLS_PLATFORM.mktool} checksum
+.else
_CHECKSUM_CMD= \
${PKGSRC_SETENV} \
DIGEST=${TOOLS_DIGEST:Q} SED=${TOOLS_CMDLINE_SED:Q} \
${AWK} -f ${PKGSRCDIR}/mk/checksum/checksum.awk --
-
+.endif
.if defined(NO_CHECKSUM) || empty(_CKSUMFILES)
checksum checksum-phase:
@@ -40,7 +43,9 @@ checksum checksum-phase:
case ${.TARGET:Q} in \
*-phase) ${TEST} ! -f ${_COOKIE.checksum} || exit 0 ;; \
esac; \
- if cd ${DISTDIR} && ${_CHECKSUM_CMD} ${DISTINFO_FILE:tA} ${_CKSUMFILES}; then \
+ cd ${DISTDIR}; \
+ if { ${_CKSUMFILES:@f@ ${ECHO} ${f};@} } \
+ | ${_CHECKSUM_CMD} -I /dev/stdin ${DISTINFO_FILE:tA}; then \
${TRUE}; \
else \
${ERROR_MSG} "Make sure the Makefile and checksum file (${DISTINFO_FILE})"; \
@@ -50,9 +55,13 @@ checksum checksum-phase:
fi
.endif
+.if defined(TOOLS_PLATFORM.mktool)
+_DISTINFO_CMD= ${TOOLS_PLATFORM.mktool} distinfo
+.else
_DISTINFO_CMD= ${PKGSRC_SETENV} DIGEST=${TOOLS_DIGEST:Q} SED=${TOOLS_SED:Q} \
TEST=${TOOLS_TEST:Q} WC=${TOOLS_WC:Q} \
${AWK} -f ${PKGSRCDIR}/mk/checksum/distinfo.awk --
+.endif
.if exists(${DISTDIR})
_DISTINFO_ARGS_COMMON+= -d ${DISTDIR}
@@ -73,13 +82,11 @@ _DISTINFO_ARGS_PATCHSUM+= ${PATCHDIR}/emul-*-patch-*
_DISTINFO_INPUTFILE= ${DISTINFO_FILE}.filelist
distinfo:
-.for file in ${_CKSUMFILES}
- @${ECHO} ${file} >> ${_DISTINFO_INPUTFILE}
-.endfor
${RUN}set -e; \
newfile=${DISTINFO_FILE}.$$$$; \
- if ${_DISTINFO_CMD} ${_DISTINFO_ARGS_COMMON} \
- -I ${_DISTINFO_INPUTFILE} ${_DISTINFO_ARGS_PATCHSUM} > $$newfile; \
+ if { ${_CKSUMFILES:@f@ ${ECHO} ${f};@} } \
+ | ${_DISTINFO_CMD} ${_DISTINFO_ARGS_COMMON} \
+ -I /dev/stdin ${_DISTINFO_ARGS_PATCHSUM} > $$newfile; \
then \
${RM} -f $$newfile; \
${ECHO_MSG} "=> distinfo: unchanged."; \
@@ -87,16 +94,13 @@ distinfo:
${RM} -f ${DISTINFO_FILE}; \
${MV} -f $$newfile ${DISTINFO_FILE}; \
fi
- @rm ${_DISTINFO_INPUTFILE}
makesum:
-.for file in ${_CKSUMFILES}
- @${ECHO} ${file} >> ${_DISTINFO_INPUTFILE}
-.endfor
${RUN}set -e; \
newfile=${DISTINFO_FILE}.$$$$; \
- if ${_DISTINFO_CMD} ${_DISTINFO_ARGS_COMMON} \
- -I ${_DISTINFO_INPUTFILE} > $$newfile; \
+ if { ${_CKSUMFILES:@f@ ${ECHO} ${f};@} } \
+ | ${_DISTINFO_CMD} ${_DISTINFO_ARGS_COMMON} \
+ -I /dev/stdin > $$newfile; \
then \
${RM} -f $$newfile; \
${ECHO_MSG} "=> distinfo: distfiles part unchanged."; \
@@ -104,7 +108,6 @@ makesum:
${RM} -f ${DISTINFO_FILE}; \
${MV} -f $$newfile ${DISTINFO_FILE}; \
fi
- @rm ${_DISTINFO_INPUTFILE}
makepatchsum:
${RUN}set -e; \
Home |
Main Index |
Thread Index |
Old Index