Re: gzimg hashes

To: Valery Ushakov <uwe%stderr.spb.ru@localhost>
Subject: Re: gzimg hashes
From: Brook Milligan <brook%biology.nmsu.edu@localhost>
Date: Sat, 18 Jul 2026 09:10:23 -0600
> On Jul 15, 2026, at 16:51, Valery Ushakov <uwe%stderr.spb.ru@localhost> wrote:
> 
> On Wed, Jul 15, 2026 at 11:54:46 -0600, Brook Milligan wrote:
> 
>> Yes, indeed a parallelism issue.  That is now fixed.
> 
> mkhashes still carries stale $NetBSD$ from its prototype
> 
> mkhashes serializes image creation, can't we exploit make parallelism
> istead of fighting it?
> 
> foo.img.gz.$(HASH) depends on foo.img.gz
> 
> $(HASH) depends on all *.img.gz.$(HASH) with a trivial cat $> > $@
> rule

Good point.  At first, I was trying to minimize the changes.  However, here is another solution along the lines you suggested.

The basic idea is to add a new target (snap_md_post_images) to the end of snap_md_post that takes care of the (up to) four steps: (i) create an uncompressed image, (ii) create compressed bootable images + corresponding hash fiies from it, (iii) compress and hash the original image, and (iv) collect all the hashes into a single file.  Each step can be done in parallel, something that itself is an improvement.  Also, all these steps are not necessary for every architecture, so some changes skip or merge these four into fewer.

Please look over the patch below and provide comments.

Thanks a lot.

Cheers,
Brook


Index: distrib/utils/embedded/mkimage
===================================================================
RCS file: /cvsroot/src/distrib/utils/embedded/mkimage,v
retrieving revision 1.97
diff -u -r1.97 mkimage
--- distrib/utils/embedded/mkimage	22 May 2026 14:44:16 -0000	1.97
+++ distrib/utils/embedded/mkimage	18 Jul 2026 14:39:36 -0000
@@ -425,8 +425,4 @@
	image="${image}.gz"
fi

-cd "${IMAGEDIR}"
-${CKSUM} -a MD5 "$(basename "${image}")" > MD5
-${CKSUM} -a SHA512 "$(basename "${image}")" > SHA512
-
echo ${bar} Image is ${image} ${bar}
Index: etc/Makefile
===================================================================
RCS file: /cvsroot/src/etc/Makefile,v
retrieving revision 1.476
diff -u -r1.476 Makefile
--- etc/Makefile	10 May 2026 14:19:10 -0000	1.476
+++ etc/Makefile	18 Jul 2026 14:39:36 -0000
@@ -48,6 +48,9 @@

GZIP_FLAGS= -9 ${GZIP_N_FLAG}

+HASHES+=	MD5
+HASHES+=	SHA512
+
# Flags for creating ISO CDROM image for mac68k and macppc that require
# tools mkhybrid(8) to create hybrid ISO9660/HFS images.
MKISOFS?=	${TOOL_MKHYBRID}
@@ -61,7 +64,7 @@
.if ${MKISOFS_FLAGS:N-v}
MKISOFS_FLAGS+=	-quiet
.endif
- 
+
# Reproducible build parameters
.if ${MKREPRO:Uno} == "no"
BUILDER!=	echo $${USER-root}@$$(hostname)
@@ -697,7 +700,7 @@
	done
.	for k in ${ALL_KERNELS}
install-debugkernels: ${DESTDIR}${DEBUGDIR}/netbsd-${k}.debug
-		
+
${DESTDIR}${DEBUGDIR}/netbsd-${k}.debug:
	${_MKTARGET_INSTALL}
	touch ${.TARGET}
Index: etc/etc.evbarm/Makefile.inc
===================================================================
RCS file: /cvsroot/src/etc/etc.evbarm/Makefile.inc,v
retrieving revision 1.138
diff -u -r1.138 Makefile.inc
--- etc/etc.evbarm/Makefile.inc	20 Feb 2026 07:20:28 -0000	1.138
+++ etc/etc.evbarm/Makefile.inc	18 Jul 2026 14:39:37 -0000
@@ -96,21 +96,27 @@
	MACHINE=${MACHINE} \
	MKDTB=${MKDTB} \
	MKPOSTFIX=${MKPOSTFIX} \
-	CKSUM=${CKSUM} \
-	IMAGEDIR=${IMAGE.dir} \
-	${HOST_SH} ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
+	${HOST_SH} ${MKIMAGE} -h ${.TARGET:R:S/smp_//} -D ${DESTDIR} \
	    -S ${NETBSDSRCDIR} -B ${IMAGEENDIAN} ${MKI_OPTS.${.TARGET}} \
-	    ${IMAGE.dir}/${.TARGET:S/smp_//}.img
-.for f in ${INSTALLBOOT_BOARDS}
-	@echo "===> Making bootable image ${IMAGE.dir:T}/${.TARGET:S/smp_//}-${f}.img.gz"
-	@cp ${IMAGE.dir}/${.TARGET:S/smp_//}.img ${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img
-	@${TOOL_INSTALLBOOT} -m ${MACHINE} -o board=${f} \
-			${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img \
-		&& ${TOOL_GZIP} -f ${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img \
-		|| rm ${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img
+	    ${IMAGE.dir}/${.TARGET:S/smp_//}
+
+__mkbootimage: .USE
+	@echo "===> Making bootable image ${IMAGE.dir:T}/${.TARGET:S/smp_//}"
+	cp ${IMAGE.dir}/${.TARGET:R:S/smp_//:C/-.*//}.img ${IMAGE.dir}/${.TARGET:R:S/smp_//}
+	${TOOL_INSTALLBOOT} -m ${MACHINE} -o board=${.TARGET:R:R:C/smp_[^-]*-//} \
+			${IMAGE.dir}/${.TARGET:R:S/smp_//} \
+	&& ${TOOL_GZIP} -f ${IMAGE.dir}/${.TARGET:R:S/smp_//} \
+	&& for hash in ${HASHES}; do \
+		(cd ${IMAGE.dir} && ${TOOL_CKSUM} -a $${hash} ${.TARGET:S/smp_//} > ${.TARGET:S/smp_//}.$${hash}) \
+	done \
+	|| rm ${IMAGE.dir}/${.TARGET:R:S/smp_//}*
+
+__mkcompressed: .USE
+	@echo "===> Compressing image ${IMAGE.dir}/${.TARGET:R:S/smp_//}"
+	${TOOL_GZIP} -f ${GZIP_N_FLAG} -9 ${IMAGE.dir}/${.TARGET:R:S/smp_//}
+.for hash in ${HASHES}
+	(cd ${IMAGE.dir} && ${TOOL_CKSUM} -a ${hash} ${.TARGET:S/smp_//} > ${.TARGET:S/smp_//}.${hash})
.endfor
-	@echo "===> Compressing image ${IMAGE.dir}/${.TARGET:S/smp_//}.img"
-	${TOOL_GZIP} -f ${GZIP_N_FLAG} -9 ${IMAGE.dir}/${.TARGET:S/smp_//}.img

ARCHES.earmv4=		armv4
ARCHES.earmv4eb=	armv4
@@ -149,34 +155,75 @@
.endif

.if !empty(MACHINE_ARCH:M*armv7*) && empty(ALL_KERNELS)
-smp_armv7: __mkimage
-MKI_OPTS.smp_armv7=	-K ${IMAGE.kern} -x
-SNAP_MD_POST_DEPS+=	smp_armv7
+smp_armv7.img: __mkimage
+smp_armv7.img.gz: __mkcompressed
+MKI_OPTS.smp_armv7.img=		-K ${IMAGE.kern} -x
+SNAP_MD_POST_DEPS_IMG+=		smp_armv7.img
+SNAP_MD_POST_DEPS_IMGZ+=	smp_armv7.img.gz
+.for board in ${INSTALLBOOT_BOARDS}
+smp_armv7-${board}.img.gz: __mkbootimage
+SNAP_MD_POST_DEPS_BOOTABLE+=	smp_armv7-${board}.img.gz
+.endfor
.endif

.if !empty(MACHINE_ARCH:Maarch64*) && empty(ALL_KERNELS)
-smp_arm64: __mkimage
-MKI_OPTS.smp_arm64=	-K ${IMAGE.kern} -x
-SNAP_MD_POST_DEPS+=	smp_arm64
-smp_arm64mbr: __mkimage
-MKI_OPTS.smp_arm64mbr=	-K ${IMAGE.kern} -x
-SNAP_MD_POST_DEPS+=	smp_arm64mbr
+smp_arm64.img: __mkimage
+smp_arm64.img.gz: __mkcompressed
+MKI_OPTS.smp_arm64.img=		-K ${IMAGE.kern} -x
+SNAP_MD_POST_DEPS_IMG+=		smp_arm64.img
+SNAP_MD_POST_DEPS_IMGZ+=	smp_arm64.img.gz
+.for board in ${INSTALLBOOT_BOARDS}
+smp_arm64-${board}.img.gz: __mkbootimage
+SNAP_MD_POST_DEPS_BOOTABLE+=	smp_arm64-${board}.img.gz
+.endfor
+smp_arm64mbr.img: __mkimage
+smp_arm64mbr.img.gz: __mkcompressed
+MKI_OPTS.smp_arm64mbr.img=	-K ${IMAGE.kern} -x
+SNAP_MD_POST_DEPS_IMG+=		smp_arm64mbr.img
+SNAP_MD_POST_DEPS_IMGZ+=	smp_arm64mbr.img.gz
+.for board in ${INSTALLBOOT_BOARDS}
+smp_arm64mbr-${board}.img.gz: __mkbootimage
+SNAP_MD_POST_DEPS_BOOTABLE+=	smp_arm64mbr-${board}.img.gz
+.endfor
.endif

.if !empty(KERNEL_SETS:MRPI)
-smp_rpi: __mkimage
+smp_rpi.img: __mkimage
+smp_rpi.img.gz: __mkcompressed
.if empty(ALL_KERNELS) || !empty(ALL_KERNELS:MRPI)
-MKI_OPTS.smp_rpi=	-K ${IMAGE.kern}/netbsd-RPI.img.gz -x
-SNAP_MD_POST_DEPS+=	smp_rpi
+MKI_OPTS.smp_rpi.img=		-K ${IMAGE.kern}/netbsd-RPI.img.gz -x
+SNAP_MD_POST_DEPS_IMG+=		smp_rpi.img
+SNAP_MD_POST_DEPS_IMGZ+=	smp_rpi.img.gz
+.for board in ${INSTALLBOOT_BOARDS}
+smp_rpi-${board}.img.gz: __mkbootimage
+SNAP_MD_POST_DEPS_BOOTABLE+=	smp_rpi-${board}.img.gz
+.endfor
.endif
-smp_rpi_inst: ${IMAGE.instk}/netbsd-RPI_INSTALL.img.gz __mkimage
+smp_rpi_inst.img: ${IMAGE.instk}/netbsd-RPI_INSTALL.img.gz
+smp_rpi_inst.img: __mkimage
+smp_rpi_inst.img.gz: __mkcompressed
.if empty(ALL_KERNELS) || !empty(ALL_KERNELS:MRPI_INSTALL)
-MKI_OPTS.smp_rpi_inst=	-b -K ${IMAGE.instk}/netbsd-RPI_INSTALL.img.gz -x
-SNAP_MD_POST_DEPS+=	smp_rpi_inst
+MKI_OPTS.smp_rpi_inst.img=	-b -K ${IMAGE.instk}/netbsd-RPI_INSTALL.img.gz -x
+SNAP_MD_POST_DEPS_IMG+=		smp_rpi_inst.img
+SNAP_MD_POST_DEPS_IMGZ+=	smp_rpi_inst.img.gz
+.for board in ${INSTALLBOOT_BOARDS}
+smp_rpi_inst-${board}.img.gz: __mkbootimage
+SNAP_MD_POST_DEPS_BOOTABLE+=	smp_rpi_inst-${board}.img.gz
+.endfor
.endif
.endif

-snap_md_post: check_DESTDIR check_RELEASEDIR snap_post .WAIT ${SNAP_MD_POST_DEPS}
+smp_hashes:
+	@echo "===> Making hash files: ${HASHES}"
+.for hash in ${HASHES}
+	cat ${IMAGE.dir}/*.${hash} > ${IMAGE.dir}/${hash} \
+	&& rm -f ${IMAGE.dir}/*.${hash}
+.endfor
+SNAP_MD_POST_DEPS_HASHES+=	smp_hashes
+
+snap_md_post_images: ${SNAP_MD_POST_DEPS_IMG} .WAIT ${SNAP_MD_POST_DEPS_BOOTABLE} .WAIT ${SNAP_MD_POST_DEPS_IMGZ} .WAIT ${SNAP_MD_POST_DEPS_HASHES}
+
+snap_md_post: check_DESTDIR check_RELEASEDIR snap_post .WAIT ${SNAP_MD_POST_DEPS} snap_md_post_images

.if !empty(MACHINE_ARCH:Maarch64*)
KERNEL_SUFFIXES=	img
Index: etc/etc.evbmips/Makefile.inc
===================================================================
RCS file: /cvsroot/src/etc/etc.evbmips/Makefile.inc,v
retrieving revision 1.32
diff -u -r1.32 Makefile.inc
--- etc/etc.evbmips/Makefile.inc	22 Mar 2025 09:25:14 -0000	1.32
+++ etc/etc.evbmips/Makefile.inc	18 Jul 2026 14:39:37 -0000
@@ -70,11 +70,13 @@
	MACHINE=${MACHINE} \
	MKDTB=${MKDTB} \
	MKPOSTFIX=${MKPOSTFIX} \
-	CKSUM=${CKSUM} \
-	IMAGEDIR=${IMAGE.dir} \
	${HOST_SH} ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
            -S ${NETBSDSRCDIR} -B ${IMAGEENDIAN} ${MKI_OPTS.${.TARGET}} \
            ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
+	@echo "===> Making hash files: ${HASHES}"
+	for hash in ${HASHES}; do \
+		(cd ${IMAGE.dir} && ${TOOL_CKSUM} -a $${hash} ${.TARGET:S/smp_//}.img.gz > $${hash}); \
+	done

.if !empty(MACHINE_ARCH:Mmips*64eb)
smp_octeon: __mkimage
Index: etc/etc.evbppc/Makefile.inc
===================================================================
RCS file: /cvsroot/src/etc/etc.evbppc/Makefile.inc,v
retrieving revision 1.22
diff -u -r1.22 Makefile.inc
--- etc/etc.evbppc/Makefile.inc	10 Jan 2026 06:56:58 -0000	1.22
+++ etc/etc.evbppc/Makefile.inc	18 Jul 2026 14:39:37 -0000
@@ -58,11 +58,13 @@
	MACHINE=${MACHINE} \
	MKDTB=${MKDTB} \
	MKPOSTFIX=${MKPOSTFIX} \
-	CKSUM=${CKSUM} \
-	IMAGEDIR=${IMAGE.dir} \
	${HOST_SH} -x ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
            -S ${NETBSDSRCDIR} -B ${IMAGEENDIAN} ${MKI_OPTS.${.TARGET}} \
            ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
+	@echo "===> Making hash files: ${HASHES}"
+	for hash in ${HASHES}; do \
+		(cd ${IMAGE.dir} && ${TOOL_CKSUM} -a $${hash} ${.TARGET:S/smp_//}.img.gz > $${hash}); \
+	done

smp_nintendo: __mkimage
	cd ${KERNSRCDIR}/arch/evbppc/stand/wii && ${MAKE} release
Index: etc/etc.riscv/Makefile.inc
===================================================================
RCS file: /cvsroot/src/etc/etc.riscv/Makefile.inc,v
retrieving revision 1.7
diff -u -r1.7 Makefile.inc
--- etc/etc.riscv/Makefile.inc	22 Mar 2025 09:25:14 -0000	1.7
+++ etc/etc.riscv/Makefile.inc	18 Jul 2026 14:39:37 -0000
@@ -15,8 +15,6 @@
IMAGE.dir=      ${IMAGE.rel}/binary/gzimg
IMAGE.kern=     ${IMAGE.rel}/binary/kernel

-#	TOOL_MKUBOOTIMAGE=${TOOL_MKUBOOTIMAGE} \
-
__mkimage: .USE
	TOOL_MAKE=${MAKE} \
	TOOL_MAKEFS=${TOOL_MAKEFS} \
@@ -34,11 +32,12 @@
	MACHINE=${MACHINE} \
	MKDTB=${MKDTB} \
	MKPOSTFIX=${MKPOSTFIX} \
-	CKSUM=${CKSUM} \
-	IMAGEDIR=${IMAGE.dir} \
	${HOST_SH} ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
            -S ${NETBSDSRCDIR} ${MKI_OPTS.${.TARGET}} \
            ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
+	for hash in ${HASHES}; do \
+		(cd ${IMAGE.dir} && ${TOOL_CKSUM} -a $${hash} ${.TARGET:S/smp_//}.img.gz > ${.TARGET:S/smp_//}.img.gz.$${hash}) \
+	done


.if ${MACHINE_ARCH} == "riscv64"
@@ -51,18 +50,29 @@
.endif

smp_riscv64: __mkimage
-MKI_OPTS.smp_riscv64=	-K ${IMAGE.kern}
-SNAP_MD_POST_DEPS+=	smp_riscv64
+MKI_OPTS.smp_riscv64=		-K ${IMAGE.kern}
+SNAP_MD_POST_DEPS_IMGZ+=	smp_riscv64
.endif


.if ${MACHINE_ARCH} == "riscv32"
smp_riscv32: __mkimage
-MKI_OPTS.smp_riscv32=	-K ${IMAGE.kern}
-SNAP_MD_POST_DEPS+=	smp_riscv32
+MKI_OPTS.smp_riscv32=		-K ${IMAGE.kern}
+SNAP_MD_POST_DEPS_IMGZ+=	smp_riscv32
.endif

-snap_md_post: check_DESTDIR check_RELEASEDIR snap_post .WAIT ${SNAP_MD_POST_DEPS}
+
+smp_hashes:
+	@echo "===> Making hash files: ${HASHES}"
+.for hash in ${HASHES}
+	cat ${IMAGE.dir}/*.${hash} > ${IMAGE.dir}/${hash} \
+	&& rm -f ${IMAGE.dir}/*.${hash}
+.endfor
+SNAP_MD_POST_DEPS_HASHES+=	smp_hashes
+
+snap_md_post_images: ${SNAP_MD_POST_DEPS_IMGZ} .WAIT ${SNAP_MD_POST_DEPS_HASHES}
+
+snap_md_post: check_DESTDIR check_RELEASEDIR snap_post .WAIT ${SNAP_MD_POST_DEPS} snap_md_post_images

INSTALLATION_DIRS+=	binary/gzimg		\
			installation/instkernel
References:
- Re: gzimg hashes
  - From: David H. Gutteridge
- Re: gzimg hashes
  - From: Brook Milligan
- Re: gzimg hashes
  - From: Brook Milligan
- Re: gzimg hashes
  - From: David H. Gutteridge
- Re: gzimg hashes
  - From: Brook Milligan
- Re: gzimg hashes
  - From: Valery Ushakov
Prev by Date: daily CVS update output
Previous by Thread: Re: gzimg hashes
Next by Thread: ahcisata0: reset failed
Indexes:
Home | Main Index | Thread Index | Old Index