tech-pkg archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

PKGSRC_USE_MKTOOLS support added



Hey all,

As proposed a few weeks ago, I've now added the support for C-based tools to speed up various parts of the mk infrastructure. You can opt in to the support (it currently defaults to off) with:

  PKGSRC_USE_MKTOOLS=yes

in your mk.conf.

For packages with large numbers of dependencies, you should see anywhere up to around 10x speed-up for the 'wrapper' phase, and perhaps more importantly a huge reduction in system time usage.

I'd love to get feedback on whether this works for you on exotic platforms. I'm using it without issues on macOS and illumos, and have had positive reports from NetBSD users. I'll also be turning it on for all my daily bulk builds. Hopefully we can switch the default to yes in the future.

Cheers,

--
Jonathan Perkin  -  Joyent, Inc.  -  www.joyent.com
--- Begin Message ---
Module Name:	pkgsrc
Committed By:	jperkin
Date:		Tue Jun  7 10:04:25 UTC 2022

Modified Files:
	pkgsrc/mk: bsd.pkg.use.mk bsd.prefs.mk
	pkgsrc/mk/buildlink3: bsd.buildlink3.mk
	pkgsrc/mk/defaults: mk.conf

Log Message:
mk: Add PKGSRC_USE_MKTOOLS support.

When enabled, pull in the pkgtools/mktools package to use C-based tools for
certain parts of the pkgsrc mk infrastructure.  Default is off for now while
any portability issues are shaken out, but it has been tested successfully on
at least illumos, macOS, and NetBSD.

The first tool is mk-buildlink-symlinks which creates the buildlink3 symlinks
as part of the "wrapper" phase.  This significantly improves performance,
especially with packages that have a lot of buildlink3 dependencies.  For
example, the time for "bmake wrapper" in x11/kde-workspace4 goes from:

  real     3:20.696394375
  user       50.553556463
  sys      2:23.883823896

to:

  real       19.873100598
  user        8.141441461
  sys        11.740602820

It's expected that other tools will be added over time to speed up other parts
of the infrastructure.


To generate a diff of this commit:
cvs rdiff -u -r1.71 -r1.72 pkgsrc/mk/bsd.pkg.use.mk
cvs rdiff -u -r1.421 -r1.422 pkgsrc/mk/bsd.prefs.mk
cvs rdiff -u -r1.247 -r1.248 pkgsrc/mk/buildlink3/bsd.buildlink3.mk
cvs rdiff -u -r1.333 -r1.334 pkgsrc/mk/defaults/mk.conf

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: pkgsrc/mk/bsd.pkg.use.mk
diff -u pkgsrc/mk/bsd.pkg.use.mk:1.71 pkgsrc/mk/bsd.pkg.use.mk:1.72
--- pkgsrc/mk/bsd.pkg.use.mk:1.71	Mon May  9 15:35:31 2022
+++ pkgsrc/mk/bsd.pkg.use.mk	Tue Jun  7 10:04:25 2022
@@ -1,4 +1,4 @@
-#	$NetBSD: bsd.pkg.use.mk,v 1.71 2022/05/09 15:35:31 nia Exp $
+#	$NetBSD: bsd.pkg.use.mk,v 1.72 2022/06/07 10:04:25 jperkin Exp $
 #
 # Turn USE_* macros into proper depedency logic.  Included near the top of
 # bsd.pkg.mk, after bsd.prefs.mk.
@@ -109,3 +109,10 @@ TOOL_DEPENDS+=		libtool-base>=${_OPSYS_L
 CONFIGURE_ENV+=		LIBTOOL="${LIBTOOL} ${LIBTOOL_FLAGS}"
 MAKE_ENV+=		LIBTOOL="${LIBTOOL} ${LIBTOOL_FLAGS}"
 .endif
+
+# PKGSRC_USE_MKTOOLS.  Cyclic dependencies prevent us from using mktools
+# when building cwrappers, so use the shell tools in that instance.
+.if ${_PKGSRC_USE_MKTOOLS} == "yes" && empty(PKGPATH:Mpkgtools/cwrappers)
+TOOL_DEPENDS+=		mktools-[0-9]*:../../pkgtools/mktools
+PKG_MKSYMLINKS?=	${LOCALBASE}/libexec/mktools/mk-buildlink-symlinks
+.endif

Index: pkgsrc/mk/bsd.prefs.mk
diff -u pkgsrc/mk/bsd.prefs.mk:1.421 pkgsrc/mk/bsd.prefs.mk:1.422
--- pkgsrc/mk/bsd.prefs.mk:1.421	Sun May  1 08:03:40 2022
+++ pkgsrc/mk/bsd.prefs.mk	Tue Jun  7 10:04:25 2022
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.prefs.mk,v 1.421 2022/05/01 08:03:40 nia Exp $
+# $NetBSD: bsd.prefs.mk,v 1.422 2022/06/07 10:04:25 jperkin Exp $
 #
 # This file includes the mk.conf file, which contains the user settings.
 #
@@ -874,6 +874,14 @@ _USE_CWRAPPERS=		yes
 _USE_CWRAPPERS=		no
 .endif
 
+# Use C-based tools to speed up pkgsrc infrastructure tasks.
+.if empty(PKGPATH:Mpkgtools/mktools) && \
+    ${PKGSRC_USE_MKTOOLS:tl} == "yes"
+_PKGSRC_USE_MKTOOLS=	yes
+.else
+_PKGSRC_USE_MKTOOLS=	no
+.endif
+
 # Wrapper framework definitions
 .include "wrapper/wrapper-defs.mk"
 

Index: pkgsrc/mk/buildlink3/bsd.buildlink3.mk
diff -u pkgsrc/mk/buildlink3/bsd.buildlink3.mk:1.247 pkgsrc/mk/buildlink3/bsd.buildlink3.mk:1.248
--- pkgsrc/mk/buildlink3/bsd.buildlink3.mk:1.247	Tue Jun  8 10:10:02 2021
+++ pkgsrc/mk/buildlink3/bsd.buildlink3.mk	Tue Jun  7 10:04:25 2022
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.buildlink3.mk,v 1.247 2021/06/08 10:10:02 bouyer Exp $
+# $NetBSD: bsd.buildlink3.mk,v 1.248 2022/06/07 10:04:25 jperkin Exp $
 #
 # Copyright (c) 2004 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -660,7 +660,7 @@ _BLNK_FILES_CMD.${_pkg_}+=	)
 _BLNK_FILES_CMD.${_pkg_}+=	| ${SORT} -u
 
 ${_BLNK_COOKIE.${_pkg_}}:
-	${RUN}					\
+	${RUN}								\
 	case "${BUILDLINK_PREFIX.${_pkg_}}" in				\
 	*not_found)							\
 		${ERROR_MSG} "${BUILDLINK_API_DEPENDS.${_pkg_}} is not installed; can't buildlink files."; \
@@ -671,6 +671,47 @@ ${_BLNK_COOKIE.${_pkg_}}:
 		${ERROR_MSG} "[bsd.buildlink3.mk] X11BASE is not set correctly."; \
 		exit 1;							\
 	}
+.if ${_PKGSRC_USE_MKTOOLS} == "yes"
+	${RUN}								\
+	case "${BUILDLINK_PREFIX.${_pkg_}}" in				\
+	${LOCALBASE})   buildlink_dir="${BUILDLINK_DIR}" ;;		\
+	${X11BASE})     buildlink_dir="${BUILDLINK_X11_DIR}" ;;		\
+	*)              buildlink_dir="${BUILDLINK_DIR}" ;;		\
+	esac;								\
+	[ -z "${BUILDLINK_PREFIX.${_pkg_}:Q}" ] ||			\
+	cd ${BUILDLINK_PREFIX.${_pkg_}} &&				\
+	${_BLNK_FILES_CMD.${_pkg_}} |					\
+	while read file; do						\
+		src="${_CROSS_DESTDIR}${BUILDLINK_PREFIX.${_pkg_}}/$$file"; \
+		[ -f "$$src" ] || continue;				\
+		dest="$$buildlink_dir/$$file";				\
+		if [ -n "${BUILDLINK_FNAME_TRANSFORM.${_pkg_}:Q}" ]; then \
+			dest=`${ECHO} $$dest | ${SED} ${BUILDLINK_FNAME_TRANSFORM.${_pkg_}}`; \
+		fi;							\
+		case "$$src" in						\
+		*.la)							\
+			dir="$${dest%/*}";				\
+			if [ ! -d "$$dir" ]; then			\
+				${MKDIR} "$$dir";			\
+			fi;						\
+			${_BLNK_LT_ARCHIVE_FILTER.${_pkg_}}		\
+				"$$src" > "$$dest";			\
+			;;						\
+		*)							\
+			${ECHO} "$$dest -> $$src" >>${.TARGET};		\
+			;;						\
+		esac;							\
+	done;								\
+	if [ -f ${.TARGET} ]; then					\
+		${PKG_MKSYMLINKS} < ${.TARGET} || {			\
+			${ECHO} "Removing ${.TARGET}";			\
+			${RM} -f ${.TARGET};				\
+			exit 1;						\
+		}							\
+	else								\
+		${TOUCH} ${TOUCH_FLAGS} ${.TARGET};			\
+	fi
+.else
 	${RUN}								\
 	case "${BUILDLINK_PREFIX.${_pkg_}}" in				\
 	${LOCALBASE})   buildlink_dir="${BUILDLINK_DIR}" ;;		\
@@ -681,8 +722,8 @@ ${_BLNK_COOKIE.${_pkg_}}:
 	cd ${BUILDLINK_PREFIX.${_pkg_}} &&				\
 	${_BLNK_FILES_CMD.${_pkg_}} |					\
 	while read file; do						\
-		src="${_CROSS_DESTDIR}${BUILDLINK_PREFIX.${_pkg_}}/$$file";		\
-		if [ ! -f "$$src" ]; then					\
+		src="${_CROSS_DESTDIR}${BUILDLINK_PREFIX.${_pkg_}}/$$file"; \
+		if [ ! -f "$$src" ]; then				\
 			msg="$$src: not found";				\
 		else							\
 			if [ -z "${BUILDLINK_FNAME_TRANSFORM.${_pkg_}:Q}" ]; then \
@@ -713,6 +754,7 @@ ${_BLNK_COOKIE.${_pkg_}}:
 		fi;							\
 		${ECHO} "$$msg" >> ${.TARGET};				\
 	done
+.endif
 
 # _BLNK_LT_ARCHIVE_FILTER.${_pkg_} is a command-line filter used in
 # the previous target for transforming libtool archives (*.la) to

Index: pkgsrc/mk/defaults/mk.conf
diff -u pkgsrc/mk/defaults/mk.conf:1.333 pkgsrc/mk/defaults/mk.conf:1.334
--- pkgsrc/mk/defaults/mk.conf:1.333	Tue Jun  7 09:35:15 2022
+++ pkgsrc/mk/defaults/mk.conf	Tue Jun  7 10:04:25 2022
@@ -1,4 +1,4 @@
-# $NetBSD: mk.conf,v 1.333 2022/06/07 09:35:15 wiz Exp $
+# $NetBSD: mk.conf,v 1.334 2022/06/07 10:04:25 jperkin Exp $
 #
 
 # This file provides default values for variables that may be overridden
@@ -22,6 +22,15 @@ USE_CWRAPPERS?=	auto
 # Possible: yes, no, auto
 # Default: auto (automatic detection, see above)
 
+PKGSRC_USE_MKTOOLS?=	no
+# Use C-based tools for various parts of the infrastructure instead of the
+# slower shell-based constructs.  Significantly improves performance, but
+# not enabled by default until portability across all platforms has been
+# confirmed.
+#
+# Possible: yes, no
+# Default: no
+
 USE_PKG_ADMIN_DIGEST?=	no
 # Record file digests instead of extracing the NetBSD RCS ID from files.
 #


--- End Message ---


Home | Main Index | Thread Index | Old Index