CVS commit: pkgsrc/mk

["Jonathan Perkin" <jperkin%netbsd.org@localhost>]

Module Name:    pkgsrc
Committed By:   jperkin
Date:           Fri Oct  9 16:00:16 UTC 2020

Modified Files:
        pkgsrc/mk/check: check-shlibs-macho.awk check-shlibs.mk
        pkgsrc/mk/pkgformat/pkg: metadata.mk
        pkgsrc/mk/platform: Darwin.mk

Log Message:
mk: Handle missing system libraries on Big Sur.

The new release of macOS removes system libraries from the file system, only
providing access to them via a linker cache and dlopen().  This obviously
breaks many assumptions about how libraries work on Unix systems, and so we
unfortunately need to cripple various checks when running on those systems.

Introduce DARWIN_NO_SYSTEM_LIBS which, when defined, will trigger alternate
behaviour in the infrastructure.  Currently this is in two places:

  * In CHECK_SHLIBS, skip any path beginning with /usr/lib.

  * In registered package metadata, any path beginning with /usr/lib is
    removed from REQUIRES.

The former fixes all package builds, while the second will be necessary for
package managers such as pkgin, as they will no longer be able to verify that
those files are available on the target system.

This is obviously a gross hack, and removes our ability to ensure that the
target system is suitable for the packages we are attempting to install, but
Apple have left us with no alternative, and users will unfortunately be left
to find out at runtime instead.

It's likely this will need to be extended to /System/Library paths too, but
this is required first to actually get packages building before we can start
running bulk builds.


To generate a diff of this commit:
cvs rdiff -u -r1.7 -r1.8 pkgsrc/mk/check/check-shlibs-macho.awk
cvs rdiff -u -r1.32 -r1.33 pkgsrc/mk/check/check-shlibs.mk
cvs rdiff -u -r1.29 -r1.30 pkgsrc/mk/pkgformat/pkg/metadata.mk
cvs rdiff -u -r1.98 -r1.99 pkgsrc/mk/platform/Darwin.mk

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: pkgsrc/mk/check/check-shlibs-macho.awk
diff -u pkgsrc/mk/check/check-shlibs-macho.awk:1.7 pkgsrc/mk/check/check-shlibs-macho.awk:1.8
--- pkgsrc/mk/check/check-shlibs-macho.awk:1.7  Thu Jun  7 07:00:10 2018
+++ pkgsrc/mk/check/check-shlibs-macho.awk      Fri Oct  9 16:00:16 2020
@@ -1,4 +1,4 @@
-# $NetBSD: check-shlibs-macho.awk,v 1.7 2018/06/07 07:00:10 dbj Exp $
+# $NetBSD: check-shlibs-macho.awk,v 1.8 2020/10/09 16:00:16 jperkin Exp $
 
 #
 # Read a list of potential Mach-O binaries from stdin.
@@ -73,6 +73,8 @@ function checkshlib(DSO,      needed, found) 
        while ((cmd | getline) > 0) {
                if ($0 !~ /^\t/)
                        continue
+               if (skip_system_libs && $0 ~ /^\t\/usr\/lib/)
+                       continue
                needed[$1] = ""
        }
        close(cmd)
@@ -119,6 +121,7 @@ BEGIN {
        wrkdir = ENVIRON["WRKDIR"]
        pkg_info_cmd = ENVIRON["PKG_INFO_CMD"]
        depends_file = ENVIRON["DEPENDS_FILE"]
+       skip_system_libs = ENVIRON["SKIP_SYSTEM_LIBS"]
 }
 
 { checkshlib($0); }

Index: pkgsrc/mk/check/check-shlibs.mk
diff -u pkgsrc/mk/check/check-shlibs.mk:1.32 pkgsrc/mk/check/check-shlibs.mk:1.33
--- pkgsrc/mk/check/check-shlibs.mk:1.32        Mon Mar 23 09:24:35 2020
+++ pkgsrc/mk/check/check-shlibs.mk     Fri Oct  9 16:00:16 2020
@@ -1,4 +1,4 @@
-# $NetBSD: check-shlibs.mk,v 1.32 2020/03/23 09:24:35 jperkin Exp $
+# $NetBSD: check-shlibs.mk,v 1.33 2020/10/09 16:00:16 jperkin Exp $
 #
 # This file verifies that all libraries used by the package can be found
 # at run-time.
@@ -64,6 +64,9 @@ CHECK_SHLIBS_NATIVE_ENV+=     PLATFORM_RPATH
 CHECK_SHLIBS_NATIVE_ENV+=      READELF=${TOOLS_PATH.readelf:Q}
 .  elif ${OBJECT_FMT} == "Mach-O"
 CHECK_SHLIBS_NATIVE=           ${PKGSRCDIR}/mk/check/check-shlibs-macho.awk
+.    if defined(DARWIN_NO_SYSTEM_LIBS)
+CHECK_SHLIBS_NATIVE_ENV+=      SKIP_SYSTEM_LIBS=1
+.    endif
 .  endif
 CHECK_SHLIBS_NATIVE_ENV+=      CROSS_DESTDIR=${_CROSS_DESTDIR:Q}
 CHECK_SHLIBS_NATIVE_ENV+=      PKG_INFO_CMD=${PKG_INFO:Q}

Index: pkgsrc/mk/pkgformat/pkg/metadata.mk
diff -u pkgsrc/mk/pkgformat/pkg/metadata.mk:1.29 pkgsrc/mk/pkgformat/pkg/metadata.mk:1.30
--- pkgsrc/mk/pkgformat/pkg/metadata.mk:1.29    Mon Jul  6 18:29:11 2020
+++ pkgsrc/mk/pkgformat/pkg/metadata.mk Fri Oct  9 16:00:16 2020
@@ -1,4 +1,4 @@
-# $NetBSD: metadata.mk,v 1.29 2020/07/06 18:29:11 maya Exp $
+# $NetBSD: metadata.mk,v 1.30 2020/10/09 16:00:16 jperkin Exp $
 
 ######################################################################
 ### The targets below are all PRIVATE.
@@ -24,6 +24,15 @@ _BUILD_DATE_cmd=     ${DATE} "+%Y-%m-%d %H:%
 _BUILD_HOST_cmd=       ${UNAME} -a
 _METADATA_TARGETS+=    ${_BUILD_INFO_FILE}
 
+#
+# Skip system libraries on Darwin releases where they do not exist.
+#
+.if defined(DARWIN_NO_SYSTEM_LIBS)
+DARWIN_REQUIRES_FILTER=        ${GREP} -v '\t/usr/lib'
+.else
+DARWIN_REQUIRES_FILTER=        ${CAT}
+.endif
+
 ${_BUILD_INFO_FILE}: ${_PLIST_NOKEYWORDS}
        ${RUN}${MKDIR} ${.TARGET:H}
        ${RUN}${RM} -f ${.TARGET}.tmp
@@ -85,7 +94,7 @@ ${_BUILD_INFO_FILE}: ${_PLIST_NOKEYWORDS
        Mach-O)                                                         \
                libs=`${AWK} '/\/lib.*\.dylib/ { print "${DESTDIR}${PREFIX}/" $$0 } END { exit 0 }' ${_PLIST_NOKEYWORDS}`; \
                if ${TEST} "$$bins" != "" -o "$$libs" != ""; then       \
-                       requires=`($$ldd $$bins $$libs 2>/dev/null || ${TRUE}) | ${AWK} '/compatibility version/ { print $$1 }' | ${SORT} -u`; \
+                       requires=`($$ldd $$bins $$libs 2>/dev/null || ${TRUE}) | ${DARWIN_REQUIRES_FILTER} | ${AWK} '/compatibility version/ { print $$1 }' | ${SORT} -u`; \
                fi;                                                     \
                ;;                                                      \
        PE)                                                             \

Index: pkgsrc/mk/platform/Darwin.mk
diff -u pkgsrc/mk/platform/Darwin.mk:1.98 pkgsrc/mk/platform/Darwin.mk:1.99
--- pkgsrc/mk/platform/Darwin.mk:1.98   Fri Aug 21 21:29:16 2020
+++ pkgsrc/mk/platform/Darwin.mk        Fri Oct  9 16:00:16 2020
@@ -1,4 +1,4 @@
-# $NetBSD: Darwin.mk,v 1.98 2020/08/21 21:29:16 sjmulder Exp $
+# $NetBSD: Darwin.mk,v 1.99 2020/10/09 16:00:16 jperkin Exp $
 #
 # Variable definitions for the Darwin operating system.
 
@@ -120,6 +120,16 @@ PKG_FAIL_REASON+=  "No suitable Xcode SDK
 .  endif
 .endif
 
+#
+# Newer macOS releases remove library files from the file system.  The only way
+# to test them is via dlopen(), which is obviously impractical for many things.
+#
+# This define turns off anything that can't reasonably supported this.
+#
+.if ${OS_VERSION:R} >= 20
+DARWIN_NO_SYSTEM_LIBS= # defined
+.endif
+
 .if ${OS_VERSION:R} >= 6
 _OPSYS_HAS_INET6=      yes     # IPv6 is standard
 .else