pkg/55938: lang/python39 build with dtrace issue

[aza.sea.agenda%gmail.com@localhost]

>Number:         55938
>Category:       pkg
>Synopsis:       lang/python39 build with dtrace issue
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    pkg-manager
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Wed Jan 20 10:40:00 +0000 2021
>Originator:     Aleksey Arens
>Release:        pkgsrc-2020Q4
>Organization:
>Environment:
NetBSD HOSTNAME_REDACTED 9.1_STABLE NetBSD 9.1_STABLE (GENERIC) #1: Sun Jan  3 20:58:50 UTC 2021  root@HOSTNAME_REDACTED:/usr/src/sys/arch/amd64/compile/obj/GENERIC amd64
>Description:
A lang/python39 build would fail with the following message if dtrace option is enabled in PKG_OPTIONS.  This problem affects also python38, python37, and python36.  All fail similarly on this system.

---8<---

--- python ---
--- Programs/_testembed ---
--- python ---
gcc -pthread -L. -L/usr/lib -Wl,-R/usr/lib -L/usr/pkg/lib -Wl,-R/usr/pkg/lib -L/usr/lib -Wl,-R/usr/lib -L/usr/pkg/lib -Wl,-R/usr/pkg/lib   -Wl,--export-dynamic -o python Programs/python.o -L. -lpython3.9 -lcrypt -lintl -lpthread  -lutil -lm   -lm
--- Programs/_testembed ---
gcc -pthread -L. -L/usr/lib -Wl,-R/usr/lib -L/usr/pkg/lib -Wl,-R/usr/pkg/lib -L/usr/lib -Wl,-R/usr/lib -L/usr/pkg/lib -Wl,-R/usr/pkg/lib   -Wl,--export-dynamic -o Programs/_testembed Programs/_testembed.o -L. -lpython3.9 -lcrypt -lintl -lpthread  -lutil -lm   -lm
--- python ---
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___import__find__load__start'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___gc__start'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___audit'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___gc__start'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___audit'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___import__find__load__start'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___function__return'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___gc__done'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___function__entry'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___import__find__load__done'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___line'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___function__entry'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___line'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___function__return'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___gc__done'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___import__find__load__done'
*** [python] Error code 1

make: stopped in /usr/obj/pkgsrc/lang/python39/work/Python-3.9.1
--- Programs/_testembed ---
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___import__find__load__start'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___gc__start'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___audit'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___gc__start'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___audit'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___import__find__load__start'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___function__return'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___gc__done'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___function__entry'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___import__find__load__done'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___line'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___function__entry'
ld: ./libpython3.9.so: undefined reference to `__dtrace_python___line'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___function__return'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___gc__done'
ld: ./libpython3.9.so: undefined reference to `__dtraceenabled_python___import__find__load__done'
*** [Programs/_testembed] Error code 1

make: stopped in /usr/obj/pkgsrc/lang/python39/work/Python-3.9.1
2 errors

make: stopped in /usr/obj/pkgsrc/lang/python39/work/Python-3.9.1
*** Error code 2

Stop.
make[1]: stopped in /usr/pkgsrc/lang/python39
*** Error code 1

Stop.
make: stopped in /usr/pkgsrc/lang/python39

>How-To-Repeat:
cd /usr/pkgsrc/lang/python39 && make PKG_OPTIONS.python39=dtrace

>Fix:
It was observed that the issue occurs, because Python/pydtrace.o file is not built.  The generation of this object file is in turn governed by the DTRACE_OBJS variable in Makefile.  The following code in the configure script is responsible for setting this variable, which is then propagated into the Makefile verbatim.  Evidently, a failure to execute dtrace compile on the build system would result in empty DTRACE_OBJS, and the aforementioned build failure.

---8<---

    # On OS X, DTrace providers do not need to be explicitly compiled and
    # linked into the binary. Correspondingly, dtrace(1) is missing the ELF
    # generation flag '-G'. We check for presence of this flag, rather than
    # hardcoding support by OS, in the interest of robustness.
    { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether DTrace probes require linking" >&5
$as_echo_n "checking whether DTrace probes require linking... " >&6; }
if ${ac_cv_dtrace_link+:} false; then :
  $as_echo_n "(cached) " >&6
else
              ac_cv_dtrace_link=no
            echo 'BEGIN{}' > conftest.d
            "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \
                ac_cv_dtrace_link=yes

fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_dtrace_link" >&5
$as_echo "$ac_cv_dtrace_link" >&6; }
    if test "$ac_cv_dtrace_link" = "yes"; then
        DTRACE_OBJS="Python/pydtrace.o"
    fi
fi

---8<---

The fix that I propose involves massaging the dtrace to become operational on the build host system.  This involves two things: passing an argument "-64" or "-32" to the dtrace binary, and adjusting permissions on /dev/ksyms and /dev/kmem.  The former is necessary, because dtrace fails as follows without these options on this system:

---8<---

$ echo "BEGIN{}" > conftest.d
$ dtrace -G -s conftest.d -o conftest.o
ld: i386 architecture of input file `conftest.o.eB1Uv2' is incompatible with i386:x86-64 output
dtrace: failed to link script conftest.d: failed to link conftest.o: ld exited with status 1

---8<---

The latter is necessary, because sometimes dtrace fails like that if permissions on /dev/ksyms and /dev/kmem are not set for the build user (also, cf. discussion at https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=193109):

---8<---

dtrace -64 -G -s conftest.d -o conftest.o
dtrace: failed to compile script conftest.d: "/usr/lib/dtrace/psinfo.d", line 46: syntax error near "u_int"

---8<---

Further research suggests that the failure occurs on NetBSD 9.X with GCC.  It did not occur on SmartOS or NetBSD-current with llvm.

The patch below fixes the issue by supplying dtrace with an appropriate flag.  The author thinks that the longer-term solution would need to involve fixing dtrace itself, which evidently has wrong default architecture preset on this combination of OS and machine architecture.

ACKNOWLEDGEMENTS.  Thanks goes to jperkin on #pkgsrc for suggesting the LP64PLATFORMS bsd make code.

---8<---

diff --git a/lang/python39/options.mk b/lang/python39/options.mk
index 5a947278cc9..76a07770fd5 100644
--- a/lang/python39/options.mk
+++ b/lang/python39/options.mk
@@ -9,9 +9,21 @@ PKG_SUGGESTED_OPTIONS=	x11
 
 PLIST_VARS+=		dtrace
 
+.if !empty(LP64PLATFORMS:@.PLAT.@${MACHINE_PLATFORM:M${.PLAT.}}@) || \
+(defined(ABI) && ${ABI} == "64")
+IS_64BIT_PLATFORM?=	yes
+.else
+IS_64BIT_PLATFORM?=	no
+.endif
+
 .if !empty(PKG_OPTIONS:Mdtrace)
 CONFIGURE_ARGS+=	--with-dtrace
 PLIST.dtrace=		yes
+.if !empty(IS_64BIT_PLATFORM:M[yY][eE][sS]) && ${OPSYS} == "NetBSD"
+CONFIGURE_ENV+=		DFLAGS=-64
+.else
+CONFIGURE_ENV+=		DFLAGS=-32
+.endif
 .endif
 
 .if !empty(PKG_OPTIONS:Mx11)