Using pbulk with distcc

Hi,

I'd like to know if anybody has ever tried to use distcc within pbulk. I actually have a fully working distcc setup ( x86 helper cross compiling for a sparc host) working outside of pbulk but I can't make it work when using pbulk.

Up to now I created my sandbox using "mksandbox". I chrooted in the sandbox (/usr/sandbox/sandbox chroot), runned a "make install" for /usr/pkgsrc/devl/ccache, popt and distcc, I tested my setup inside my chroot and pkgsrc was using correctly distcc.

Then I modified pbulk.conf and the pbulk mk.conf to use distcc/ccache and avoid circular dependencies.

Here is my mk.conf

# Example /usr/pbulk/etc/mk.conf file produced by bootstrap-pkgsrc
# Thu Jan 18 04:26:10 GMT 2018
.ifdef BSD_PKG_MK # begin pkgsrc settings

ABI= 32
PKG_DBDIR= /usr/pbulk/pkgdb
LOCALBASE= /usr/pbulk
VARBASE= /usr/pbulk/var
PKG_TOOLS_BIN= /usr/pbulk/sbin
PKGINFODIR= info
PKGMANDIR= man
PKG_DEVELOPER= yes

.endif # end pkgsrc settings
PKGSRC_COMPILER=distcc ccache gcc
MAKE_JOBS=4
DISTCC_HOSTS=nbqc1-distcc1.neiluj.ca localhost
PKG_RCD_SCRIPTS=yes
ALLOW_VULNERABLE_PACKAGES=YES
PKG_DEFAULT_OPTIONS+=
KRB5_ACCEPTED=heimdal mit-krb5
USE_CWRAPPERS=yes
PKG_OPTIONS.irssi+= ssl perl inet6
DEPENDS_TARGET= bulk-install
BATCH= yes
BULK_PREREQ+= pkgtools/lintpkgsrc
BULK_PREREQ+= pkgtools/pkg_install
BULK_PREREQ+= devel/ccache
BULK_PREREQ+= devel/distcc
BULK_PREREQ+= devel/popt
# http://wiki.netbsd.org/tutorials/pkgsrc/cross_compile_distcc/
.for DISTCCDEPS in devel/ccache sysutils/checkperms pkgtools/digest devel/distcc devel/popt devel/libtool-base lang/f2c devel/gmake

. if "${PKGPATH}" == "${DISTCCDEPS}"
IGNORE_DISTCC= yes
IGNORE_CCACHE= yes
. endif

.endfor

And my pbulk.conf :

config_version=0.65
base_url=http://www.bla.org/reports/current/
reuse_scan_results=no
master_mode=no
pkg_rsync_args="-av --delete-excluded -e ssh"
pkg_rsync_target="pkgsrc@{My_SERVER_IP_HERE}:/srv/repo/netbsd/pkgsrc2017Q4/sparc"
report_rsync_args="-avz --delete-excluded -e ssh"
report_rsync_target="pkgsrc@{My_SERVER_IP_HERE}:/srv/repo/netbsd/pkgsrc2017Q4/sparc"
report_subject_prefix="pkgsrc"
report_recipients="juliensavard17%gmail.com@localhost"
bootstrapkit=/usr/pkgsrc/bootstrap/bootstrap.tar.gz
ignore_missing_dependencies=no
skip_age_check=no
report_graph_script_limit=512
unprivileged_user=pbulk
use_unprivileged_checksum=no
cross_compile=no
target_arch=sparc
target_destdir=/usr/src/destdir.${target_arch}
bulklog=/var/log/pbulk
packages=/usr/pbulk/packages
prefix=/usr/pkg
pkgsrc=/usr/pkgsrc
pkgdb=/var/db/pkg
varbase=/var
pkg_info=/usr/pbulk/sbin/pkg_info
pkg_add=/usr/pbulk/sbin/pkg_add
pkg_delete=/usr/pbulk/sbin/pkg_delete
bzip2=/usr/bin/bzip2
digest=/usr/pbulk/bin/digest
gzip="/usr/bin/gzip -nf -9"
make=${prefix}/bin/bmake
mail=/usr/bin/mail
neato=/usr/pbulk/bin/neato
rsync=/usr/pbulk/bin/rsync
sed=/usr/bin/sed
tar=/bin/tar
loc=${bulklog}/meta
pbuild=/usr/pbulk/bin/pbulk-build
presolve=/usr/pbulk/bin/pbulk-resolve
pscan=/usr/pbulk/bin/pbulk-scan
keep_wrkdir=no
keep_prefix=no
client_prepare=/usr/pbulk/libexec/pbulk/client-prepare
client_prepare_action=/usr/pbulk/libexec/pbulk/client-clean
client_prepare_chroot=:
pkg_up_to_date_script=/usr/pbulk/libexec/pbulk/pkg-up-to-date
pbuild_script=/usr/pbulk/libexec/pbulk/pkg-build
pbuild_start_script=/usr/pbulk/libexec/pbulk/build-client-start
pscan_start_script=/usr/pbulk/libexec/pbulk/scan-client-start
report_script=/usr/pbulk/libexec/pbulk/create-report
report_html_script=/usr/pbulk/libexec/pbulk/create-report-html
report_txt_script=/usr/pbulk/libexec/pbulk/create-report-txt
report_graph_script=/usr/pbulk/libexec/pbulk/create-broken-graph
packages_script=/usr/pbulk/libexec/pbulk/compute-packages
script_phase_pre_build=/usr/pbulk/libexec/pbulk/pre-build
script_phase_build=/usr/pbulk/libexec/pbulk/build
script_phase_report=/usr/pbulk/libexec/pbulk/report
script_phase_scan=/usr/pbulk/libexec/pbulk/scan
script_phase_upload=/usr/pbulk/libexec/pbulk/upload
sync_buildlog=:
sync_package=default_sync_package
pkg_sufx=.tgz
reuse_scan_results=yes
loc=${bulklog}/meta
make=${prefix}/bin/bmake
make=/usr/bin/make
limited_list=/usr/pbulk/etc/pbulk.list

Point is : It seems pbulk clear the /usr/pkg/bin directory at the beginning of the build ( or between each build). Clearing /usr/pkg/bin mean no distcc and no ccache binaries.

Compiling pkgsrc with an old sparc can be very very very long so i'm all ear if anybody has ever done something similar.