Subject: syspkg patches for review
To: None <tech-install@netbsd.org>
From: David Young <dyoung@pobox.com>
List: tech-install
Date: 06/15/2003 01:55:47
--CE+1k2dSO48ffgeK
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
The regpkg script runs glacially slow on my K6-500. I sped things up by
* changing regpkg to seperate packing lists into files and directories
using xargs/find, and
* squeezing many times more work out of each make(1) invocation.
Also, I have fixed listpkgs, makeplist, and makeflist so that there are
no longer inconsistencies in the packing lists and the lists of package
names which they produce.
Will anyone object if I apply these patches, which speed things up a lot,
and which also produce more consistent lists?
Dave
--
David Young OJC Technologies
dyoung@ojctech.com Urbana, IL * (217) 278-3933
--CE+1k2dSO48ffgeK
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="d.regpkg"
Index: regpkg
===================================================================
RCS file: /cvsroot/src/distrib/sets/regpkg,v
retrieving revision 1.2
diff -u -r1.2 regpkg
--- regpkg 2003/06/13 02:32:26 1.2
+++ regpkg 2003/06/15 06:38:17
@@ -117,12 +117,18 @@
esac
# create the build information
-echo "OPSYS=`uname -s`" > ${SYSPKGDIR}/+BUILD_INFO
-echo "OS_VERSION=`uname -r`" >> ${SYSPKGDIR}/+BUILD_INFO
-echo "OBJECT_FMT=`printf '.include <bsd.own.mk>\nall:\n\t@echo ${OBJECT_FMT}' | make -f- all`" >> ${SYSPKGDIR}/+BUILD_INFO
-echo "MACHINE_ARCH=`printf '.include <bsd.own.mk>\nall:\n\t@echo ${MACHINE_ARCH}' | make -f- all`" >> ${SYSPKGDIR}/+BUILD_INFO
-echo "MACHINE_GNU_ARCH=`printf '.include <bsd.own.mk>\nall:\n\t@echo ${MACHINE_GNU_ARCH}' | make -f- all`" >> ${SYSPKGDIR}/+BUILD_INFO
-echo "_PKGTOOLS_VER=`pkg_create -V`" >> ${SYSPKGDIR}/+BUILD_INFO
+{
+echo "OPSYS=`uname -s`"
+echo "OS_VERSION=`uname -r`"
+make -f- all <<EOF
+.include <bsd.own.mk>
+all:
+ @echo OBJECT_FMT=${OBJECT_FMT}
+ @echo MACHINE_ARCH=${MACHINE_ARCH}
+ @echo MACHINE_GNU_ARCH=${MACHINE_GNU_ARCH}
+EOF
+echo "_PKGTOOLS_VER=`pkg_create -V`"
+} > ${SYSPKGDIR}/+BUILD_INFO
# test for attributes
args=""
@@ -143,3 +149,4 @@
-s /dev/null -S /dev/null -O $pkg-$t \
> ${SYSPKGDIR}/+CONTENTS
+rm $PLIST
--CE+1k2dSO48ffgeK
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="d.makeplist"
Index: makeplist
===================================================================
RCS file: /cvsroot/src/distrib/sets/makeplist,v
retrieving revision 1.7
diff -u -r1.7 makeplist
--- makeplist 2003/06/12 19:55:18 1.7
+++ makeplist 2003/06/15 06:37:18
@@ -5,9 +5,26 @@
#
# set defaults
-MAKE="${MAKE:-make} -j 1"
-machine=${MACHINE:-`printf 'xxx:\n\techo ${MACHINE}' | ${MAKE} -s -f-`}
-arch=${MACHINE_ARCH:-`printf 'xxx:\n\techo ${MACHINE_ARCH}' | ${MAKE} -s -f-`}
+for x in $(
+${MAKE:-make} -f- all <<EOF
+.include <bsd.own.mk>
+all:
+ @echo machine=\${MACHINE}
+ @echo machine_arch=\${MACHINE_ARCH}
+ @echo machine_cpu=\${MACHINE_CPU}
+ @echo object_fmt=\${OBJECT_FMT}
+ @echo toolchain_missing=\${TOOLCHAIN_MISSING}
+.if defined(USE_XF86_4) && (\${USE_XF86_4} != no)
+ @echo x11_version=4
+.else
+ @echo x11_version=3
+.endif
+
+EOF
+); do
+ eval $x
+done
+
setd=`dirname $0`
prefix=/
@@ -15,7 +32,7 @@
exec 1>&2
echo "Usage: $0 [-a arch] [-m machine] [-s setsdir] [-p prefix] setname pkgname"
-echo " -a arch set arch (e.g, m68k, mips, powerpc) [$arch]"
+echo " -a arch set arch (e.g, m68k, mips, powerpc) [$machine_arch]"
echo " -m machine set machine (e.g, amiga, i386, macppc) [$machine]"
echo " -s setsdir directory to find sets [$setd]"
echo " -p prefix prefix for created plist [$prefix]"
@@ -28,7 +45,7 @@
while : ; do
case $1 in
-a*)
- arch=$2; shift
+ machine_arch=$2; shift
;;
-m*)
machine=$2; shift
@@ -62,48 +79,130 @@
exit 1
fi
-# Convert mipse[lb] to mips after processing command line arguments.
-arch=`echo $arch | sed s,^mipse.,mips, | sed s,^sh3e.,sh3,`
+# Determine lib type.
+if [ "$object_fmt" = "ELF" ]; then
+ shlib=elf
+else
+ shlib=aout
+fi
+stlib=$shlib
-# Compute toolchain used on target cpu.
-case "$machine" in
-ns32k) shlib=aout ;;
-*) case "$arch" in
- sh3) shlib="" ;;
- *) shlib=elf ;;
- esac
-esac
+# Turn off shlibs for some ports.
+if [ "$machine_cpu" = "sh3" -o "$machine_arch" = "m68000" ]; then
+ shlib=no
+fi
+lkm=yes
+# Turn off LKMs for some ports.
+if [ "$machine" = "evbppc" ]; then
+ lkm=no
+fi
filename=/tmp/makeplist.$$
ffilename=/tmp/makeplist.files.$$
dfilename=/tmp/makeplist.dirs.$$
+
+(
+ cat $setd/lists/$setname/mi
+ if [ "$machine" != "$machine_arch" ]; then
+ # Prefer an ad.${machine_arch} over an ad.${machine_cpu},
+ # since the arch-specific one will be more specific than
+ # the cpu-specific one.
+ if [ -f $setd/lists/$setname/ad.${machine_arch} ]; then
+ cat $setd/lists/$setname/ad.${machine_arch}
+ elif [ -f $setd/lists/$setname/ad.${machine_cpu} ]; then
+ cat $setd/lists/$setname/ad.${machine_cpu}
+ fi
+ if [ "$shlib" != "no" -a \
+ -f $setd/lists/$setname/ad.${machine_cpu}.shl ]; then
+ cat $setd/lists/$setname/ad.${machine_cpu}.shl
+ fi
+ fi
+ if [ -f $setd/lists/$setname/md.${machine}.${machine_arch} ]; then
+ cat $setd/lists/$setname/md.${machine}.${machine_arch}
+ elif [ -f $setd/lists/$setname/md.${machine} ]; then
+ cat $setd/lists/$setname/md.${machine}
+ fi
+ if [ -f $setd/lists/$setname/stl.mi ]; then
+ cat $setd/lists/$setname/stl.mi
+ fi
+ if [ -f $setd/lists/$setname/stl.${stlib} ]; then
+ cat $setd/lists/$setname/stl.${stlib}
+ fi
+ if [ "$shlib" != "no" ]; then
+ if [ -f $setd/lists/$setname/shl.mi ]; then
+ cat $setd/lists/$setname/shl.mi
+ fi
+ if [ -f $setd/lists/$setname/shl.${shlib} ]; then
+ cat $setd/lists/$setname/shl.${shlib}
+ fi
+ fi
+ if [ "$lkm" != "no" ]; then
+ if [ -f $setd/lists/$setname/lkm.mi ]; then
+ cat $setd/lists/$setname/lkm.mi
+ fi
+ fi
+ if [ "$lintlibs" != no ]; then
+ if [ -f $setd/lists/$setname/lint.mi ]; then
+ cat $setd/lists/$setname/lint.mi
+ fi
+ fi
+ if [ "$toolchain_missing" != "yes" ]; then
+ if [ -f $setd/lists/$setname/tc.mi ]; then
+ cat $setd/lists/$setname/tc.mi
+ fi
+ if [ "$shlib" != "no" ]; then
+ if [ -f $setd/lists/$setname/tc.shl ]; then
+ cat $setd/lists/$setname/tc.shl
+ fi
+ fi
+ fi
+
+ if [ -f $setd/lists/$setname/rescue.mi ]; then
+ cat $setd/lists/$setname/rescue.mi
+ fi
+ if [ -f $setd/lists/$setname/rescue.${machine} ]; then
+ cat $setd/lists/$setname/rescue.${machine}
+ fi
+ if [ "$machine" != "$machine_arch" ]; then
+ # Prefer a rescue.ad.${machine_arch} over a
+ # rescue.ad.${machine_cpu}, since the arch-
+ # specific one will be more specific than the
+ # cpu-specific one.
+ if [ -f $setd/lists/$setname/rescue.ad.${machine_arch} ]; then
+ cat $setd/lists/$setname/rescue.ad.${machine_arch}
+ elif [ -f $setd/lists/$setname/rescue.ad.${machine_cpu} ]; then
+ cat $setd/lists/$setname/rescue.ad.${machine_cpu}
+ fi
+ if [ "$shlib" != "no" -a -f \
+ $setd/lists/$setname/rescue.ad.${machine_cpu}.shl ]; then
+ cat $setd/lists/$setname/rescue.ad.${machine_cpu}.shl
+ fi
+ fi
+) | env PLISTPKG=$pkgname awk -- '/^#/ { next } $2 == ENVIRON["PLISTPKG"] {sub("^\./", "", $1); print $1}' | sort -u > $filename
+
+SELECTDIRS="-maxdepth 0 -type d"
+SELECTNONDIRS="-maxdepth 0 ! -type d"
+
+cd $prefix
+#
+# match the directories
+#
+xargs echo $SELECTDIRS < $filename | \
+while read ignore ignore ignore ignore args; do
+ [ -z "$args" ] && break
+ find $args $SELECTDIRS
+done | awk '{ print "@dirrm " $1; }' > $dfilename
+
+#
+# match the non-directories
+#
+xargs echo $SELECTNONDIRS < $filename | \
+while read ignore ignore ignore ignore ignore args; do
+ [ -z "$args" ] && break
+ find $args $SELECTNONDIRS
+done > $ffilename
-echo "@cwd $prefix" > $filename
-(
- cat $setd/lists/$setname/mi
-# where does cpu get set? XXX - agc
- if [ "$machine" != "$cpu" -a -f $setd/lists/$setname/ad.${arch} ]; then
- cat $setd/lists/$setname/ad.${arch}
- fi
- if [ -f $setd/lists/$setname/md.${machine} ]; then
- cat $setd/lists/$setname/md.${machine}
- fi
- if [ "$shlib" != "" ]; then
- if [ -f $setd/lists/$setname/shl.mi ]; then
- cat $setd/lists/$setname/shl.mi
- fi
- if [ -f $setd/lists/$setname/shl.${shlib} ]; then
- cat $setd/lists/$setname/shl.${shlib}
- fi
- fi
-)| env PLISTPKG=$pkgname awk -- '/^#/ { next } $2 == ENVIRON["PLISTPKG"] {sub("^\./", "", $1); print $1}' | sort -u >> $filename
-
-env FFILENAME=$ffilename DFILENAME=$dfilename awk '
-/^@cwd/ { prefix = $2; next }
-{
- s = sprintf("if [ -d %s%s ]; then echo @dirrm %s >> %s; else echo %s >> %s; fi", prefix, $0, $0, ENVIRON["DFILENAME"], $0, ENVIRON["FFILENAME"]);
- system(s);
-}' $filename
+cd -
echo "@cwd $prefix"
if [ -s $ffilename ]; then
--CE+1k2dSO48ffgeK
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="d.listpkgs"
Index: listpkgs
===================================================================
RCS file: /cvsroot/src/distrib/sets/listpkgs,v
retrieving revision 1.4
diff -u -r1.4 listpkgs
--- listpkgs 2002/07/23 09:02:45 1.4
+++ listpkgs 2003/06/15 06:37:52
@@ -68,19 +68,81 @@
fi
(
+ # TBD factor w/ makeflist
cat $setd/lists/$setname/mi
- if [ "$machine" != "$cpu" -a -f $setd/lists/$setname/ad.${arch} ]; then
- cat $setd/lists/$setname/ad.${arch}
+ if [ "$machine" != "$machine_arch" ]; then
+ # Prefer an ad.${machine_arch} over an ad.${machine_cpu},
+ # since the arch-specific one will be more specific than
+ # the cpu-specific one.
+ if [ -f $setd/lists/$setname/ad.${machine_arch} ]; then
+ cat $setd/lists/$setname/ad.${machine_arch}
+ elif [ -f $setd/lists/$setname/ad.${machine_cpu} ]; then
+ cat $setd/lists/$setname/ad.${machine_cpu}
+ fi
+ if [ "$shlib" != "no" -a \
+ -f $setd/lists/$setname/ad.${machine_cpu}.shl ]; then
+ cat $setd/lists/$setname/ad.${machine_cpu}.shl
+ fi
fi
- if [ -f $setd/lists/$setname/md.${machine} ]; then
+ if [ -f $setd/lists/$setname/md.${machine}.${machine_arch} ]; then
+ cat $setd/lists/$setname/md.${machine}.${machine_arch}
+ elif [ -f $setd/lists/$setname/md.${machine} ]; then
cat $setd/lists/$setname/md.${machine}
+ fi
+ if [ -f $setd/lists/$setname/stl.mi ]; then
+ cat $setd/lists/$setname/stl.mi
+ fi
+ if [ -f $setd/lists/$setname/stl.${stlib} ]; then
+ cat $setd/lists/$setname/stl.${stlib}
fi
- if [ "$shlib" != "" ]; then
+ if [ "$shlib" != "no" ]; then
if [ -f $setd/lists/$setname/shl.mi ]; then
cat $setd/lists/$setname/shl.mi
fi
if [ -f $setd/lists/$setname/shl.${shlib} ]; then
cat $setd/lists/$setname/shl.${shlib}
+ fi
+ fi
+ if [ "$lkm" != "no" ]; then
+ if [ -f $setd/lists/$setname/lkm.mi ]; then
+ cat $setd/lists/$setname/lkm.mi
+ fi
+ fi
+ if [ "$lintlibs" != no ]; then
+ if [ -f $setd/lists/$setname/lint.mi ]; then
+ cat $setd/lists/$setname/lint.mi
+ fi
+ fi
+ if [ "$toolchain_missing" != "yes" ]; then
+ if [ -f $setd/lists/$setname/tc.mi ]; then
+ cat $setd/lists/$setname/tc.mi
+ fi
+ if [ "$shlib" != "no" ]; then
+ if [ -f $setd/lists/$setname/tc.shl ]; then
+ cat $setd/lists/$setname/tc.shl
+ fi
+ fi
+ fi
+
+ if [ -f $setd/lists/$setname/rescue.mi ]; then
+ cat $setd/lists/$setname/rescue.mi
+ fi
+ if [ -f $setd/lists/$setname/rescue.${machine} ]; then
+ cat $setd/lists/$setname/rescue.${machine}
+ fi
+ if [ "$machine" != "$machine_arch" ]; then
+ # Prefer a rescue.ad.${machine_arch} over a
+ # rescue.ad.${machine_cpu}, since the arch-
+ # specific one will be more specific than the
+ # cpu-specific one.
+ if [ -f $setd/lists/$setname/rescue.ad.${machine_arch} ]; then
+ cat $setd/lists/$setname/rescue.ad.${machine_arch}
+ elif [ -f $setd/lists/$setname/rescue.ad.${machine_cpu} ]; then
+ cat $setd/lists/$setname/rescue.ad.${machine_cpu}
+ fi
+ if [ "$shlib" != "no" -a -f \
+ $setd/lists/$setname/rescue.ad.${machine_cpu}.shl ]; then
+ cat $setd/lists/$setname/rescue.ad.${machine_cpu}.shl
fi
fi
)| awk -- '/^[^#]/ {print $2}' | sort -u
--CE+1k2dSO48ffgeK--