[pkgsrc/trunk]: pkgsrc/pkgtools Add pkg_regress, a regression testing system ...

[gavan <gavan%pkgsrc.org@localhost>]

details:   https://anonhg.NetBSD.org/pkgsrc/rev/a9bb73e7141b
branches:  trunk
changeset: 480012:a9bb73e7141b
user:      gavan <gavan%pkgsrc.org@localhost>
date:      Sat Aug 28 00:26:57 2004 +0000

description:
Add pkg_regress, a regression testing system for the pkgsrc infrastructure.
Tests are found in pkgsrc/regress, tests are run by executing pkg_regress.

Based on an idea discussed at pkgsrcCon 2004. Thanks to dillo@ for review
and refinements, and to jlam@ for review.

diffstat:

 pkgtools/Makefile                         |    3 +-
 pkgtools/pkg_regress/DESCR                |    1 +
 pkgtools/pkg_regress/Makefile             |   33 ++++++
 pkgtools/pkg_regress/PLIST                |    2 +
 pkgtools/pkg_regress/files/pkg_regress.sh |  145 ++++++++++++++++++++++++++++++
 5 files changed, 183 insertions(+), 1 deletions(-)

diffs (214 lines):

diff -r a7dfcd883c51 -r a9bb73e7141b pkgtools/Makefile
--- a/pkgtools/Makefile Sat Aug 28 00:19:17 2004 +0000
+++ b/pkgtools/Makefile Sat Aug 28 00:26:57 2004 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.60 2004/08/27 01:51:00 schmonz Exp $
+# $NetBSD: Makefile,v 1.61 2004/08/28 00:26:57 gavan Exp $
 #
 
 COMMENT=       Tools for use in the packages collection
@@ -19,6 +19,7 @@
 SUBDIR+=       pkg_filecheck
 #SUBDIR+=      pkg_install     # Only install on demand of bsd.pkg.mk
 SUBDIR+=       pkg_install-info
+SUBDIR+=       pkg_regress
 SUBDIR+=       pkg_tarup
 SUBDIR+=       pkgclean
 SUBDIR+=       pkgconflict
diff -r a7dfcd883c51 -r a9bb73e7141b pkgtools/pkg_regress/DESCR
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/pkgtools/pkg_regress/DESCR        Sat Aug 28 00:26:57 2004 +0000
@@ -0,0 +1,1 @@
+pkg_regress runs the regression test suite for the pkgsrc infrastructure.
diff -r a7dfcd883c51 -r a9bb73e7141b pkgtools/pkg_regress/Makefile
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/pkgtools/pkg_regress/Makefile     Sat Aug 28 00:26:57 2004 +0000
@@ -0,0 +1,33 @@
+# $NetBSD: Makefile,v 1.1 2004/08/28 00:26:57 gavan Exp $
+
+DISTNAME=      pkg_regress-0.0
+CATEGORIES=    pkgtools
+MASTER_SITES=  # empty
+DISTFILES=     # empty
+
+MAINTAINER=    gavan%NetBSD.org@localhost
+HOMEPAGE=      ftp://ftp.NetBSD.org/pub/NetBSD/packages/pkgsrc/Packages.txt
+COMMENT=       Run pkgsrc infrastructure regression test suite
+
+PKG_INSTALLATION_TYPES=        overwrite pkgviews
+
+WRKSRC=                ${WRKDIR}
+EXTRACT_ONLY=  # empty
+USE_LANGUAGES= # empty
+NO_CHECKSUM=   YES
+
+INSTALLATION_DIRS=     sbin
+
+.include "../../mk/bsd.prefs.mk"
+
+do-build:
+       ${SED} -e 's#@MAKE@#${MAKE}#g' \
+         -e 's#@PKGSRCDIR@#${_PKGSRCDIR}#g' \
+         -e 's#@SH@#${SH}#g' \
+         -e 's#@EGREP@#${EGREP}#g' \
+         ${FILESDIR}/pkg_regress.sh >${WRKSRC}/pkg_regress.sh
+
+do-install:
+       ${INSTALL_SCRIPT} ${WRKSRC}/pkg_regress.sh ${PREFIX}/sbin/pkg_regress
+
+.include "../../mk/bsd.pkg.mk"
diff -r a7dfcd883c51 -r a9bb73e7141b pkgtools/pkg_regress/PLIST
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/pkgtools/pkg_regress/PLIST        Sat Aug 28 00:26:57 2004 +0000
@@ -0,0 +1,2 @@
+@comment $NetBSD: PLIST,v 1.1 2004/08/28 00:26:57 gavan Exp $
+sbin/pkg_regress
diff -r a7dfcd883c51 -r a9bb73e7141b pkgtools/pkg_regress/files/pkg_regress.sh
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/pkgtools/pkg_regress/files/pkg_regress.sh Sat Aug 28 00:26:57 2004 +0000
@@ -0,0 +1,145 @@
+#!@SH@ -e
+#
+# $NetBSD: pkg_regress.sh,v 1.1 2004/08/28 00:28:36 gavan Exp $
+#
+
+if [ -z "$PKGSRCDIR" ]
+then
+    PKGSRCDIR=@PKGSRCDIR@
+fi
+
+TEST_EGREP=@EGREP@
+TEST_MAKE=@MAKE@
+
+# hooks overridable by test spec file
+
+do_setup()
+{
+}
+
+do_cleanup()
+{
+}
+
+check_result()
+{
+}
+
+# result checking routines
+
+exit_status()
+{
+       # Test exit status
+       if [ "$1" -ne "${TEST_EXITSTATUS}" ]
+       then
+           TEST_RESULT=1
+       fi
+}
+
+output_require()
+{
+       # Test positive match against output
+       for PATTERN in "$@"
+       do
+           if ! ${TEST_EGREP} -q "${PATTERN}" <${TEST_OUTFILE} >/dev/null
+           then
+               TEST_RESULT=1
+           fi
+       done
+}
+
+output_prohibit()
+{
+       # Test negative match against output
+       for PATTERN in "$@"
+       do
+           if ${TEST_EGREP} -q "${PATTERN}" <${TEST_OUTFILE} >/dev/null
+           then
+               TEST_RESULT=1
+           fi
+       done
+}
+
+# runtest runs a test in a subshell, so that environment settings etc in
+# one test do not interfere with other tests.
+runtest() {
+    if (
+       cd $1
+       TEST_RESULT=0
+       TEST_EXITSTATUS=0
+       TEST_OUTFILE=`mktemp -t pkg_regress` || exit 1
+       . ./spec
+
+       do_setup
+
+       # Run the test. We use an if statement to ensure that the script
+       # isn't terminated if it is executed with sh -e.
+       if ${TEST_MAKE} ${MAKEARGS_TEST} >${TEST_OUTFILE} 2>&1
+       then
+           TEST_EXITSTATUS=$?
+       else
+           TEST_EXITSTATUS=$?
+       fi
+
+       check_result
+
+       # Perform cleanup
+
+       do_cleanup
+
+       if [ -n "${MAKEARGS_CLEAN}" ]
+       then
+           ${TEST_MAKE} ${MAKEARGS_CLEAN} >>${TEST_OUTFILE}
+       fi
+
+       if [ -n "${TEST_VERBOSE}" ]
+       then
+           cat ${TEST_OUTFILE}
+       fi
+
+       rm -f ${TEST_OUTFILE}
+       exit ${TEST_RESULT}
+    )
+    then
+       TEST_PASS=`expr ${TEST_PASS} + 1`
+    else
+       TEST_FAIL=`expr ${TEST_FAIL} + 1`
+       TEST_FAILURES="${TEST_FAILURES} $1"
+    fi
+}
+
+
+TEST_PASS=0
+TEST_FAIL=0
+TEST_FAILURES=
+
+cd $PKGSRCDIR/regress
+
+case $1 in
+    -v) TEST_VERBOSE=1
+       shift ;;
+esac
+
+if [ $# -ne 0 ]
+then
+    TEST_LIST="$@"
+else
+    TEST_LIST="*"
+fi
+
+for dir in ${TEST_LIST}
+do
+    if [ -f $dir/spec ]
+    then
+       runtest $dir
+    fi
+done
+
+if [ -n "${TEST_FAILURES}" ]
+then
+    echo "Tests failed: ${TEST_FAILURES}"
+    echo
+fi
+
+echo "Statistics:"
+echo "  $TEST_PASS passed, $TEST_FAIL failed"