CVS commit: pkgsrc/regress/infra-unittests

["Roland Illig" <rillig%netbsd.org@localhost>]

Module Name:    pkgsrc
Committed By:   rillig
Date:           Fri Mar 20 08:56:42 UTC 2020

Modified Files:
        pkgsrc/regress/infra-unittests: mocked-include.sh subst.sh test.subr
            tools-bison.sh

Log Message:
regress/infra-unittests: restructure test infrastructure

Before, the first assertion failure quit immediately. This prevented
getting a complete picture of the situation that failed. Now the
assertions continue the test and fail at the very end.


To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.2 pkgsrc/regress/infra-unittests/mocked-include.sh \
    pkgsrc/regress/infra-unittests/tools-bison.sh
cvs rdiff -u -r1.3 -r1.4 pkgsrc/regress/infra-unittests/subst.sh
cvs rdiff -u -r1.2 -r1.3 pkgsrc/regress/infra-unittests/test.subr

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: pkgsrc/regress/infra-unittests/mocked-include.sh
diff -u pkgsrc/regress/infra-unittests/mocked-include.sh:1.1 pkgsrc/regress/infra-unittests/mocked-include.sh:1.2
--- pkgsrc/regress/infra-unittests/mocked-include.sh:1.1        Thu Mar 21 21:45:30 2019
+++ pkgsrc/regress/infra-unittests/mocked-include.sh    Fri Mar 20 08:56:42 2020
@@ -6,15 +6,19 @@ set -eu
 
 . "./test.subr"
 
-create_file "including.mk" <<EOF
-.include "mk/bsd.prefs.mk"
-EOF
+if test_case_begin "mock"; then
+
+       create_file_lines "including.mk" \
+               '.include "mk/bsd.prefs.mk"'
 
-create_pkgsrc_file "mk/bsd.prefs.mk" <<EOF
+       create_pkgsrc_file "mk/bsd.prefs.mk" <<EOF
 all:
        @echo 'the mocked definition wins'
 EOF
 
-out=$(test_file "including.mk")
+       out=$(test_file "including.mk")
+
+       assert_that "$out" --equals "the mocked definition wins"
 
-assert_that "$out" --equals "the mocked definition wins"
+       test_case_end
+fi
Index: pkgsrc/regress/infra-unittests/tools-bison.sh
diff -u pkgsrc/regress/infra-unittests/tools-bison.sh:1.1 pkgsrc/regress/infra-unittests/tools-bison.sh:1.2
--- pkgsrc/regress/infra-unittests/tools-bison.sh:1.1   Thu Mar 21 21:45:30 2019
+++ pkgsrc/regress/infra-unittests/tools-bison.sh       Fri Mar 20 08:56:42 2020
@@ -15,11 +15,13 @@ pkg_admin=$(mock_cmd mock-pkg_admin \
        --when-args "pmatch bison>=2.0 bison-1.5" --then-exit 1
 )
 
-# A package may add more than one entry to the BISON_REQD list. The
-# platform-provided bison may only be used if all of the BISON_REQD
-# entries are below the platform-provided version.
-#
-create_file "multiple-reqd-entries.mk" <<EOF
+if test_case_begin "multiple BISON_REQD entries"; then
+
+       # A package may add more than one entry to the BISON_REQD list. The
+       # platform-provided bison may only be used if all of the BISON_REQD
+       # entries are below the platform-provided version.
+
+       create_file "multiple-reqd-entries.mk" <<EOF
 BISON_REQD=            1.0 2.0
 USE_TOOLS=             bison
 TOOLS_PLATFORM.bison=  $tmpdir/mock-bison
@@ -33,14 +35,21 @@ all:
 .include "mk/tools/bison.mk"
 EOF
 
-out=$(test_file "multiple-reqd-entries.mk")
+       out=$(test_file "multiple-reqd-entries.mk")
+
+       assert_that "$out" --equals "yes"
+
+       test_case_end
+fi
 
-assert_that "$out" --equals "yes"
 
-# Both required versions are lower than the version of the mocked bison,
-# which is 1.5. Therefore the platform-provided bison can be used.
-#
-create_file "multiple-reqd-entries.mk" <<EOF
+if test_case_begin "multiple BISON_REQD entries, both lower"; then
+
+       # Both required versions are lower than the version of the mocked
+       # bison, which is 1.5. Therefore the platform-provided bison can
+       # be used.
+
+       create_file "multiple-reqd-entries.mk" <<EOF
 BISON_REQD=            1.0 1.1
 USE_TOOLS=             bison
 TOOLS_PLATFORM.bison=  $tmpdir/mock-bison
@@ -54,6 +63,9 @@ all:
 .include "mk/tools/bison.mk"
 EOF
 
-out=$(test_file "multiple-reqd-entries.mk")
+       out=$(test_file "multiple-reqd-entries.mk")
+
+       assert_that "$out" --equals "no"
 
-assert_that "$out" --equals "no"
+       test_case_end
+fi

Index: pkgsrc/regress/infra-unittests/subst.sh
diff -u pkgsrc/regress/infra-unittests/subst.sh:1.3 pkgsrc/regress/infra-unittests/subst.sh:1.4
--- pkgsrc/regress/infra-unittests/subst.sh:1.3 Fri Mar 20 06:17:48 2020
+++ pkgsrc/regress/infra-unittests/subst.sh     Fri Mar 20 08:56:42 2020
@@ -6,9 +6,7 @@ set -eu
 
 . "./test.subr"
 
-testcase() {
-       test_name="$1"
-
+test_case_set_up() {
        rm -rf "$tmpdir"/.subst_*_done "$tmpdir"/.subst-empty
        rm -rf "$tmpdir"/*
        ls -A "$tmpdir"
@@ -44,7 +42,7 @@ EOF
 }
 
 
-if testcase "single file"; then
+if test_case_begin "single file"; then
 
        # A single file is patched successfully.
 
@@ -67,10 +65,12 @@ EOF
 
        assert_that "output" --file-contains-exactly "=> Substituting \"class\" in subst-single.txt"
        assert_that "subst-single.txt" --file-contains-exactly "after"
+
+       test_case_end
 fi
 
 
-if testcase "several individual files"; then
+if test_case_begin "several individual files"; then
 
        # Several individual files are patched successfully.
 
@@ -96,10 +96,12 @@ EOF
        assert_that "first" --file-contains-exactly "the first example"
        assert_that "second" --file-contains-exactly "the second example"
        assert_that "third" --file-contains-exactly "the third example"
+
+       test_case_end
 fi
 
 
-if testcase "several files by pattern"; then
+if test_case_begin "several files by pattern"; then
 
        # Several files are patched successfully.
        # The filenames are given by a pattern.
@@ -126,10 +128,12 @@ EOF
        assert_that "pattern-first" --file-contains-exactly "the first example"
        assert_that "pattern-second" --file-contains-exactly "the second example"
        assert_that "pattern-third" --file-contains-exactly "the third example"
+
+       test_case_end
 fi
 
 
-if testcase "pattern with 1 noop"; then
+if test_case_begin "pattern with 1 noop"; then
 
        # Several files are given via a pattern.
        # Most of the files are patched, but one stays the same.
@@ -163,10 +167,12 @@ EOF
        assert_that "pattern-first" --file-contains-exactly "the first example"
        assert_that "pattern-second" --file-contains-exactly "the second is already an example"
        assert_that "pattern-third" --file-contains-exactly "the third example"
+
+       test_case_end
 fi
 
 
-if testcase "single file noop, noop_ok=yes"; then
+if test_case_begin "single file noop, noop_ok=yes"; then
 
        create_file "testcase.mk" <<EOF
 SUBST_CLASSES+=                class
@@ -191,10 +197,12 @@ EOF
        assert_that "actual-output" --file-equals "expected-output"
        assert_that "single" --file-contains-exactly "already an example"
        assert_that "$exitcode" --equals "0"
+
+       test_case_end
 fi
 
 
-if testcase "single file noop, noop_ok=no"; then
+if test_case_begin "single file noop, noop_ok=no"; then
 
        create_file "testcase.mk" <<EOF
 SUBST_CLASSES+=                class
@@ -224,10 +232,12 @@ EOF
        assert_that "actual-output" --file-equals "expected-output"
        assert_that "single" --file-contains-exactly "already an example"
        assert_that "$exitcode" --equals "1"
+
+       test_case_end
 fi
 
 
-if testcase "single file nonexistent"; then
+if test_case_begin "single file nonexistent"; then
 
        create_file "testcase.mk" <<EOF
 SUBST_CLASSES+=                class
@@ -254,10 +264,12 @@ EOF
                "$make: stopped in $PWD"
        assert_that "actual-output" --file-equals "expected-output"
        assert_that "$exitcode" --equals "1"
+
+       test_case_end
 fi
 
 
-if testcase "single file nonexistent ok"; then
+if test_case_begin "single file nonexistent ok"; then
 
        create_file "testcase.mk" <<EOF
 SUBST_CLASSES+=                class
@@ -279,10 +291,12 @@ EOF
                'warning: [subst.mk:class] Ignoring non-existent file "./nonexistent".'
        assert_that "actual-output" --file-equals "expected-output"
        assert_that "$exitcode" --equals "0"
+
+       test_case_end
 fi
 
 
-if testcase "several patterns, 1 nonexistent"; then
+if test_case_begin "several patterns, 1 nonexistent"; then
 
        create_file "testcase.mk" <<EOF
 SUBST_CLASSES+=                class
@@ -306,10 +320,12 @@ EOF
        assert_that "actual-output" --file-equals "expected-output"
        assert_that "exists" --file-contains-exactly "this example exists"
        assert_that "$exitcode" --equals "0"
+
+       test_case_end
 fi
 
 
-if testcase "multiple missing files, all are reported at once"; then
+if test_case_begin "multiple missing files, all are reported at once"; then
 
        create_file "testcase.mk" <<EOF
 SUBST_CLASSES+=                class
@@ -330,10 +346,12 @@ EOF
                'warning: [subst.mk:class] Ignoring non-existent file "./exist".'
        assert_that "actual-output" --file-equals "expected-output"
        assert_that "$exitcode" --equals "0"
+
+       test_case_end
 fi
 
 
-if testcase "multiple no-op files, all are reported at once"; then
+if test_case_begin "multiple no-op files, all are reported at once"; then
 
        create_file "testcase.mk" <<EOF
 SUBST_CLASSES+=                class
@@ -357,6 +375,8 @@ EOF
                'info: [subst.mk:class] Nothing changed in ./third.'
        assert_that "actual-output" --file-equals "expected-output"
        assert_that "$exitcode" --equals "0"
+
+       test_case_end
 fi
 
 # TODO: Add test that ensures SUBST_FILES is evaluated as late as possible.

Index: pkgsrc/regress/infra-unittests/test.subr
diff -u pkgsrc/regress/infra-unittests/test.subr:1.2 pkgsrc/regress/infra-unittests/test.subr:1.3
--- pkgsrc/regress/infra-unittests/test.subr:1.2        Thu Mar 19 16:58:35 2020
+++ pkgsrc/regress/infra-unittests/test.subr    Fri Mar 20 08:56:42 2020
@@ -1,5 +1,5 @@
 #! /bin/sh
-# $NetBSD: test.subr,v 1.2 2020/03/19 16:58:35 rillig Exp $
+# $NetBSD: test.subr,v 1.3 2020/03/20 08:56:42 rillig Exp $
 set -eu
 
 # This file defines utilities for testing Makefile fragments in a mocked
@@ -54,9 +54,9 @@ set -eu
 
 : "${cleanup:=yes}"
 : "${make:=bmake}"
+: "${if_verbose=:}"
 tmpdir="${TMP:-/tmp}/infra-unittests-$$"
 mocked_pkgsrcdir="$tmpdir/pkgsrc"
-test_name="unknown test"
 rm -rf "$tmpdir"
 mkdir -p "$mocked_pkgsrcdir"
 
@@ -70,7 +70,29 @@ done
        exit 1
 }
 
-maybe_cleanup() {
+test_case_name="unknown test"
+
+test_case_begin() {
+       test_case_name="$1"
+
+       test_case_set_up
+}
+
+# Can be redefined by actual tests.
+test_case_set_up() {
+}
+
+# Can be redefined by actual tests.
+test_case_tear_down() {
+}
+
+test_case_end() {
+       test_case_tear_down
+
+       test_case_name="unknown test"
+}
+
+test_subr_cleanup() {
        exit_status=$?
        if [ $exit_status -ne 0 ]; then
                printf 'info: the test files are in %s\n' "$tmpdir" 1>&2
@@ -78,8 +100,15 @@ maybe_cleanup() {
        fi
 
        [ "$cleanup" = "yes" ] && rm -rf "$tmpdir"
+
+       $if_verbose printf '%s%d assertions succeeded, %d assertions failed\n' \
+               "$assert_fail_sep" "$assert_succeeded" "$assert_failed"
+
+       if [ "$assert_failed" != 0 ]; then
+               exit 1
+       fi
 }
-trap "maybe_cleanup" EXIT
+trap "test_subr_cleanup" EXIT
 
 mock_cmd() {
        cmdname="$1"
@@ -144,27 +173,61 @@ EOF
        "$make" -f "$tmpdir/test.subr.main.mk" "$@"
 }
 
+assert_succeeded=0
+assert_failed=0
+assert_fail_sep=''
+
+assert_succeed() {
+       assert_succeeded=`expr "$assert_succeeded" + 1`
+}
+
+assert_fail() {
+       printf '%s' "$assert_fail_sep" 1>&2
+       assert_fail_sep='
+'
+
+       printf 'assertion failed in "%s": ' "$test_case_name" 1>&2
+
+       printf "$@" 1>&2
+       assert_failed=`expr "$assert_failed" + 1`
+}
+
 assert_that() {
        case "$2" in
        (--equals)
-               [ "x$1" = "x$3" ] && return 0
-               printf 'assertion failed in "%s":\nexpected: <%s>\nbut was:  <%s>\n' "$test_name" "$3" "$1" 1>&2
-               exit 1
+               if [ "x$1" = "x$3" ]; then
+                       assert_succeed
+                       return 0
+               fi
+               assert_fail '\n  expected: <%s>\n  but was:  <%s>\n' "$3" "$1"
                ;;
 
        (--file-contains-exactly)
                printf '%s\n' "$3" > "$tmpdir/expected"
-               diff -u "$tmpdir/expected" "$tmpdir/$1" > /dev/null && return 0
-               printf 'assertion failed in "%s": file %s has unexpected content:\n' "$test_name" "$1" 1>&2
-               diff -u "$tmpdir/expected" "$tmpdir/$1" 1>&2
-               exit 1
+               if diff -u "$tmpdir/expected" "$tmpdir/$1" > /dev/null; then
+                       assert_succeed
+                       return 0
+               fi
+               assert_fail 'file "%s" has unexpected content:\n' "$1"
+               diff -u "$tmpdir/expected" "$tmpdir/$1" 1>&2 || true
                ;;
 
        (--file-equals)
-               diff -u "$tmpdir/$3" "$tmpdir/$1" > /dev/null && return 0
-               printf 'assertion failed in "%s": files %s and %s differ:\n' "$test_name" "$1" "$3" 1>&2
-               diff -u "$tmpdir/$3" "$tmpdir/$1" 1>&2
-               exit 1
+               if diff -u "$tmpdir/$3" "$tmpdir/$1" > /dev/null; then
+                       assert_succeed
+                       return 0
+               fi
+               assert_fail 'files "%s" and "%s" differ:\n' "$1" "$3"
+               diff -u "$tmpdir/$3" "$tmpdir/$1" 1>&2 || true
+               ;;
+
+       (--file-is-empty)
+               if diff -u "/dev/null" "$tmpdir/$1" > /dev/null; then
+                       assert_succeed
+                       return 0
+               fi
+               assert_fail 'file "%s" is not empty:\n' "$1"
+               diff -u "/dev/null" "$tmpdir/$1" 1>&2 || true
                ;;
 
        (*)