[src/trunk]: src PR/49085: Jarmo Jaakkola: fix several parsing errors

[christos <christos%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/25cf428e5a4a
branches:  trunk
changeset: 331760:25cf428e5a4a
user:      christos <christos%NetBSD.org@localhost>
date:      Sat Aug 23 14:50:24 2014 +0000

description:
PR/49085: Jarmo Jaakkola: fix several parsing errors

Don't exit from var.c:Var_Parse() before possible modifiers are handled
on D and F modified versions of local variables.  Properly expand $(?D)
and $(?F) too.

Make line continuations in rule's commands POSIX compliant.

Fix the syntax error caused by lib(member) as the last target before
a dependency operator.

Document the line continuation change in the manual page.  Also talk
more about the POSIX style local variables and their modifiers.

Add tests covering the fixed problems into d_posix.mk.  The test is
a known failure at the moment because of PR 49086 and PR 49092.

[XXX: unconverted tests]

diffstat:

 tests/usr.bin/make/d_posix.mk  |  142 +++++++++++++++++++++++++++++++++++++
 tests/usr.bin/make/d_posix.out |   91 +++++++++++++++++++++++
 tests/usr.bin/make/t_make.sh   |   28 +++++-
 usr.bin/make/make.1            |  157 ++++++++++++++++++++++++++++++++--------
 usr.bin/make/parse.c           |   62 ++++++++++------
 usr.bin/make/var.c             |   66 +++++++++--------
 6 files changed, 454 insertions(+), 92 deletions(-)

diffs (truncated from 792 to 300 lines):

diff -r 7c12af8ea25b -r 25cf428e5a4a tests/usr.bin/make/d_posix.mk
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/usr.bin/make/d_posix.mk     Sat Aug 23 14:50:24 2014 +0000
@@ -0,0 +1,142 @@
+# $NetBSD: d_posix.mk,v 1.3 2014/08/23 14:50:24 christos Exp $
+
+# Keep the default suffixes from interfering, just in case.
+.SUFFIXES:
+
+all:   line-continuations suffix-substitution localvars
+
+#
+# Line continuations
+#
+
+# Escaped newlines and leading whitespace from the next line are replaced
+# with single space, except in commands, where the escape and the newline
+# are retained, but a single leading tab (if any) from the next line is
+# removed. (PR 49085)
+# Expect:
+# $(VAR) = "foo  bar baz"
+# a
+# b
+# c
+VAR = foo\
+\
+         bar\
+ baz
+
+line-continuations:
+       @echo '$$(VAR) = "$(VAR)"'
+       @echo 'aXbXc' | sed -e 's/X/\
+       /g'
+
+
+#
+# Suffix substitution
+#
+
+# The only variable modifier accepted by POSIX.
+# $(VAR:s1=s2): replace s1, if found, with s2 at end of each word in
+# $(VAR).  s1 and s2 may contain macro expansions.
+# Expect: foo baR baz, bar baz, foo bar baz, fooadd baradd bazadd
+suffix-substitution:
+       @echo '$(VAR:r=R), $(VAR:foo=), $(VAR:not_there=wrong), $(VAR:=add)'
+
+
+#
+# Local variables: regular forms, D/F forms and suffix substitution.
+#
+
+# In the past substitutions did not work with the D/F forms and those
+# forms were not available for $?.  (PR 49085)
+
+# dir/obj_1.o is inferred, obj2.o has an explicit rule.
+localvars: dir/obj_1.o obj2.o
+
+# $@ = target or archive name  $< = implied source
+# $* = target without suffix   $? = sources newer than target
+# $% = archive member name
+LOCALS = \
+       "Local variables\n\
+       \$$(@)=\"$(@)\" \$$(<)=\"$(<)\"\n\
+       \$$(*)=\"$(*)\" \$$(?)=\"$(?)\"\n\
+       \$$(%%)=\"$(%)\"\n\n"
+
+# $XD = directory part of X    $XF = file part of X
+# X is one of the local variables.
+LOCAL_ALTERNATIVES = \
+       "Directory and filename parts of local variables\n\
+       \$$(@D)=\"$(@D)\" \$$(@F)=\"$(@F)\"\n\
+       \$$(<D)=\"$(<D)\" \$$(<F)=\"$(<F)\"\n\
+       \$$(*D)=\"$(*D)\" \$$(*F)=\"$(*F)\"\n\
+       \$$(?D)=\"$(?D)\" \$$(?F)=\"$(?F)\"\n\
+       \$$(%%D)=\"$(%D)\" \$$(%%F)=\"$(%F)\"\n\n"
+
+# Do all kinds of meaningless substitutions on local variables to see
+# if they work.  Add, remove and replace things.
+VAR2 = .o
+VAR3 = foo
+LOCAL_SUBSTITUTIONS = \
+       "Local variable substitutions\n\
+       \$$(@:.o=)=\"$(@:.o=)\" \$$(<:.c=.C)=\"$(<:.c=.C)\"\n\
+       \$$(*:=.h)=\"$(*:=.h)\" \$$(?:.h=.H)=\"$(?:.h=.H)\"\n\
+       \$$(%%:=)=\"$(%:=)\"\n\n"
+
+LOCAL_ALTERNATIVE_SUBSTITUTIONS = \
+       "Target with suffix transformations\n\
+       \$$(@D:=append)=\"$(@D:=append)\"\n\
+       \$$(@F:.o=.O)=\"$(@F:.o=.O)\"\n\
+       \n\
+       Implied source with suffix transformations\n\
+       \$$(<D:r=rr)=\"$(<D:r=rr)\"\n\
+       \$$(<F:.c=.C)=\"$(<F:.c=.C)\"\n\
+       \n\
+       Suffixless target with suffix transformations\n\
+       \$$(*D:.=dot)=\"$(*D:.=dot)\"\n\
+       \$$(*F:.a=)=\"$(*F:.a=)\"\n\
+       \n\
+       Out-of-date dependencies with suffix transformations\n\
+       \$$(?D:ir=)=\"$(?D:ir=)\"\n\
+       \$$(?F:.h=.H)=\"$(?F:.h=.H)\"\n\
+       \n\
+       Member with suffix transformations\n\
+       \$$(%%D:.=)=\"$(%D:.=)\"\n\
+       \$$(%%F:\$$(VAR2)=\$$(VAR))=\"$(%F:$(VAR2)=$(VAR))\"\n\n"
+
+.SUFFIXES: .c .o
+
+.c.o:
+       @printf $(LOCALS)
+       @printf $(LOCAL_ALTERNATIVES)
+       @printf $(LOCAL_SUBSTITUTIONS)
+       @printf $(LOCAL_ALTERNATIVE_SUBSTITUTIONS)
+       cc -c -o '$(@)' '$(<)'
+
+# Some of these rules are padded with useless extra dependencies just so
+# that $(?) has more than one file.
+
+dir/obj_1.o: dir/obj_1.h
+
+# According to POSIX, $* is only required for inference rules and $<'s
+# value is unspecified outside of inference rules.  Strictly speaking
+# we shouldn't be expanding them here but who cares.  At least we get
+# to check that the program does nothing stupid (like crash) with them.
+# The C file is named differently from the object file because there
+# was a bug which forced dependencies based on inference rules on all
+# applicable targets (PR 49086).
+obj2.o: obj_2.c obj_2.h dir/obj_1.h
+       @printf $(LOCALS)
+       @printf $(LOCAL_ALTERNATIVES)
+       @printf $(LOCAL_SUBSTITUTIONS)
+       @printf $(LOCAL_ALTERNATIVE_SUBSTITUTIONS)
+       cc -c -o '$(@)' 'obj_2.c'
+
+# Hey, this is make, we can make our own test data setup!  obj2.c is not
+# used, so it should not get created.  It's here as a bait for
+# a regression into the forced dependencies discussed earlier.
+dir/obj_1.c obj2.c obj_2.c:
+       mkdir -p '$(@D)'
+       printf '#include "$(@F:.c=.h)"\nconst char* $(@F:.c=) = "$(@)";' \
+           >'$(@)'
+
+dir/obj_1.h obj_2.h:
+       mkdir -p '$(@D)'
+       touch '$(@)'
diff -r 7c12af8ea25b -r 25cf428e5a4a tests/usr.bin/make/d_posix.out
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/usr.bin/make/d_posix.out    Sat Aug 23 14:50:24 2014 +0000
@@ -0,0 +1,91 @@
+$(VAR) = "foo  bar baz"
+a
+b
+c
+foo baR baz,  bar baz, foo bar baz, fooadd baradd bazadd
+mkdir -p 'dir'
+touch 'dir/obj_1.h'
+mkdir -p 'dir'
+printf '#include "obj_1.h"\nconst char* obj_1 = "dir/obj_1.c";' \
+    >'dir/obj_1.c'
+Local variables
+ $(@)="dir/obj_1.o" $(<)="dir/obj_1.c"
+ $(*)="dir/obj_1" $(?)="dir/obj_1.h dir/obj_1.c"
+ $(%)=""
+
+Directory and filename parts of local variables
+ $(@D)="dir" $(@F)="obj_1.o"
+ $(<D)="dir" $(<F)="obj_1.c"
+ $(*D)="dir" $(*F)="obj_1"
+ $(?D)="dir dir" $(?F)="obj_1.h obj_1.c"
+ $(%D)="" $(%F)=""
+
+Local variable substitutions
+ $(@:.o=)="dir/obj_1" $(<:.c=.C)="dir/obj_1.C"
+ $(*:=.h)="dir/obj_1.h" $(?:.h=.H)="dir/obj_1.H dir/obj_1.c"
+ $(%:=)=""
+
+Target with suffix transformations
+ $(@D:=append)="dirappend"
+ $(@F:.o=.O)="obj_1.O"
+ 
+ Implied source with suffix transformations
+ $(<D:r=rr)="dirr"
+ $(<F:.c=.C)="obj_1.C"
+ 
+ Suffixless target with suffix transformations
+ $(*D:.=dot)="dir"
+ $(*F:.a=)="obj_1"
+ 
+ Out-of-date dependencies with suffix transformations
+ $(?D:ir=)="d d"
+ $(?F:.h=.H)="obj_1.H obj_1.c"
+ 
+ Member with suffix transformations
+ $(%D:.=)=""
+ $(%F:$(VAR2)=$(VAR))=""
+
+cc -c -o 'dir/obj_1.o' 'dir/obj_1.c'
+mkdir -p '.'
+printf '#include "obj_2.h"\nconst char* obj_2 = "obj_2.c";' \
+    >'obj_2.c'
+mkdir -p '.'
+touch 'obj_2.h'
+Local variables
+ $(@)="obj2.o" $(<)=""
+ $(*)="obj2" $(?)="obj_2.c obj_2.h dir/obj_1.h"
+ $(%)=""
+
+Directory and filename parts of local variables
+ $(@D)="." $(@F)="obj2.o"
+ $(<D)="" $(<F)=""
+ $(*D)="." $(*F)="obj2"
+ $(?D)=". . dir" $(?F)="obj_2.c obj_2.h obj_1.h"
+ $(%D)="" $(%F)=""
+
+Local variable substitutions
+ $(@:.o=)="obj2" $(<:.c=.C)=""
+ $(*:=.h)="obj2.h" $(?:.h=.H)="obj_2.c obj_2.H dir/obj_1.H"
+ $(%:=)=""
+
+Target with suffix transformations
+ $(@D:=append)=".append"
+ $(@F:.o=.O)="obj2.O"
+ 
+ Implied source with suffix transformations
+ $(<D:r=rr)=""
+ $(<F:.c=.C)=""
+ 
+ Suffixless target with suffix transformations
+ $(*D:.=dot)="dot"
+ $(*F:.a=)="obj2"
+ 
+ Out-of-date dependencies with suffix transformations
+ $(?D:ir=)=". . d"
+ $(?F:.h=.H)="obj_2.c obj_2.H obj_1.H"
+ 
+ Member with suffix transformations
+ $(%D:.=)=""
+ $(%F:$(VAR2)=$(VAR))=""
+
+cc -c -o 'obj2.o' 'obj_2.c'
diff -r 7c12af8ea25b -r 25cf428e5a4a tests/usr.bin/make/t_make.sh
--- a/tests/usr.bin/make/t_make.sh      Sat Aug 23 08:03:33 2014 +0000
+++ b/tests/usr.bin/make/t_make.sh      Sat Aug 23 14:50:24 2014 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: t_make.sh,v 1.2 2014/08/22 16:45:32 apb Exp $
+# $NetBSD: t_make.sh,v 1.3 2014/08/23 14:50:24 christos Exp $
 #
 # Copyright (c) 2008, 2010, 2014 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -32,12 +32,28 @@
        local makename="${1}"; shift
 
        local srcdir="$(atf_get_srcdir)"
-       local testdir="$(atf_get_srcdir)/unit-tests"
+       local in="${srcdir}/d_${name}.mk"
+       local out="${srcdir}/d_${name}.out"
 
-       atf_check -s exit:0 -o ignore -e ignore \
-           make -f "${testdir}/Makefile" "${makename}.out"
-       atf_check -o file:"${testdir}/${makename}.exp" \
-           cat "${makename}.out"
+       if [ "x${name}" = "xposix" ]; then
+               # Include $(INPUTFILE) for d_posix.mk, so it can re-run make
+               # on the same makefile.  Make sets $(MAKEFILE), but it is
+               # not in POSIX, so it can't be used as such.  It can't be
+               # set explicitly because make always sets it itself and
+               # the test shouldn't use anything not provided for by in
+               # the POSIX standard.
+               args="INPUTFILE='${in}'"
+               atf_expect_fail 'PR/49086 [$(<)], PR/49092 [output order]'
+               atf_check -o file:"${out}" -x \
+                   "make -kf'${in}' ${args} 2>&1 | sed -e 's,${srcdir}/d_,,'"
+       else
+               local testdir="$(atf_get_srcdir)/unit-tests"
+
+               atf_check -s exit:0 -o ignore -e ignore \
+               make -f "${testdir}/Makefile" "${makename}.out"
+               atf_check -o file:"${testdir}/${makename}.exp" \
+                   cat "${makename}.out"
+       fi
 }
 
 # Defines a test case for make(1), parsing a given file and comparing the
diff -r 7c12af8ea25b -r 25cf428e5a4a usr.bin/make/make.1
--- a/usr.bin/make/make.1       Sat Aug 23 08:03:33 2014 +0000
+++ b/usr.bin/make/make.1       Sat Aug 23 14:50:24 2014 +0000
@@ -1,4 +1,4 @@
-.\"    $NetBSD: make.1,v 1.230 2014/02/15 18:55:30 sjg Exp $
+.\"    $NetBSD: make.1,v 1.231 2014/08/23 14:50:24 christos Exp $
 .\"
 .\" Copyright (c) 1990, 1993
 .\"    The Regents of the University of California.  All rights reserved.
@@ -382,8 +382,11 @@
 In general, lines may be continued from one line to the next by ending
 them with a backslash
 .Pq Ql \e .
-The trailing newline character and initial whitespace on the following
-line are compressed into a single space.
+For any line that is not a shell command line (i.e. it does not begin