pkgsrc-Changes archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
CVS commit: pkgsrc/pkgtools/pkglint
Module Name: pkgsrc
Committed By: rillig
Date: Sat Oct 26 09:51:48 UTC 2019
Modified Files:
pkgsrc/pkgtools/pkglint: Makefile PLIST
pkgsrc/pkgtools/pkglint/files: buildlink3_test.go category.go
check_test.go files_test.go mkline.go mkline_test.go
mklinechecker.go mklinechecker_test.go mklines.go mkparser_test.go
mktokenslexer_test.go mktypes_test.go package.go
shtokenizer_test.go substcontext.go toplevel.go util.go
varalignblock.go vargroups_test.go vartypecheck.go
vartypecheck_test.go
Added Files:
pkgsrc/pkgtools/pkglint/files: mklineparser.go mklineparser_test.go
Log Message:
pkgtools/pkglint: update to 19.3.2
Changes since 19.3.1:
* Pkglint no longer warns about a missing :Q modifier if there is also
a :D modifier, since the latter hides the original variable value
from the expression value.
* Variable names like .CURDIR are now allowed in the _VARGROUPS section.
* In dependency lines like "${_COOKIE.extract}:", pkglint no longer
warns about the unknown target. No matter whether this is a file name
or even a list of other targets, there's no chance for a typo here.
* If some dependencies are included conditionally, and the package
Makefile and buildlink3.mk disagree, and the conditions depend on
PKG_OPTIONS, pkglint outputs a helpful explanation.
* The check for including builtin.mk directly can be disabled by giving
a reason in a comment at the end of the line.
To generate a diff of this commit:
cvs rdiff -u -r1.601 -r1.602 pkgsrc/pkgtools/pkglint/Makefile
cvs rdiff -u -r1.14 -r1.15 pkgsrc/pkgtools/pkglint/PLIST
cvs rdiff -u -r1.33 -r1.34 pkgsrc/pkgtools/pkglint/files/buildlink3_test.go \
pkgsrc/pkgtools/pkglint/files/mkparser_test.go
cvs rdiff -u -r1.22 -r1.23 pkgsrc/pkgtools/pkglint/files/category.go
cvs rdiff -u -r1.49 -r1.50 pkgsrc/pkgtools/pkglint/files/check_test.go
cvs rdiff -u -r1.27 -r1.28 pkgsrc/pkgtools/pkglint/files/files_test.go
cvs rdiff -u -r1.59 -r1.60 pkgsrc/pkgtools/pkglint/files/mkline.go
cvs rdiff -u -r1.68 -r1.69 pkgsrc/pkgtools/pkglint/files/mkline_test.go
cvs rdiff -u -r1.47 -r1.48 pkgsrc/pkgtools/pkglint/files/mklinechecker.go
cvs rdiff -u -r1.43 -r1.44 \
pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go
cvs rdiff -u -r0 -r1.1 pkgsrc/pkgtools/pkglint/files/mklineparser.go \
pkgsrc/pkgtools/pkglint/files/mklineparser_test.go
cvs rdiff -u -r1.55 -r1.56 pkgsrc/pkgtools/pkglint/files/mklines.go
cvs rdiff -u -r1.4 -r1.5 pkgsrc/pkgtools/pkglint/files/mktokenslexer_test.go
cvs rdiff -u -r1.14 -r1.15 pkgsrc/pkgtools/pkglint/files/mktypes_test.go
cvs rdiff -u -r1.64 -r1.65 pkgsrc/pkgtools/pkglint/files/package.go \
pkgsrc/pkgtools/pkglint/files/vartypecheck.go
cvs rdiff -u -r1.17 -r1.18 pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go
cvs rdiff -u -r1.28 -r1.29 pkgsrc/pkgtools/pkglint/files/substcontext.go
cvs rdiff -u -r1.21 -r1.22 pkgsrc/pkgtools/pkglint/files/toplevel.go
cvs rdiff -u -r1.53 -r1.54 pkgsrc/pkgtools/pkglint/files/util.go
cvs rdiff -u -r1.5 -r1.6 pkgsrc/pkgtools/pkglint/files/varalignblock.go
cvs rdiff -u -r1.3 -r1.4 pkgsrc/pkgtools/pkglint/files/vargroups_test.go
cvs rdiff -u -r1.56 -r1.57 pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: pkgsrc/pkgtools/pkglint/Makefile
diff -u pkgsrc/pkgtools/pkglint/Makefile:1.601 pkgsrc/pkgtools/pkglint/Makefile:1.602
--- pkgsrc/pkgtools/pkglint/Makefile:1.601 Fri Oct 18 14:58:55 2019
+++ pkgsrc/pkgtools/pkglint/Makefile Sat Oct 26 09:51:47 2019
@@ -1,7 +1,6 @@
-# $NetBSD: Makefile,v 1.601 2019/10/18 14:58:55 bsiegert Exp $
+# $NetBSD: Makefile,v 1.602 2019/10/26 09:51:47 rillig Exp $
-PKGNAME= pkglint-19.3.1
-PKGREVISION= 1
+PKGNAME= pkglint-19.3.2
CATEGORIES= pkgtools
DISTNAME= tools
MASTER_SITES= ${MASTER_SITE_GITHUB:=golang/}
Index: pkgsrc/pkgtools/pkglint/PLIST
diff -u pkgsrc/pkgtools/pkglint/PLIST:1.14 pkgsrc/pkgtools/pkglint/PLIST:1.15
--- pkgsrc/pkgtools/pkglint/PLIST:1.14 Sun Sep 8 22:47:47 2019
+++ pkgsrc/pkgtools/pkglint/PLIST Sat Oct 26 09:51:47 2019
@@ -1,4 +1,4 @@
-@comment $NetBSD: PLIST,v 1.14 2019/09/08 22:47:47 rillig Exp $
+@comment $NetBSD: PLIST,v 1.15 2019/10/26 09:51:47 rillig Exp $
bin/pkglint
gopkg/pkg/${GO_PLATFORM}/netbsd.org/pkglint.a
gopkg/pkg/${GO_PLATFORM}/netbsd.org/pkglint/getopt.a
@@ -52,6 +52,8 @@ gopkg/src/netbsd.org/pkglint/mkline.go
gopkg/src/netbsd.org/pkglint/mkline_test.go
gopkg/src/netbsd.org/pkglint/mklinechecker.go
gopkg/src/netbsd.org/pkglint/mklinechecker_test.go
+gopkg/src/netbsd.org/pkglint/mklineparser.go
+gopkg/src/netbsd.org/pkglint/mklineparser_test.go
gopkg/src/netbsd.org/pkglint/mklines.go
gopkg/src/netbsd.org/pkglint/mklines_test.go
gopkg/src/netbsd.org/pkglint/mkparser.go
Index: pkgsrc/pkgtools/pkglint/files/buildlink3_test.go
diff -u pkgsrc/pkgtools/pkglint/files/buildlink3_test.go:1.33 pkgsrc/pkgtools/pkglint/files/buildlink3_test.go:1.34
--- pkgsrc/pkgtools/pkglint/files/buildlink3_test.go:1.33 Fri Oct 11 23:30:02 2019
+++ pkgsrc/pkgtools/pkglint/files/buildlink3_test.go Sat Oct 26 09:51:48 2019
@@ -196,7 +196,7 @@ func (s *Suite) Test_CheckLinesBuildlink
t.CheckOutputEmpty()
}
-func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch__version(c *check.C) {
+func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch__version_ok(c *check.C) {
t := s.Init(c)
t.SetUpPackage("editors/emacs22",
@@ -229,6 +229,35 @@ func (s *Suite) Test_CheckLinesBuildlink
t.CheckOutputEmpty()
}
+func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch__version_bad(c *check.C) {
+ t := s.Init(c)
+
+ t.SetUpPackage("editors/emacs-client",
+ "PKGNAME=\temacs-client-22.0")
+ t.CreateFileLines("editors/emacs-client/buildlink3.mk",
+ MkCvsID,
+ "",
+ "BUILDLINK_TREE+=\temacs",
+ "",
+ ".if !defined(EMACS_BUILDLINK3_MK)",
+ "EMACS_BUILDLINK3_MK:=",
+ "",
+ "BUILDLINK_API_DEPENDS.emacs+=\temacs-client>=1.0",
+ "BUILDLINK_ABI_DEPENDS.emacs+=\temacs-client>=1.0",
+ "",
+ ".endif\t# EMACS_BUILDLINK3_MK",
+ "",
+ "BUILDLINK_TREE+=\t-emacs")
+ t.FinishSetUp()
+
+ G.Check(t.File("editors/emacs-client"))
+
+ t.CheckOutputLines(
+ "ERROR: ~/editors/emacs-client/buildlink3.mk:3: " +
+ "Package name mismatch between \"emacs\" in this file " +
+ "and \"emacs-client\" from Makefile:4.")
+}
+
func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_multiple_inclusion(c *check.C) {
t := s.Init(c)
Index: pkgsrc/pkgtools/pkglint/files/mkparser_test.go
diff -u pkgsrc/pkgtools/pkglint/files/mkparser_test.go:1.33 pkgsrc/pkgtools/pkglint/files/mkparser_test.go:1.34
--- pkgsrc/pkgtools/pkglint/files/mkparser_test.go:1.33 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/mkparser_test.go Sat Oct 26 09:51:48 2019
@@ -7,6 +7,7 @@ import (
func (s *Suite) Test_MkParser_MkTokens(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
testRest := func(input string, expectedTokens []*MkToken, expectedRest string) {
line := t.NewLines("Test_MkParser_MkTokens.mk", input).Lines[0]
@@ -22,19 +23,10 @@ func (s *Suite) Test_MkParser_MkTokens(c
t.CheckEquals(p.Rest(), expectedRest)
}
test := func(input string, expectedToken *MkToken) {
- testRest(input, []*MkToken{expectedToken}, "")
- }
- literal := func(text string) *MkToken {
- return &MkToken{Text: text}
- }
- varuse := func(varname string, modifiers ...string) *MkToken {
- text := "${" + varname
- for _, modifier := range modifiers {
- text += ":" + modifier
- }
- text += "}"
- return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)}
+ testRest(input, b.Tokens(expectedToken), "")
}
+ literal := b.TextToken
+ varuse := b.VaruseToken
// Everything except VarUses is passed through unmodified.
@@ -53,38 +45,42 @@ func (s *Suite) Test_MkParser_MkTokens(c
test("$$var1 $$var2 $$? $$",
literal("$$var1 $$var2 $$? $$"))
- testRest("hello, ${W:L:tl}orld", []*MkToken{
- literal("hello, "),
- varuse("W", "L", "tl"),
- literal("orld")},
+ testRest("hello, ${W:L:tl}orld",
+ b.Tokens(
+ literal("hello, "),
+ varuse("W", "L", "tl"),
+ literal("orld")),
"")
- testRest("ftp://${PKGNAME}/ ${MASTER_SITES:=subdir/}", []*MkToken{
- literal("ftp://";),
- varuse("PKGNAME"),
- literal("/ "),
- varuse("MASTER_SITES", "=subdir/")},
+ testRest("ftp://${PKGNAME}/ ${MASTER_SITES:=subdir/}",
+ b.Tokens(
+ literal("ftp://";),
+ varuse("PKGNAME"),
+ literal("/ "),
+ varuse("MASTER_SITES", "=subdir/")),
"")
testRest("${VAR:S,a,b,c,d,e,f}",
- []*MkToken{{
- Text: "${VAR:S,a,b,c,d,e,f}",
- Varuse: NewMkVarUse("VAR", "S,a,b,")}},
+ b.Tokens(b.VaruseTextToken("${VAR:S,a,b,c,d,e,f}", "VAR", "S,a,b,")),
"")
t.CheckOutputLines(
"WARN: Test_MkParser_MkTokens.mk:1: Invalid variable modifier \"c,d,e,f\" for \"VAR\".")
- testRest("Text${VAR:Mmodifier}${VAR2}more text${VAR3}", []*MkToken{
- literal("Text"),
- varuse("VAR", "Mmodifier"),
- varuse("VAR2"),
- literal("more text"),
- varuse("VAR3")},
+ testRest("Text${VAR:Mmodifier}${VAR2}more text${VAR3}",
+ b.Tokens(
+ literal("Text"),
+ varuse("VAR", "Mmodifier"),
+ varuse("VAR2"),
+ literal("more text"),
+ varuse("VAR3")),
"")
}
func (s *Suite) Test_MkParser_VarUse(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
+ varuse := b.VaruseToken
+ varuseText := b.VaruseTextToken
testRest := func(input string, expectedTokens []*MkToken, expectedRest string, diagnostics ...string) {
line := t.NewLines("Test_MkParser_VarUse.mk", input).Lines[0]
@@ -102,23 +98,11 @@ func (s *Suite) Test_MkParser_VarUse(c *
t.CheckEquals(p.Rest(), expectedRest)
t.CheckOutput(diagnostics)
}
- tokens := func(tokens ...*MkToken) []*MkToken { return tokens }
test := func(input string, expectedToken *MkToken, diagnostics ...string) {
- testRest(input, []*MkToken{expectedToken}, "", diagnostics...)
- }
- varuse := func(varname string, modifiers ...string) *MkToken {
- text := "${" + varname
- for _, modifier := range modifiers {
- text += ":" + modifier
- }
- text += "}"
- return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)}
- }
- varuseText := func(text, varname string, modifiers ...string) *MkToken {
- return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)}
+ testRest(input, b.Tokens(expectedToken), "", diagnostics...)
}
- t.Use(testRest, tokens, test, varuse, varuseText)
+ t.Use(testRest, test, varuse, varuseText)
test("${VARIABLE}",
varuse("VARIABLE"))
@@ -446,6 +430,7 @@ func (s *Suite) Test_MkParser_varUseModi
func (s *Suite) Test_MkParser_varUseModifier__condition_without_colon(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
line := t.NewLine("filename.mk", 123, "${${VAR}:?yes:no}${${VAR}:?yes}")
p := NewMkParser(line, line.Text)
@@ -453,8 +438,8 @@ func (s *Suite) Test_MkParser_varUseModi
varUse1 := p.VarUse()
varUse2 := p.VarUse()
- t.CheckDeepEquals(varUse1, NewMkVarUse("${VAR}", "?yes:no"))
- t.CheckDeepEquals(varUse2, NewMkVarUse("${VAR}"))
+ t.CheckDeepEquals(varUse1, b.VarUse("${VAR}", "?yes:no"))
+ t.CheckDeepEquals(varUse2, b.VarUse("${VAR}"))
t.CheckEquals(p.Rest(), "")
t.CheckOutputLines(
@@ -463,13 +448,14 @@ func (s *Suite) Test_MkParser_varUseModi
func (s *Suite) Test_MkParser_varUseModifier__malformed_in_parentheses(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
line := t.NewLine("filename.mk", 123, "$(${VAR}:?yes)")
p := NewMkParser(line, line.Text)
varUse := p.VarUse()
- t.CheckDeepEquals(varUse, NewMkVarUse("${VAR}"))
+ t.CheckDeepEquals(varUse, b.VarUse("${VAR}"))
t.CheckEquals(p.Rest(), "")
t.CheckOutputLines(
@@ -479,13 +465,14 @@ func (s *Suite) Test_MkParser_varUseModi
func (s *Suite) Test_MkParser_varUseModifier__varuse_in_malformed_modifier(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
line := t.NewLine("filename.mk", 123, "${${VAR}:?yes${INNER}}")
p := NewMkParser(line, line.Text)
varUse := p.VarUse()
- t.CheckDeepEquals(varUse, NewMkVarUse("${VAR}"))
+ t.CheckDeepEquals(varUse, b.VarUse("${VAR}"))
t.CheckEquals(p.Rest(), "")
t.CheckOutputLines(
@@ -494,13 +481,14 @@ func (s *Suite) Test_MkParser_varUseModi
func (s *Suite) Test_MkParser_varUseModifierAt__missing_at_after_variable_name(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
line := t.NewLine("filename.mk", 123, "${VAR:@varname}")
p := NewMkParser(line, line.Text)
varUse := p.VarUse()
- t.CheckDeepEquals(varUse, NewMkVarUse("VAR"))
+ t.CheckDeepEquals(varUse, b.VarUse("VAR"))
t.CheckEquals(p.Rest(), "")
t.CheckOutputLines(
"WARN: filename.mk:123: Invalid variable modifier \"@varname\" for \"VAR\".")
@@ -508,31 +496,34 @@ func (s *Suite) Test_MkParser_varUseModi
func (s *Suite) Test_MkParser_varUseModifierAt__dollar(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
line := t.NewLine("filename.mk", 123, "${VAR:@var@$$var@}")
p := NewMkParser(line, line.Text)
varUse := p.VarUse()
- t.CheckDeepEquals(varUse, NewMkVarUse("VAR", "@var@$$var@"))
+ t.CheckDeepEquals(varUse, b.VarUse("VAR", "@var@$$var@"))
t.CheckEquals(p.Rest(), "")
t.CheckOutputEmpty()
}
func (s *Suite) Test_MkParser_varUseModifierAt__incomplete_without_warning(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
p := NewMkParser(nil, "${VAR:@var@$$var}rest")
varUse := p.VarUse()
- t.CheckDeepEquals(varUse, NewMkVarUse("VAR", "@var@$$var}rest"))
+ t.CheckDeepEquals(varUse, b.VarUse("VAR", "@var@$$var}rest"))
t.CheckEquals(p.Rest(), "")
t.CheckOutputEmpty()
}
func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
t.SetUpCommandLine("--explain")
@@ -540,10 +531,10 @@ func (s *Suite) Test_MkParser_VarUse__am
p := NewMkParser(line, line.Text[1:])
tokens := p.MkTokens()
- t.CheckDeepEquals(tokens, []*MkToken{
- {"$V", NewMkVarUse("V")},
- {"arname ", nil},
- {"$X", NewMkVarUse("X")}})
+ t.CheckDeepEquals(tokens, b.Tokens(
+ b.VaruseTextToken("$V", "V"),
+ b.TextToken("arname "),
+ b.VaruseTextToken("$X", "X")))
t.CheckOutputLines(
"ERROR: module.mk:123: $Varname is ambiguous. Use ${Varname} if you mean a Make variable or $$Varname if you mean a shell variable.",
@@ -563,6 +554,8 @@ func (s *Suite) Test_MkParser_VarUse__am
func (s *Suite) Test_MkParser_MkCond(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
+ varUse := b.VarUse
testRest := func(input string, expectedTree *MkCond, expectedRest string) {
// As of July 2019 p.MkCond does not emit warnings;
@@ -576,57 +569,57 @@ func (s *Suite) Test_MkParser_MkCond(c *
test := func(input string, expectedTree *MkCond) {
testRest(input, expectedTree, "")
}
- varUse := func(name string, modifiers ...string) MkCondTerm {
- return MkCondTerm{Var: NewMkVarUse(name, modifiers...)}
+ varTerm := func(name string, modifiers ...string) MkCondTerm {
+ return MkCondTerm{Var: varUse(name, modifiers...)}
}
str := func(s string) MkCondTerm { return MkCondTerm{Str: s} }
num := func(s string) MkCondTerm { return MkCondTerm{Num: s} }
- t.Use(testRest, test, varUse)
+ t.Use(testRest, test, varTerm)
test("${OPSYS:MNetBSD}",
- &MkCond{Term: &MkCondTerm{Var: NewMkVarUse("OPSYS", "MNetBSD")}})
+ &MkCond{Term: &MkCondTerm{Var: varUse("OPSYS", "MNetBSD")}})
test("defined(VARNAME)",
&MkCond{Defined: "VARNAME"})
test("empty(VARNAME)",
- &MkCond{Empty: NewMkVarUse("VARNAME")})
+ &MkCond{Empty: varUse("VARNAME")})
test("!empty(VARNAME)",
- &MkCond{Not: &MkCond{Empty: NewMkVarUse("VARNAME")}})
+ &MkCond{Not: &MkCond{Empty: varUse("VARNAME")}})
test("!empty(VARNAME:M[yY][eE][sS])",
- &MkCond{Not: &MkCond{Empty: NewMkVarUse("VARNAME", "M[yY][eE][sS]")}})
+ &MkCond{Not: &MkCond{Empty: varUse("VARNAME", "M[yY][eE][sS]")}})
// Colons are unescaped at this point because they cannot be mistaken for separators anymore.
test("!empty(USE_TOOLS:Mautoconf\\:run)",
- &MkCond{Not: &MkCond{Empty: NewMkVarUse("USE_TOOLS", "Mautoconf:run")}})
+ &MkCond{Not: &MkCond{Empty: varUse("USE_TOOLS", "Mautoconf:run")}})
test("${VARNAME} != \"Value\"",
- &MkCond{Compare: &MkCondCompare{varUse("VARNAME"), "!=", str("Value")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("VARNAME"), "!=", str("Value")}})
test("${VARNAME:Mi386} != \"Value\"",
- &MkCond{Compare: &MkCondCompare{varUse("VARNAME", "Mi386"), "!=", str("Value")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("VARNAME", "Mi386"), "!=", str("Value")}})
test("${VARNAME} != Value",
- &MkCond{Compare: &MkCondCompare{varUse("VARNAME"), "!=", str("Value")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("VARNAME"), "!=", str("Value")}})
test("\"${VARNAME}\" != Value",
- &MkCond{Compare: &MkCondCompare{varUse("VARNAME"), "!=", str("Value")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("VARNAME"), "!=", str("Value")}})
test("${pkg} == \"${name}\"",
- &MkCond{Compare: &MkCondCompare{varUse("pkg"), "==", varUse("name")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("pkg"), "==", varTerm("name")}})
test("\"${pkg}\" == \"${name}\"",
- &MkCond{Compare: &MkCondCompare{varUse("pkg"), "==", varUse("name")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("pkg"), "==", varTerm("name")}})
// The right-hand side is not analyzed further to keep the data types simple.
test("${ABC} == \"${A}B${C}\"",
- &MkCond{Compare: &MkCondCompare{varUse("ABC"), "==", str("${A}B${C}")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("ABC"), "==", str("${A}B${C}")}})
test("${ABC} == \"${A}\\\"${B}\\\\${C}$${shellvar}${D}\"",
- &MkCond{Compare: &MkCondCompare{varUse("ABC"), "==", str("${A}\"${B}\\${C}$${shellvar}${D}")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("ABC"), "==", str("${A}\"${B}\\${C}$${shellvar}${D}")}})
test("exists(/etc/hosts)",
&MkCond{Call: &MkCondCall{"exists", "/etc/hosts"}})
@@ -636,13 +629,13 @@ func (s *Suite) Test_MkParser_MkCond(c *
test("${OPSYS} == \"NetBSD\" || ${OPSYS} == \"OpenBSD\"",
&MkCond{Or: []*MkCond{
- {Compare: &MkCondCompare{varUse("OPSYS"), "==", str("NetBSD")}},
- {Compare: &MkCondCompare{varUse("OPSYS"), "==", str("OpenBSD")}}}})
+ {Compare: &MkCondCompare{varTerm("OPSYS"), "==", str("NetBSD")}},
+ {Compare: &MkCondCompare{varTerm("OPSYS"), "==", str("OpenBSD")}}}})
test("${OPSYS} == \"NetBSD\" && ${MACHINE_ARCH} == \"i386\"",
&MkCond{And: []*MkCond{
- {Compare: &MkCondCompare{varUse("OPSYS"), "==", str("NetBSD")}},
- {Compare: &MkCondCompare{varUse("MACHINE_ARCH"), "==", str("i386")}}}})
+ {Compare: &MkCondCompare{varTerm("OPSYS"), "==", str("NetBSD")}},
+ {Compare: &MkCondCompare{varTerm("MACHINE_ARCH"), "==", str("i386")}}}})
test("defined(A) && defined(B) || defined(C) && defined(D)",
&MkCond{Or: []*MkCond{
@@ -655,11 +648,11 @@ func (s *Suite) Test_MkParser_MkCond(c *
test("${MACHINE_ARCH:Mi386} || ${MACHINE_OPSYS:MNetBSD}",
&MkCond{Or: []*MkCond{
- {Term: &MkCondTerm{Var: NewMkVarUse("MACHINE_ARCH", "Mi386")}},
- {Term: &MkCondTerm{Var: NewMkVarUse("MACHINE_OPSYS", "MNetBSD")}}}})
+ {Term: &MkCondTerm{Var: varUse("MACHINE_ARCH", "Mi386")}},
+ {Term: &MkCondTerm{Var: varUse("MACHINE_OPSYS", "MNetBSD")}}}})
test("${VAR} == \"${VAR}suffix\"",
- &MkCond{Compare: &MkCondCompare{varUse("VAR"), "==", str("${VAR}suffix")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("VAR"), "==", str("${VAR}suffix")}})
// Exotic cases
@@ -673,7 +666,7 @@ func (s *Suite) Test_MkParser_MkCond(c *
test("${VAR} == 0xCAFEBABE",
&MkCond{
Compare: &MkCondCompare{
- varUse("VAR"),
+ varTerm("VAR"),
"==",
num("0xCAFEBABE")}})
@@ -681,26 +674,26 @@ func (s *Suite) Test_MkParser_MkCond(c *
&MkCond{Not: &MkCond{
And: []*MkCond{
{Defined: "A"},
- {Empty: NewMkVarUse("VARNAME")}}}})
+ {Empty: varUse("VARNAME")}}}})
test("${REQD_MAJOR} > ${MAJOR}",
- &MkCond{Compare: &MkCondCompare{varUse("REQD_MAJOR"), ">", varUse("MAJOR")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("REQD_MAJOR"), ">", varTerm("MAJOR")}})
test("${OS_VERSION} >= 6.5",
- &MkCond{Compare: &MkCondCompare{varUse("OS_VERSION"), ">=", num("6.5")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("OS_VERSION"), ">=", num("6.5")}})
test("${OS_VERSION} == 5.3",
- &MkCond{Compare: &MkCondCompare{varUse("OS_VERSION"), "==", num("5.3")}})
+ &MkCond{Compare: &MkCondCompare{varTerm("OS_VERSION"), "==", num("5.3")}})
test("!empty(${OS_VARIANT:MIllumos})", // Probably not intended
- &MkCond{Not: &MkCond{Empty: NewMkVarUse("${OS_VARIANT:MIllumos}")}})
+ &MkCond{Not: &MkCond{Empty: varUse("${OS_VARIANT:MIllumos}")}})
// There may be whitespace before the parenthesis; see devel/bmake/files/cond.c:^compare_function.
test("defined (VARNAME)",
&MkCond{Defined: "VARNAME"})
test("${\"${PKG_OPTIONS:Moption}\":?--enable-option:--disable-option}",
- &MkCond{Term: &MkCondTerm{Var: NewMkVarUse("\"${PKG_OPTIONS:Moption}\"", "?--enable-option:--disable-option")}})
+ &MkCond{Term: &MkCondTerm{Var: varUse("\"${PKG_OPTIONS:Moption}\"", "?--enable-option:--disable-option")}})
// Contrary to most other programming languages, the == operator binds
// more tightly that the ! operator.
@@ -708,7 +701,7 @@ func (s *Suite) Test_MkParser_MkCond(c *
// TODO: Since this operator precedence is surprising there should be a warning,
// suggesting to replace "!${VAR} == value" with "${VAR} != value".
test("!${VAR} == value",
- &MkCond{Not: &MkCond{Compare: &MkCondCompare{varUse("VAR"), "==", str("value")}}})
+ &MkCond{Not: &MkCond{Compare: &MkCondCompare{varTerm("VAR"), "==", str("value")}}})
// The left-hand side of the comparison can be a quoted string.
test("\"${VAR}suffix\" == value",
@@ -744,11 +737,11 @@ func (s *Suite) Test_MkParser_MkCond(c *
"exists(/unfinished")
testRest("!empty(PKG_OPTIONS:Msndfile) || defined(PKG_OPTIONS:Msamplerate)",
- &MkCond{Not: &MkCond{Empty: NewMkVarUse("PKG_OPTIONS", "Msndfile")}},
+ &MkCond{Not: &MkCond{Empty: varUse("PKG_OPTIONS", "Msndfile")}},
"|| defined(PKG_OPTIONS:Msamplerate)")
testRest("${LEFT} &&",
- &MkCond{Term: &MkCondTerm{Var: NewMkVarUse("LEFT")}},
+ &MkCond{Term: &MkCondTerm{Var: varUse("LEFT")}},
"&&")
testRest("\"unfinished string literal",
@@ -785,7 +778,7 @@ func (s *Suite) Test_MkParser_MkCond(c *
// Too many closing parentheses are a syntax error.
testRest("(${VAR}))",
- &MkCond{Term: &MkCondTerm{Var: NewMkVarUse("VAR")}},
+ &MkCond{Term: &MkCondTerm{Var: varUse("VAR")}},
")")
// The left-hand side of the comparison cannot be an unquoted string literal.
@@ -806,6 +799,7 @@ func (s *Suite) Test_MkParser_MkCond(c *
func (s *Suite) Test_MkParser_mkCondCompare(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
mkline := t.NewMkLine("Makefile", 123, ".if ${PKGPATH} == category/pack.age-3+")
p := NewMkParser(mkline.Line, mkline.Args())
@@ -816,7 +810,7 @@ func (s *Suite) Test_MkParser_mkCondComp
cond,
&MkCond{
Compare: &MkCondCompare{
- Left: MkCondTerm{Var: NewMkVarUse("PKGPATH")},
+ Left: MkCondTerm{Var: b.VarUse("PKGPATH")},
Op: "==",
Right: MkCondTerm{Str: "category/pack.age-3+"}}})
@@ -886,7 +880,7 @@ func (s *Suite) Test_MkParser_VarUse__pa
func (s *Suite) Test_MkParser_VarUseModifiers(c *check.C) {
t := s.Init(c)
- varUse := NewMkVarUse
+ varUse := NewMkTokenBuilder().VarUse
test := func(text string, varUse *MkVarUse, diagnostics ...string) {
line := t.NewLine("Makefile", 20, "\t"+text)
p := NewMkParser(line, text)
@@ -924,7 +918,7 @@ func (s *Suite) Test_MkParser_VarUseModi
func (s *Suite) Test_MkParser_varUseModifierSubst(c *check.C) {
t := s.Init(c)
- varUse := NewMkVarUse
+ varUse := NewMkTokenBuilder().VarUse
test := func(text string, varUse *MkVarUse, rest string, diagnostics ...string) {
line := t.NewLine("Makefile", 20, "\t"+text)
p := NewMkParser(line, text)
@@ -982,7 +976,7 @@ func (s *Suite) Test_MkParser_varUseModi
func (s *Suite) Test_MkParser_varUseModifierAt(c *check.C) {
t := s.Init(c)
- varUse := NewMkVarUse
+ varUse := NewMkTokenBuilder().VarUse
test := func(text string, varUse *MkVarUse, rest string, diagnostics ...string) {
line := t.NewLine("Makefile", 20, "\t"+text)
p := NewMkParser(line, text)
Index: pkgsrc/pkgtools/pkglint/files/category.go
diff -u pkgsrc/pkgtools/pkglint/files/category.go:1.22 pkgsrc/pkgtools/pkglint/files/category.go:1.23
--- pkgsrc/pkgtools/pkglint/files/category.go:1.22 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/category.go Sat Oct 26 09:51:48 2019
@@ -68,7 +68,7 @@ func CheckdirCategory(dir string) {
mlex.Skip()
name := mkline.Value()
- if mkline.IsCommentedVarassign() && mkline.VarassignComment() == "" {
+ if mkline.IsCommentedVarassign() && !mkline.HasComment() {
mkline.Warnf("%q commented out without giving a reason.", name)
}
Index: pkgsrc/pkgtools/pkglint/files/check_test.go
diff -u pkgsrc/pkgtools/pkglint/files/check_test.go:1.49 pkgsrc/pkgtools/pkglint/files/check_test.go:1.50
--- pkgsrc/pkgtools/pkglint/files/check_test.go:1.49 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/check_test.go Sat Oct 26 09:51:48 2019
@@ -852,7 +852,7 @@ func (t *Tester) NewMkLine(filename stri
basename == "mk.conf",
"filename %q must be realistic, otherwise the variable permissions are wrong", filename)
- return MkLineParser{}.Parse(t.NewLine(filename, lineno, text))
+ return NewMkLineParser().Parse(t.NewLine(filename, lineno, text))
}
func (t *Tester) NewShellLineChecker(text string) *ShellLineChecker {
Index: pkgsrc/pkgtools/pkglint/files/files_test.go
diff -u pkgsrc/pkgtools/pkglint/files/files_test.go:1.27 pkgsrc/pkgtools/pkglint/files/files_test.go:1.28
--- pkgsrc/pkgtools/pkglint/files/files_test.go:1.27 Sun Jul 14 21:25:47 2019
+++ pkgsrc/pkgtools/pkglint/files/files_test.go Sat Oct 26 09:51:48 2019
@@ -105,7 +105,7 @@ func (s *Suite) Test_convertToLogicalLin
func (s *Suite) Test_nextLogicalLine__commented_multi(c *check.C) {
t := s.Init(c)
- mklines := t.SetUpFileMkLines("filename.mk",
+ mklines := t.NewMkLines("filename.mk",
"#COMMENTED= \\",
"#\tcontinuation 1 \\",
"#\tcontinuation 2")
@@ -113,7 +113,7 @@ func (s *Suite) Test_nextLogicalLine__co
// The leading comments are stripped from the continuation lines as well.
t.CheckEquals(mkline.Value(), "continuation 1 \tcontinuation 2")
- t.CheckEquals(mkline.VarassignComment(), "")
+ t.CheckEquals(mkline.HasComment(), false)
}
func (s *Suite) Test_convertToLogicalLines__missing_newline_at_eof(c *check.C) {
Index: pkgsrc/pkgtools/pkglint/files/mkline.go
diff -u pkgsrc/pkgtools/pkglint/files/mkline.go:1.59 pkgsrc/pkgtools/pkglint/files/mkline.go:1.60
--- pkgsrc/pkgtools/pkglint/files/mkline.go:1.59 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/mkline.go Sat Oct 26 09:51:48 2019
@@ -15,6 +15,8 @@ import (
type MkLine struct {
*Line
+ splitResult mkLineSplitResult
+
// One of the following mkLine* types.
//
// For the larger of these types, a pointer is used instead of a direct
@@ -34,8 +36,6 @@ type mkLineAssign struct {
valueMk []*MkToken // The value, sent through splitIntoMkWords
valueMkRest string // nonempty in case of parse errors
fields []string // The value, space-separated according to shell quoting rules
- spaceAfterValue string
- comment string
}
type mkLineShell struct {
@@ -69,164 +69,28 @@ type mkLineDependency struct {
sources string
}
-type MkLineParser struct{}
-
-// Parse parses the text of a Makefile line to see what kind of line
-// it is: variable assignment, include, comment, etc.
-//
-// See devel/bmake/parse.c:/^Parse_File/
-func (p MkLineParser) Parse(line *Line) *MkLine {
- text := line.Text
-
- // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing.
- if hasPrefix(text, " ") && line.Basename != "bsd.buildlink3.mk" {
- line.Warnf("Makefile lines should not start with space characters.")
- line.Explain(
- "If this line should be a shell command connected to a target, use a tab character for indentation.",
- "Otherwise remove the leading whitespace.")
- }
-
- // Check for shell commands first because these cannot have comments
- // at the end of the line.
- if hasPrefix(text, "\t") {
- lex := textproc.NewLexer(text)
- for lex.SkipByte('\t') {
- }
-
- // Just for the side effects of the warnings.
- _ = p.split(line, lex.Rest())
-
- return p.parseShellcmd(line)
- }
-
- data := p.split(line, text)
-
- if mkline := p.parseVarassign(line); mkline != nil {
- return mkline
- }
- if mkline := p.parseCommentOrEmpty(line); mkline != nil {
- return mkline
- }
- if mkline := p.parseDirective(line, data); mkline != nil {
- return mkline
- }
- if mkline := p.parseInclude(line); mkline != nil {
- return mkline
- }
- if mkline := p.parseSysinclude(line); mkline != nil {
- return mkline
- }
- if mkline := p.parseDependency(line); mkline != nil {
- return mkline
- }
- if mkline := p.parseMergeConflict(line); mkline != nil {
- return mkline
- }
-
- // The %q is deliberate here since it shows possible strange characters.
- line.Errorf("Unknown Makefile line format: %q.", text)
- return &MkLine{line, nil}
-}
-
-func (p MkLineParser) parseVarassign(line *Line) *MkLine {
- m, a := p.MatchVarassign(line, line.Text)
- if !m {
- return nil
- }
-
- if a.spaceAfterVarname != "" {
- varname := a.varname
- op := a.op
- switch {
- case hasSuffix(varname, "+") && (op == opAssign || op == opAssignAppend):
- break
- case matches(varname, `^[a-z]`) && op == opAssignEval:
- break
- default:
- fix := line.Autofix()
- fix.Notef("Unnecessary space after variable name %q.", varname)
- fix.Replace(varname+a.spaceAfterVarname+op.String(), varname+op.String())
- fix.Apply()
- }
- }
-
- if a.comment != "" && a.value != "" && a.spaceAfterValue == "" {
- line.Warnf("The # character starts a Makefile comment.")
- line.Explain(
- "In a variable assignment, an unescaped # starts a comment that",
- "continues until the end of the line.",
- "To escape the #, write \\#.",
- "",
- "If this # character intentionally starts a comment,",
- "it should be preceded by a space in order to make it more visible.")
- }
-
- return &MkLine{line, a}
-}
-
-func (p MkLineParser) parseShellcmd(line *Line) *MkLine {
- return &MkLine{line, mkLineShell{line.Text[1:]}}
-}
-
-func (p MkLineParser) parseCommentOrEmpty(line *Line) *MkLine {
- trimmedText := trimHspace(line.Text)
-
- if strings.HasPrefix(trimmedText, "#") {
- return &MkLine{line, mkLineComment{}}
- }
-
- if trimmedText == "" {
- return &MkLine{line, mkLineEmpty{}}
- }
-
- return nil
-}
-
-func (p MkLineParser) parseInclude(line *Line) *MkLine {
- m, indent, directive, includedFile := MatchMkInclude(line.Text)
- if !m {
- return nil
- }
-
- return &MkLine{line, &mkLineInclude{directive == "include", false, indent, includedFile, nil}}
-}
-
-func (p MkLineParser) parseSysinclude(line *Line) *MkLine {
- m, indent, directive, includedFile := match3(line.Text, `^\.([\t ]*)(s?include)[\t ]+<([^>]+)>[\t ]*(?:#.*)?$`)
- if !m {
- return nil
- }
-
- return &MkLine{line, &mkLineInclude{directive == "include", true, indent, includedFile, nil}}
-}
-
-func (p MkLineParser) parseDependency(line *Line) *MkLine {
- // XXX: Replace this regular expression with proper parsing.
- // There might be a ${VAR:M*.c} in these variables, which the below regular expression cannot handle.
- m, targets, whitespace, sources := match3(line.Text, `^([^\t :]+(?:[\t ]*[^\t :]+)*)([\t ]*):[\t ]*([^#]*?)(?:[\t ]*#.*)?$`)
- if !m {
- return nil
- }
-
- if whitespace != "" {
- line.Notef("Space before colon in dependency line.")
- }
- return &MkLine{line, mkLineDependency{targets, sources}}
-}
-
-func (p MkLineParser) parseMergeConflict(line *Line) *MkLine {
- if !matches(line.Text, `^(<<<<<<<|=======|>>>>>>>)`) {
- return nil
- }
-
- return &MkLine{line, nil}
-}
-
// String returns the filename and line numbers.
func (mkline *MkLine) String() string {
return sprintf("%s:%s", mkline.Filename, mkline.Linenos())
}
+func (mkline *MkLine) HasComment() bool { return mkline.splitResult.hasComment }
+
+// Comment returns the comment after the first unescaped #.
+//
+// A special case are variable assignments. If these are commented out
+// entirely, they still count as variable assignments, which means that
+// their comment is the one after the value, if any.
+//
+// Shell commands (lines that start with a tab) cannot have comments, as
+// the # characters are passed uninterpreted to the shell.
+//
+// Example:
+// VAR=value # comment
+//
+// In the above line, the comment is " comment", including the leading space.
+func (mkline *MkLine) Comment() string { return mkline.splitResult.comment }
+
// IsVarassign returns true for variable assignments of the form VAR=value.
//
// See IsCommentedVarassign.
@@ -348,16 +212,6 @@ func (mkline *MkLine) ValueAlign() strin
func (mkline *MkLine) Value() string { return mkline.data.(*mkLineAssign).value }
-// VarassignComment applies to variable assignments and returns the comment.
-//
-// Example:
-// VAR=value # comment
-//
-// In the above line, the comment is "# comment".
-//
-// The leading "#" is included so that pkglint can distinguish between no comment at all and an empty comment.
-func (mkline *MkLine) VarassignComment() string { return mkline.data.(*mkLineAssign).comment }
-
// FirstLineContainsValue returns whether the variable assignment of a
// multiline contains a textual value in the first line.
//
@@ -371,7 +225,9 @@ func (mkline *MkLine) FirstLineContainsV
// Parsing the continuation marker as variable value is cheating but works well.
text := strings.TrimSuffix(mkline.raw[0].orignl, "\n")
- _, a := MkLineParser{}.MatchVarassign(mkline.Line, text)
+ parser := NewMkLineParser()
+ splitResult := parser.split(nil, text, true)
+ _, a := parser.MatchVarassign(mkline.Line, text, &splitResult)
return a.value != "\\"
}
@@ -787,180 +643,6 @@ var (
unescapeMkCommentSafeChars = textproc.NewByteSet("\\#[\n").Inverse()
)
-// unescapeComment takes a Makefile line, as written in a file, and splits
-// it into the main part and the comment.
-//
-// The comment starts at the first #. Except if it is preceded by an odd number
-// of backslashes. Or by an opening bracket.
-//
-// The main text is returned including leading and trailing whitespace. Any
-// escaped # is returned in its unescaped form, that is, \# becomes #.
-//
-// The comment is returned including the leading "#", if any. If the line has
-// no comment, it is an empty string.
-func (MkLineParser) unescapeComment(text string) (main, comment string) {
- var sb strings.Builder
-
- lexer := textproc.NewLexer(text)
-
-again:
- if plain := lexer.NextBytesSet(unescapeMkCommentSafeChars); plain != "" {
- sb.WriteString(plain)
- goto again
- }
-
- switch {
- case lexer.SkipString("\\#"):
- sb.WriteByte('#')
-
- case lexer.PeekByte() == '\\' && len(lexer.Rest()) >= 2:
- sb.WriteString(lexer.Rest()[:2])
- lexer.Skip(2)
-
- case lexer.SkipByte('\\'):
- sb.WriteByte('\\')
-
- case lexer.SkipString("[#"):
- // See devel/bmake/files/parse.c:/as in modifier/
- sb.WriteString("[#")
-
- case lexer.SkipByte('['):
- sb.WriteByte('[')
-
- default:
- main = sb.String()
- if lexer.PeekByte() == '#' {
- return main, lexer.Rest()
- }
-
- assert(lexer.EOF())
- return main, ""
- }
-
- goto again
-}
-
-type mkLineSplitResult struct {
- // The text of the line, without the comment at the end of the line,
- // and with # signs unescaped.
- main string
- tokens []*MkToken
- spaceBeforeComment string
- hasComment bool
- comment string
-}
-
-// split parses a logical line from a Makefile (that is, after joining
-// the lines that end in a backslash) into two parts: the main part and the
-// comment.
-//
-// This applies to all line types except those starting with a tab, which
-// contain the shell commands to be associated with make targets. These cannot
-// have comments.
-//
-// If line is given, it is used for logging parse errors and warnings
-// about round parentheses instead of curly braces, as well as ambiguous
-// variables of the form $v instead of ${v}.
-func (MkLineParser) split(line *Line, text string) mkLineSplitResult {
- assert(!hasPrefix(text, "\t"))
-
- mainWithSpaces, comment := MkLineParser{}.unescapeComment(text)
-
- parser := NewMkParser(line, mainWithSpaces)
- lexer := parser.lexer
-
- parseOther := func() string {
- var sb strings.Builder
-
- for !lexer.EOF() {
- if lexer.SkipString("$$") {
- sb.WriteString("$$")
- continue
- }
-
- other := lexer.NextBytesFunc(func(b byte) bool { return b != '$' })
- if other == "" {
- break
- }
-
- sb.WriteString(other)
- }
-
- return sb.String()
- }
-
- var tokens []*MkToken
- for !lexer.EOF() {
- mark := lexer.Mark()
-
- if varUse := parser.VarUse(); varUse != nil {
- tokens = append(tokens, &MkToken{lexer.Since(mark), varUse})
-
- } else if other := parseOther(); other != "" {
- tokens = append(tokens, &MkToken{other, nil})
-
- } else {
- assert(lexer.SkipByte('$'))
- tokens = append(tokens, &MkToken{"$", nil})
- }
- }
-
- hasComment := comment != ""
- if hasComment {
- comment = comment[1:]
- }
-
- mainTrimmed := rtrimHspace(mainWithSpaces)
- spaceBeforeComment := mainWithSpaces[len(mainTrimmed):]
- if spaceBeforeComment != "" {
- tokenText := &tokens[len(tokens)-1].Text
- *tokenText = rtrimHspace(*tokenText)
- if *tokenText == "" {
- if len(tokens) == 1 {
- tokens = nil
- } else {
- tokens = tokens[:len(tokens)-1]
- }
- }
- }
-
- return mkLineSplitResult{mainTrimmed, tokens, spaceBeforeComment, hasComment, comment}
-}
-
-func (p MkLineParser) parseDirective(line *Line, data mkLineSplitResult) *MkLine {
- text := line.Text
- if !hasPrefix(text, ".") {
- return nil
- }
-
- lexer := textproc.NewLexer(data.main[1:])
-
- indent := lexer.NextHspace()
- directive := lexer.NextBytesSet(LowerDash)
- switch directive {
- case "if", "else", "elif", "endif",
- "ifdef", "ifndef",
- "for", "endfor", "undef",
- "error", "warning", "info",
- "export", "export-env", "unexport", "unexport-env":
- break
- default:
- // Intentionally not supported are: ifmake ifnmake elifdef elifndef elifmake elifnmake.
- return nil
- }
-
- lexer.SkipHspace()
-
- args := lexer.Rest()
-
- // In .if and .endif lines the space surrounding the comment is irrelevant.
- // Especially for checking that the .endif comment matches the .if condition,
- // it must be trimmed.
- trimmedComment := trimHspace(data.comment)
-
- return &MkLine{line, &mkLineDirective{indent, directive, args, trimmedComment, nil, nil, nil}}
-}
-
// VariableNeedsQuoting determines whether the given variable needs the :Q
// modifier in the given context.
//
@@ -975,10 +657,10 @@ func (mkline *MkLine) VariableNeedsQuoti
// TODO: Systematically test this function, each and every case, from top to bottom.
// TODO: Re-check the order of all these if clauses whether it really makes sense.
- if varuse.HasModifier("D") && varuse.HasModifier("U") {
- // Take the simple way for now. Handling this kind of
- // conditional expressions correctly and completely would
- // require a larger rewrite.
+ if varuse.HasModifier("D") {
+ // The :D modifier discards the value of the original variable and
+ // replaces it with the expression from the :D modifier.
+ // Therefore the original variable does not need to be quoted.
return unknown
}
@@ -1587,119 +1269,6 @@ var (
VarparamBytes = textproc.NewByteSet("A-Za-z_0-9#*+---./[")
)
-func (p MkLineParser) MatchVarassign(line *Line, text string) (bool, *mkLineAssign) {
-
- // A commented variable assignment does not have leading whitespace.
- // Otherwise line 1 of almost every Makefile fragment would need to
- // be scanned for a variable assignment even though it only contains
- // the $NetBSD CVS Id.
- clex := textproc.NewLexer(text)
- commented := clex.SkipByte('#')
- if commented && clex.SkipHspace() || clex.EOF() {
- return false, nil
- }
-
- withoutLeadingComment := text
- if commented {
- withoutLeadingComment = withoutLeadingComment[1:]
- }
-
- data := p.split(nil, withoutLeadingComment)
-
- lexer := NewMkTokensLexer(data.tokens)
- mainStart := lexer.Mark()
-
- for !commented && lexer.SkipByte(' ') {
- }
-
- varnameStart := lexer.Mark()
- // TODO: duplicated code in MkParser.Varname
- for lexer.NextBytesSet(VarbaseBytes) != "" || lexer.NextVarUse() != nil {
- }
- if lexer.SkipByte('.') || hasPrefix(data.main, "SITES_") {
- for lexer.NextBytesSet(VarparamBytes) != "" || lexer.NextVarUse() != nil {
- }
- }
-
- varname := lexer.Since(varnameStart)
-
- if varname == "" {
- return false, nil
- }
-
- spaceAfterVarname := lexer.NextHspace()
-
- opStart := lexer.Mark()
- switch lexer.PeekByte() {
- case '!', '+', ':', '?':
- lexer.Skip(1)
- }
- if !lexer.SkipByte('=') {
- return false, nil
- }
- op := NewMkOperator(lexer.Since(opStart))
-
- if hasSuffix(varname, "+") && op == opAssign && spaceAfterVarname == "" {
- varname = varname[:len(varname)-1]
- op = opAssignAppend
- }
-
- lexer.SkipHspace()
-
- value := trimHspace(lexer.Rest())
- parsedValueAlign := condStr(commented, "#", "") + lexer.Since(mainStart)
- valueAlign := p.getRawValueAlign(line.raw[0].orignl, parsedValueAlign)
- spaceBeforeComment := data.spaceBeforeComment
- if value == "" {
- valueAlign += spaceBeforeComment
- spaceBeforeComment = ""
- }
-
- return true, &mkLineAssign{
- commented: commented,
- varname: varname,
- varcanon: varnameCanon(varname),
- varparam: varnameParam(varname),
- spaceAfterVarname: spaceAfterVarname,
- op: op,
- valueAlign: valueAlign,
- value: value,
- valueMk: nil, // filled in lazily
- valueMkRest: "", // filled in lazily
- fields: nil, // filled in lazily
- spaceAfterValue: spaceBeforeComment,
- comment: condStr(data.hasComment, "#", "") + data.comment,
- }
-}
-
-func (MkLineParser) getRawValueAlign(raw, parsed string) string {
- r := textproc.NewLexer(raw)
- p := textproc.NewLexer(parsed)
- mark := r.Mark()
-
- for !p.EOF() {
- pch := p.PeekByte()
- rch := r.PeekByte()
-
- switch {
- case pch == rch:
- p.Skip(1)
- r.Skip(1)
-
- case pch == ' ', pch == '\t':
- p.SkipHspace()
- r.SkipHspace()
-
- default:
- assert(pch == '#')
- assert(r.SkipString("\\#"))
- p.Skip(1)
- }
- }
-
- return r.Since(mark)
-}
-
func MatchMkInclude(text string) (m bool, indentation, directive, filename string) {
lexer := textproc.NewLexer(text)
if lexer.SkipByte('.') {
@@ -1717,7 +1286,7 @@ func MatchMkInclude(text string) (m bool
filename = lexer.NextBytesFunc(func(c byte) bool { return c != '"' })
if filename != "" && lexer.SkipByte('"') {
lexer.NextHspace()
- if lexer.EOF() || lexer.SkipByte('#') {
+ if lexer.EOF() {
m = true
return
}
Index: pkgsrc/pkgtools/pkglint/files/mkline_test.go
diff -u pkgsrc/pkgtools/pkglint/files/mkline_test.go:1.68 pkgsrc/pkgtools/pkglint/files/mkline_test.go:1.69
--- pkgsrc/pkgtools/pkglint/files/mkline_test.go:1.68 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/mkline_test.go Sat Oct 26 09:51:48 2019
@@ -2,273 +2,8 @@ package pkglint
import (
"gopkg.in/check.v1"
- "strings"
)
-func (s *Suite) Test_MkLineParser_Parse__varassign(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101,
- "VARNAME.param?=value # varassign comment")
-
- t.CheckEquals(mkline.IsVarassign(), true)
- t.CheckEquals(mkline.Varname(), "VARNAME.param")
- t.CheckEquals(mkline.Varcanon(), "VARNAME.*")
- t.CheckEquals(mkline.Varparam(), "param")
- t.CheckEquals(mkline.Op(), opAssignDefault)
- t.CheckEquals(mkline.Value(), "value")
- t.CheckEquals(mkline.VarassignComment(), "# varassign comment")
-}
-
-func (s *Suite) Test_MkLineParser_Parse__varassign_empty_multiline(c *check.C) {
- t := s.Init(c)
-
- mklines := t.NewMkLines("test.mk",
- "VAR=\t\\",
- "\t\\",
- "\t\\",
- "\t# nothing",
- "",
- "VAR=\t1\\",
- "\t\\",
- "\t\\",
- "\t# a single letter")
-
- // Bmake and pkglint agree that the variable value is an empty string.
- // They don't agree on the exact whitespace in the line, though,
- // but this doesn't matter in practice. To see the difference, run:
- // bmake -dA 2>&1 | grep 'ParseReadLine.*VAR'
- // See devel/bmake/files/parse.c:/non-comment, non-blank line/
- t.CheckEquals(mklines.mklines[0].Text, "VAR= # nothing")
- t.CheckEquals(mklines.mklines[2].Text, "VAR=\t1 # a single letter")
-
- mkline := mklines.mklines[0]
- t.CheckEquals(mkline.IsVarassign(), true)
- t.CheckEquals(mkline.Varname(), "VAR")
- t.CheckEquals(mkline.Op(), opAssign)
- t.CheckEquals(mkline.Value(), "")
- t.CheckEquals(mkline.VarassignComment(), "# nothing")
-}
-
-func (s *Suite) Test_MkLineParser_Parse__varassign_space_around_operator(c *check.C) {
- t := s.Init(c)
-
- t.SetUpCommandLine("--show-autofix", "--source")
- t.NewMkLine("test.mk", 101,
- "pkgbase = package")
-
- t.CheckOutputLines(
- "NOTE: test.mk:101: Unnecessary space after variable name \"pkgbase\".",
- "AUTOFIX: test.mk:101: Replacing \"pkgbase =\" with \"pkgbase=\".",
- "-\tpkgbase = package",
- "+\tpkgbase= package")
-}
-
-func (s *Suite) Test_MkLineParser_Parse__shellcmd(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101,
- "\tshell command # shell comment")
-
- t.CheckEquals(mkline.IsShellCommand(), true)
- t.CheckEquals(mkline.ShellCommand(), "shell command # shell comment")
-}
-
-func (s *Suite) Test_MkLineParser_Parse__comment(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101,
- "# whole line comment")
-
- t.CheckEquals(mkline.IsComment(), true)
-}
-
-func (s *Suite) Test_MkLineParser_Parse__empty(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101, "")
-
- t.CheckEquals(mkline.IsEmpty(), true)
-}
-
-func (s *Suite) Test_MkLineParser_Parse__directive(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101,
- ". if !empty(PKGNAME:M*-*) && ${RUBY_RAILS_SUPPORTED:[\\#]} == 1 # directive comment")
-
- t.CheckEquals(mkline.IsDirective(), true)
- t.CheckEquals(mkline.Indent(), " ")
- t.CheckEquals(mkline.Directive(), "if")
- t.CheckEquals(mkline.Args(), "!empty(PKGNAME:M*-*) && ${RUBY_RAILS_SUPPORTED:[#]} == 1")
- t.CheckEquals(mkline.DirectiveComment(), "directive comment")
-}
-
-func (s *Suite) Test_MkLineParser_Parse__include(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101,
- ". include \"../../mk/bsd.prefs.mk\" # include comment")
-
- t.CheckEquals(mkline.IsInclude(), true)
- t.CheckEquals(mkline.Indent(), " ")
- t.CheckEquals(mkline.MustExist(), true)
- t.CheckEquals(mkline.IncludedFile(), "../../mk/bsd.prefs.mk")
-
- t.CheckEquals(mkline.IsSysinclude(), false)
-}
-
-func (s *Suite) Test_MkLineParser_Parse__sysinclude(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101,
- ". include <subdir.mk> # sysinclude comment")
-
- t.CheckEquals(mkline.IsSysinclude(), true)
- t.CheckEquals(mkline.Indent(), " ")
- t.CheckEquals(mkline.MustExist(), true)
- t.CheckEquals(mkline.IncludedFile(), "subdir.mk")
-
- t.CheckEquals(mkline.IsInclude(), false)
-}
-
-func (s *Suite) Test_MkLineParser_Parse__dependency(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101,
- "target1 target2: source1 source2")
-
- t.CheckEquals(mkline.IsDependency(), true)
- t.CheckEquals(mkline.Targets(), "target1 target2")
- t.CheckEquals(mkline.Sources(), "source1 source2")
-}
-
-func (s *Suite) Test_MkLineParser_Parse__dependency_space(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101,
- "target : source")
-
- t.CheckEquals(mkline.Targets(), "target")
- t.CheckEquals(mkline.Sources(), "source")
- t.CheckOutputLines(
- "NOTE: test.mk:101: Space before colon in dependency line.")
-}
-
-func (s *Suite) Test_MkLineParser_Parse__varassign_append(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101,
- "VARNAME+=value")
-
- t.CheckEquals(mkline.IsVarassign(), true)
- t.CheckEquals(mkline.Varname(), "VARNAME")
- t.CheckEquals(mkline.Varcanon(), "VARNAME")
- t.CheckEquals(mkline.Varparam(), "")
-}
-
-func (s *Suite) Test_MkLineParser_Parse__merge_conflict(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("test.mk", 101,
- "<<<<<<<<<<<<<<<<<")
-
- // Merge conflicts are of neither type.
- t.CheckEquals(mkline.IsVarassign(), false)
- t.CheckEquals(mkline.IsDirective(), false)
- t.CheckEquals(mkline.IsInclude(), false)
- t.CheckEquals(mkline.IsEmpty(), false)
- t.CheckEquals(mkline.IsComment(), false)
- t.CheckEquals(mkline.IsDependency(), false)
- t.CheckEquals(mkline.IsShellCommand(), false)
- t.CheckEquals(mkline.IsSysinclude(), false)
-}
-
-func (s *Suite) Test_MkLineParser_Parse__autofix_space_after_varname(c *check.C) {
- t := s.Init(c)
-
- t.SetUpCommandLine("-Wspace")
- filename := t.CreateFileLines("Makefile",
- MkCvsID,
- "VARNAME +=\t${VARNAME}",
- "VARNAME+ =\t${VARNAME+}",
- "VARNAME+ +=\t${VARNAME+}",
- "VARNAME+ ?=\t${VARNAME}",
- "pkgbase := pkglint")
-
- CheckFileMk(filename)
-
- t.CheckOutputLines(
- "NOTE: ~/Makefile:2: Unnecessary space after variable name \"VARNAME\".",
-
- // The assignment operators other than = and += cannot lead to ambiguities.
- "NOTE: ~/Makefile:5: Unnecessary space after variable name \"VARNAME+\".")
-
- t.SetUpCommandLine("-Wspace", "--autofix")
-
- CheckFileMk(filename)
-
- t.CheckOutputLines(
- "AUTOFIX: ~/Makefile:2: Replacing \"VARNAME +=\" with \"VARNAME+=\".",
- "AUTOFIX: ~/Makefile:5: Replacing \"VARNAME+ ?=\" with \"VARNAME+?=\".")
- t.CheckFileLines("Makefile",
- MkCvsID+"",
- "VARNAME+=\t${VARNAME}",
- "VARNAME+ =\t${VARNAME+}",
- "VARNAME+ +=\t${VARNAME+}",
- "VARNAME+?=\t${VARNAME}",
- "pkgbase := pkglint")
-}
-
-func (s *Suite) Test_MkLineParser_Parse__varname_with_hash(c *check.C) {
- t := s.Init(c)
-
- mkline := t.NewMkLine("Makefile", 123, "VARNAME.#=\tvalue")
-
- // Parse error because the # starts a comment.
- t.CheckEquals(mkline.IsVarassign(), false)
-
- mkline2 := t.NewMkLine("Makefile", 124, "VARNAME.\\#=\tvalue")
-
- t.CheckEquals(mkline2.IsVarassign(), true)
- t.CheckEquals(mkline2.Varname(), "VARNAME.#")
-
- t.CheckOutputLines(
- "ERROR: Makefile:123: Unknown Makefile line format: \"VARNAME.#=\\tvalue\".")
-}
-
-// Ensures that pkglint parses escaped # characters in the same way as bmake.
-//
-// To check that bmake parses them the same, set a breakpoint after the t.NewMkLines
-// and look in t.tmpdir for the location of the file. Then run bmake with that file.
-func (s *Suite) Test_MkLineParser_Parse__escaped_hash_in_value(c *check.C) {
- t := s.Init(c)
-
- mklines := t.SetUpFileMkLines("Makefile",
- "VAR0=\tvalue#",
- "VAR1=\tvalue\\#",
- "VAR2=\tvalue\\\\#",
- "VAR3=\tvalue\\\\\\#",
- "VAR4=\tvalue\\\\\\\\#",
- "",
- "all:",
- ".for var in VAR0 VAR1 VAR2 VAR3 VAR4",
- "\t@printf '%s\\n' ${${var}}''",
- ".endfor")
- parsed := mklines.mklines
-
- t.CheckEquals(parsed[0].Value(), "value")
- t.CheckEquals(parsed[1].Value(), "value#")
- t.CheckEquals(parsed[2].Value(), "value\\\\")
- t.CheckEquals(parsed[3].Value(), "value\\\\#")
- t.CheckEquals(parsed[4].Value(), "value\\\\\\\\")
-
- t.CheckOutputLines(
- "WARN: ~/Makefile:1: The # character starts a Makefile comment.",
- "WARN: ~/Makefile:3: The # character starts a Makefile comment.",
- "WARN: ~/Makefile:5: The # character starts a Makefile comment.")
-}
-
func (s *Suite) Test_MkLine_Varparam(c *check.C) {
t := s.Init(c)
@@ -293,26 +28,45 @@ func (s *Suite) Test_MkLine_ValueAlign__
func (s *Suite) Test_MkLine_FirstLineContainsValue(c *check.C) {
t := s.Init(c)
- mklines := t.NewMkLines("filename.mk",
- MkCvsID,
- "VAR=\tvalue",
- "VAR= value \\",
- "\tstarts in first line",
- "VAR= \\",
- "\tvalue starts in second line",
- "#VAR= value \\",
- "\tstarts in first line",
- "#VAR= \\",
- "\tvalue starts in second line")
+ lines := func(texts ...string) []string { return texts }
+ test := func(lines []string, expected bool) {
+ mklines := t.NewMkLines("filename.mk", lines...)
+ actual := mklines.mklines[0].FirstLineContainsValue()
+ t.CheckEquals(actual, expected)
+ }
+ testAssert := func(lines ...string) {
+ mklines := t.NewMkLines("filename.mk", lines...)
+ t.ExpectAssert(func() { mklines.mklines[0].FirstLineContainsValue() })
+ }
+
+ // Not a variable assignment.
+ testAssert(MkCvsID)
- t.ExpectAssert(func() { mklines.mklines[0].FirstLineContainsValue() })
+ // Not a multiline variable assignment.
+ testAssert("VAR=\tvalue")
- t.ExpectAssert(func() { mklines.mklines[1].FirstLineContainsValue() })
+ test(
+ lines(
+ "VAR= value \\",
+ "\tstarts in first line"),
+ true)
- t.CheckEquals(mklines.mklines[2].FirstLineContainsValue(), true)
- t.CheckEquals(mklines.mklines[3].FirstLineContainsValue(), false)
- t.CheckEquals(mklines.mklines[4].FirstLineContainsValue(), true)
- t.CheckEquals(mklines.mklines[5].FirstLineContainsValue(), false)
+ test(
+ lines(
+ "VAR= \\",
+ "\tvalue starts in second line"),
+ false)
+
+ test(
+ lines(
+ "#VAR= value \\",
+ "\tstarts in first line"),
+ true)
+
+ test(lines(
+ "#VAR= \\",
+ "\tvalue starts in second line"),
+ false)
}
// Up to July 2019, there was a method MkLine.IsMultiAligned, which has
@@ -510,86 +264,6 @@ func (s *Suite) Test_VarUseContext_Strin
t.CheckEquals(vuc.String(), "(Pkgname (package-settable) time:unknown quoting:backt wordpart:false)")
}
-// In variable assignments, a plain '#' introduces a line comment, unless
-// it is escaped by a backslash. In shell commands, on the other hand, it
-// is interpreted literally.
-func (s *Suite) Test_MkLineParser_Parse__number_sign(c *check.C) {
- t := s.Init(c)
-
- mklineVarassignEscaped := t.NewMkLine("filename.mk", 1, "SED_CMD=\t's,\\#,hash,g'")
-
- t.CheckEquals(mklineVarassignEscaped.Varname(), "SED_CMD")
- t.CheckEquals(mklineVarassignEscaped.Value(), "'s,#,hash,g'")
-
- mklineCommandEscaped := t.NewMkLine("filename.mk", 1, "\tsed -e 's,\\#,hash,g'")
-
- t.CheckEquals(mklineCommandEscaped.ShellCommand(), "sed -e 's,\\#,hash,g'")
-
- // From shells/zsh/Makefile.common, rev. 1.78
- mklineCommandUnescaped := t.NewMkLine("filename.mk", 1, "\t# $ sha1 patches/patch-ac")
-
- t.CheckEquals(mklineCommandUnescaped.ShellCommand(), "# $ sha1 patches/patch-ac")
- t.CheckOutputEmpty() // No warning about parsing the lonely dollar sign.
-
- mklineVarassignUnescaped := t.NewMkLine("filename.mk", 1, "SED_CMD=\t's,#,hash,'")
-
- t.CheckEquals(mklineVarassignUnescaped.Value(), "'s,")
- t.CheckOutputLines(
- "WARN: filename.mk:1: The # character starts a Makefile comment.")
-}
-
-func (s *Suite) Test_MkLineParser_Parse__varassign_leading_space(c *check.C) {
- t := s.Init(c)
-
- _ = t.NewMkLine("rubyversion.mk", 427, " _RUBYVER=\t2.15")
- _ = t.NewMkLine("bsd.buildlink3.mk", 132, " ok:=yes")
-
- // In mk/buildlink3/bsd.buildlink3.mk, the leading space is really helpful,
- // therefore no warnings for that file.
- t.CheckOutputLines(
- "WARN: rubyversion.mk:427: Makefile lines should not start with space characters.")
-}
-
-// Exotic code examples from the pkgsrc infrastructure.
-// Hopefully, pkgsrc packages don't need such complicated code.
-// Still, pkglint needs to parse them correctly, or it would not
-// be able to parse and check the infrastructure files as well.
-//
-// See Pkgsrc.loadUntypedVars.
-func (s *Suite) Test_MkLineParser_Parse__infrastructure(c *check.C) {
- t := s.Init(c)
-
- mklines := t.NewMkLines("infra.mk",
- MkCvsID,
- " USE_BUILTIN.${_pkg_:S/^-//}:=no",
- ".error \"Something went wrong\"",
- ".export WRKDIR",
- ".export",
- ".unexport-env WRKDIR",
- "",
- ".ifmake target1", // Luckily, this is not used in the wild.
- ".elifnmake target2", // Neither is this.
- ".endif")
-
- t.CheckEquals(mklines.mklines[1].Varcanon(), "USE_BUILTIN.*")
- t.CheckEquals(mklines.mklines[2].Directive(), "error")
- t.CheckEquals(mklines.mklines[3].Directive(), "export")
-
- t.CheckOutputLines(
- "WARN: infra.mk:2: Makefile lines should not start with space characters.",
- "ERROR: infra.mk:8: Unknown Makefile line format: \".ifmake target1\".",
- "ERROR: infra.mk:9: Unknown Makefile line format: \".elifnmake target2\".")
-
- mklines.Check()
-
- t.CheckOutputLines(
- "WARN: infra.mk:2: USE_BUILTIN.${_pkg_:S/^-//} is defined but not used.",
- "WARN: infra.mk:2: _pkg_ is used but not defined.",
- "ERROR: infra.mk:5: \".export\" requires arguments.",
- "NOTE: infra.mk:2: This variable value should be aligned to column 41.",
- "ERROR: infra.mk:10: Unmatched .endif.")
-}
-
func (s *Suite) Test_MkLine_VariableNeedsQuoting__unknown_rhs(c *check.C) {
t := s.Init(c)
@@ -1068,9 +742,7 @@ func (s *Suite) Test_MkLine_VariableNeed
// If any, the variable PKGSRCDIR should be quoted, but that is a safe
// variable since it is a pkgsrc-specific directory and it appears as
// part of a word, therefore it cannot result in an empty string.
- // FIXME: Don't warn in this situation.
- t.CheckOutputLines(
- "WARN: ~/Makefile:6: The variable BATCH should be quoted as part of a shell word.")
+ t.CheckOutputEmpty()
}
// As of October 2018, these examples from real pkgsrc end up in the
@@ -1458,12 +1130,11 @@ func (s *Suite) Test_MkLine_ValueFields_
func (s *Suite) Test_MkLine_ValueTokens(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
+ text := b.TextToken
+ varUseText := b.VaruseTextToken
+ tokens := b.Tokens
- text := func(text string) *MkToken { return &MkToken{text, nil} }
- varUseText := func(text string, varname string, modifiers ...string) *MkToken {
- return &MkToken{text, NewMkVarUse(varname, modifiers...)}
- }
- tokens := func(tokens ...*MkToken) []*MkToken { return tokens }
test := func(value string, expected []*MkToken, diagnostics ...string) {
mkline := t.NewMkLine("Makefile", 1, "PATH=\t"+value)
actualTokens, _ := mkline.ValueTokens()
@@ -1516,16 +1187,15 @@ func (s *Suite) Test_MkLine_ValueTokens_
func (s *Suite) Test_MkLine_ValueTokens__caching(c *check.C) {
t := s.Init(c)
-
- tokens := func(tokens ...*MkToken) []*MkToken { return tokens }
+ b := NewMkTokenBuilder()
mkline := t.NewMkLine("Makefile", 1, "PATH=\tvalue ${UNFINISHED")
valueTokens, rest := mkline.ValueTokens()
t.CheckDeepEquals(valueTokens,
- tokens(
- &MkToken{"value ", nil},
- &MkToken{"${UNFINISHED", NewMkVarUse("UNFINISHED")}))
+ b.Tokens(
+ b.TextToken("value "),
+ b.VaruseTextToken("${UNFINISHED", "UNFINISHED")))
t.CheckEquals(rest, "")
t.CheckOutputLines(
"WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".")
@@ -1539,16 +1209,12 @@ func (s *Suite) Test_MkLine_ValueTokens_
func (s *Suite) Test_MkLine_ValueTokens__caching_parse_error(c *check.C) {
t := s.Init(c)
-
- tokens := func(tokens ...*MkToken) []*MkToken { return tokens }
- varuseText := func(text, varname string, modifiers ...string) *MkToken {
- return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)}
- }
+ b := NewMkTokenBuilder()
mkline := t.NewMkLine("Makefile", 1, "PATH=\t${UNFINISHED")
valueTokens, rest := mkline.ValueTokens()
- t.CheckDeepEquals(valueTokens, tokens(varuseText("${UNFINISHED", "UNFINISHED")))
+ t.CheckDeepEquals(valueTokens, b.Tokens(b.VaruseTextToken("${UNFINISHED", "UNFINISHED")))
t.CheckEquals(rest, "")
t.CheckOutputLines(
"WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".")
@@ -1650,209 +1316,6 @@ func (s *Suite) Test_MkLine_ResolveVarsI
"WARN: ~/buildlink3.mk:2: PKGPATH.multimedia/totem is used but not defined.")
}
-func (s *Suite) Test_MkLineParser_MatchVarassign(c *check.C) {
- t := s.Init(c)
-
- testLine := func(line *Line, commented bool, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment string, diagnostics ...string) {
- text := line.Text
-
- m, actual := MkLineParser{}.MatchVarassign(line, text)
-
- assert(m)
- expected := mkLineAssign{
- commented: commented,
- varname: varname,
- varcanon: varnameCanon(varname),
- varparam: varnameParam(varname),
- spaceAfterVarname: spaceAfterVarname,
- op: NewMkOperator(op),
- valueAlign: align,
- value: value,
- valueMk: nil,
- valueMkRest: "",
- fields: nil,
- spaceAfterValue: spaceAfterValue,
- comment: comment,
- }
- t.CheckDeepEquals(*actual, expected)
- t.CheckOutput(diagnostics)
- }
-
- test := func(text string, commented bool, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment string, diagnostics ...string) {
- line := t.NewLine("filename.mk", 123, text)
- testLine(line, commented, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment, diagnostics...)
- }
-
- testInvalid := func(text string, diagnostics ...string) {
- line := t.NewLine("filename.mk", 123, text)
- m, _ := MkLineParser{}.MatchVarassign(line, text)
- if m {
- c.Errorf("Text %q matches variable assignment but shouldn't.", text)
- }
- t.CheckOutput(diagnostics)
- }
-
- lines := func(text ...string) *Line {
- mklines := t.NewMkLines("filename.mk",
- text...)
- return mklines.mklines[0].Line
- }
-
- test("C++=c11", false, "C+", "", "+=", "C++=", "c11", "", "")
- test("V=v", false, "V", "", "=", "V=", "v", "", "")
- test("VAR=#comment", false, "VAR", "", "=", "VAR=", "", "", "#comment")
- test("VAR=\\#comment", false, "VAR", "", "=", "VAR=", "#comment", "", "")
- test("VAR=\\\\\\##comment", false, "VAR", "", "=", "VAR=", "\\\\#", "", "#comment")
- test("VAR=\\", false, "VAR", "", "=", "VAR=", "\\", "", "")
- test("VAR += value", false, "VAR", " ", "+=", "VAR += ", "value", "", "")
- test(" VAR=value", false, "VAR", "", "=", " VAR=", "value", "", "")
- test("VAR=value #comment", false, "VAR", "", "=", "VAR=", "value", " ", "#comment")
- test("NFILES=${FILES:[#]}", false, "NFILES", "", "=", "NFILES=", "${FILES:[#]}", "", "")
-
- // To humans, the base variable name seems to be SITES_, being parameterized
- // with distfile-1.0.tar.gz. For pkglint though, the base variable name is
- // SITES_distfile-1.
- test("SITES_distfile-1.0.tar.gz=https://example.org/";,
- false,
- "SITES_distfile-1.0.tar.gz",
- "",
- "=",
- "SITES_distfile-1.0.tar.gz=",
- "https://example.org/";,
- "",
- "")
-
- test("SITES_${distfile}=https://example.org/";,
- false,
- "SITES_${distfile}",
- "",
- "=",
- "SITES_${distfile}=",
- "https://example.org/";,
- "",
- "")
-
- t.ExpectAssert(func() { testInvalid("\tVAR=value") })
- testInvalid("?=value")
- testInvalid("<=value")
- testInvalid("#")
- testInvalid("VAR.$$=value")
-
- // A commented variable assignment must start immediately after the comment character.
- // There must be no additional whitespace before the variable name.
- test("#VAR=value", true, "VAR", "", "=", "#VAR=", "value", "", "")
-
- // A single space is typically used for writing documentation, not for commenting out code.
- // Therefore this line doesn't count as commented variable assignment.
- testInvalid("# VAR=value")
-
- // Ensure that the alignment for the variable value is correct.
- test("BUILD_DIRS=\tdir1 dir2",
- false,
- "BUILD_DIRS",
- "",
- "=",
- "BUILD_DIRS=\t",
- "dir1 dir2",
- "",
- "")
-
- // Ensure that the alignment for the variable value is correct,
- // even if the whole line is commented.
- test("#BUILD_DIRS=\tdir1 dir2",
- true,
- "BUILD_DIRS",
- "",
- "=",
- "#BUILD_DIRS=\t",
- "dir1 dir2",
- "",
- "")
-
- test("MASTER_SITES=\t#none",
- false,
- "MASTER_SITES",
- "",
- "=",
- "MASTER_SITES=\t",
- "",
- "",
- "#none")
-
- test("MASTER_SITES=\t# none",
- false,
- "MASTER_SITES",
- "",
- "=",
- "MASTER_SITES=\t",
- "",
- "",
- "# none")
-
- test("EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d",
-
- false,
- "EGDIRS",
- "",
- "=",
- "EGDIRS=\t",
- "${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d",
- "",
- "")
-
- test("VAR:=\t${VAR:M-*:[\\#]}",
- false,
- "VAR",
- "",
- ":=",
- "VAR:=\t",
- "${VAR:M-*:[#]}",
- "",
- "")
-
- test("#VAR=value",
- true, "VAR", "", "=", "#VAR=", "value", "", "")
-
- testInvalid("# VAR=value")
- testInvalid("#\tVAR=value")
- testInvalid(MkCvsID)
-
- testLine(
- lines(
- "VAR=\t\t\t\\",
- "\tvalue"),
- false,
- "VAR",
- "",
- "=",
- "VAR=\t\t\t",
- "value",
- "",
- "")
-
- testLine(
- lines(
- "#VAR=\t\t\t\\",
- "#\tvalue"),
- true,
- "VAR",
- "",
- "=",
- "#VAR=\t\t\t",
- "value",
- "",
- "")
-}
-
-func (s *Suite) Test_MkLineParser_getRawValueAlign__assertion(c *check.C) {
- t := s.Init(c)
-
- var p MkLineParser
-
- // This is unrealistic; just for code coverage of the assertion.
- t.ExpectAssert(func() { p.getRawValueAlign("a", "b") })
-}
-
func (s *Suite) Test_NewMkOperator(c *check.C) {
t := s.Init(c)
@@ -2147,531 +1610,15 @@ func (s *Suite) Test_MkLine_UnquoteShell
"may lead to unintended file globbing.")
}
-func (s *Suite) Test_MkLineParser_unescapeComment(c *check.C) {
- t := s.Init(c)
-
- test := func(text string, main, comment string) {
- aMain, aComment := MkLineParser{}.unescapeComment(text)
- t.CheckDeepEquals(
- []interface{}{text, aMain, aComment},
- []interface{}{text, main, comment})
- }
-
- test("",
- "",
- "")
- test("text",
- "text",
- "")
-
- // The leading space from the comment is preserved to make parsing as exact
- // as possible.
- //
- // The difference between "#defined" and "# defined" is relevant in a few
- // cases, such as the API documentation of the infrastructure files.
- test("# comment",
- "",
- "# comment")
- test("#\tcomment",
- "",
- "#\tcomment")
- test("# comment",
- "",
- "# comment")
-
- // Other than in the shell, # also starts a comment in the middle of a word.
- test("COMMENT=\tThe C# compiler",
- "COMMENT=\tThe C",
- "# compiler")
- test("COMMENT=\tThe C\\# compiler",
- "COMMENT=\tThe C# compiler",
- "")
-
- test("${TARGET}: ${SOURCES} # comment",
- "${TARGET}: ${SOURCES} ",
- "# comment")
-
- // A # starts a comment, except if it immediately follows a [.
- // This is done so that the length modifier :[#] can be written without
- // escaping the #.
- test("VAR=\t${OTHER:[#]} # comment",
- "VAR=\t${OTHER:[#]} ",
- "# comment")
-
- // The # in the :[#] modifier may be escaped or not. Both forms are equivalent.
- test("VAR:=\t${VAR:M-*:[\\#]}",
- "VAR:=\t${VAR:M-*:[#]}",
- "")
-
- // The character [ prevents the following # from starting a comment, even
- // outside of variable modifiers.
- test("COMMENT=\t[#] $$\\# $$# comment",
- "COMMENT=\t[#] $$# $$",
- "# comment")
-
- // A backslash always escapes the next character, be it a # for a comment
- // or something else. This makes it difficult to write a literal \# in a
- // Makefile, but that's an edge case anyway.
- test("VAR0=\t#comment",
- "VAR0=\t",
- "#comment")
- test("VAR1=\t\\#no-comment",
- "VAR1=\t#no-comment",
- "")
- test("VAR2=\t\\\\#comment",
- "VAR2=\t\\\\",
- "#comment")
-
- // The backslash is only removed when it escapes a comment.
- // In particular, it cannot be used to escape a dollar that starts a
- // variable use.
- test("VAR0=\t$T",
- "VAR0=\t$T",
- "")
- test("VAR1=\t\\$T",
- "VAR1=\t\\$T",
- "")
- test("VAR2=\t\\\\$T",
- "VAR2=\t\\\\$T",
- "")
-
- // To escape a dollar, write it twice.
- test("$$shellvar $${shellvar} \\${MKVAR} [] \\x",
- "$$shellvar $${shellvar} \\${MKVAR} [] \\x",
- "")
-
- // Parse errors are recorded in the rest return value.
- test("${UNCLOSED",
- "${UNCLOSED",
- "")
-
- // In this early phase of parsing, unfinished variable uses are not
- // interpreted and do not influence the detection of the comment start.
- test("text before ${UNCLOSED # comment",
- "text before ${UNCLOSED ",
- "# comment")
-
- // The dollar-space refers to a normal Make variable named " ".
- // The lonely dollar at the very end refers to the variable named "",
- // which is specially protected in bmake to always contain the empty string.
- // It is heavily used in .for loops in the form ${:Uvalue}.
- test("Lonely $ character $",
- "Lonely $ character $",
- "")
-
- // An even number of backslashes does not escape the #.
- // Therefore it starts a comment here.
- test("VAR2=\t\\\\#comment",
- "VAR2=\t\\\\",
- "#comment")
-}
-
-func (s *Suite) Test_MkLineParser_split(c *check.C) {
- t := s.Init(c)
-
- varuse := func(varname string, modifiers ...string) *MkToken {
- var text strings.Builder
- text.WriteString("${")
- text.WriteString(varname)
- for _, modifier := range modifiers {
- text.WriteString(":")
- text.WriteString(modifier)
- }
- text.WriteString("}")
- return &MkToken{Text: text.String(), Varuse: NewMkVarUse(varname, modifiers...)}
- }
- varuseText := func(text, varname string, modifiers ...string) *MkToken {
- return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)}
- }
- text := func(text string) *MkToken {
- return &MkToken{text, nil}
- }
- tokens := func(tokens ...*MkToken) []*MkToken {
- return tokens
- }
-
- test := func(text string, data mkLineSplitResult, diagnostics ...string) {
- line := t.NewLine("filename.mk", 123, text)
- actualData := MkLineParser{}.split(line, text)
-
- t.CheckOutput(diagnostics)
- t.CheckDeepEquals([]interface{}{text, actualData}, []interface{}{text, data})
- }
-
- t.Use(text, varuse, varuseText, tokens)
-
- test(
- "",
- mkLineSplitResult{})
-
- test(
- "text",
- mkLineSplitResult{
- main: "text",
- tokens: tokens(text("text")),
- })
-
- // Leading space is always kept.
- test(
- " text",
- mkLineSplitResult{
- main: " text",
- tokens: tokens(text(" text")),
- })
-
- // Trailing space does not end up in the tokens since it is usually
- // ignored.
- test(
- "text\t",
- mkLineSplitResult{
- main: "text",
- tokens: tokens(text("text")),
- spaceBeforeComment: "\t",
- })
-
- test(
- "text\t# intended comment",
- mkLineSplitResult{
- main: "text",
- tokens: tokens(text("text")),
- spaceBeforeComment: "\t",
- hasComment: true,
- comment: " intended comment",
- })
-
- // Trailing space is saved in a separate field to detect accidental
- // unescaped # in the middle of a word, like the URL fragment in this
- // example.
- test(
- "url#fragment",
- mkLineSplitResult{
- main: "url",
- tokens: tokens(text("url")),
- hasComment: true,
- comment: "fragment",
- })
-
- // The leading space from the comment is preserved to make parsing as exact
- // as possible.
- //
- // The difference between "#defined" and "# defined" is relevant in a few
- // cases, such as the API documentation of the infrastructure files.
- test("# comment",
- mkLineSplitResult{
- hasComment: true,
- comment: " comment",
- })
-
- test("#\tcomment",
- mkLineSplitResult{
- hasComment: true,
- comment: "\tcomment",
- })
-
- test("# comment",
- mkLineSplitResult{
- hasComment: true,
- comment: " comment",
- })
-
- // Other than in the shell, # also starts a comment in the middle of a word.
- test("COMMENT=\tThe C# compiler",
- mkLineSplitResult{
- main: "COMMENT=\tThe C",
- tokens: tokens(text("COMMENT=\tThe C")),
- hasComment: true,
- comment: " compiler",
- })
-
- test("COMMENT=\tThe C\\# compiler",
- mkLineSplitResult{
- main: "COMMENT=\tThe C# compiler",
- tokens: tokens(text("COMMENT=\tThe C# compiler")),
- hasComment: false,
- comment: "",
- })
-
- test("${TARGET}: ${SOURCES} # comment",
- mkLineSplitResult{
- main: "${TARGET}: ${SOURCES}",
- tokens: tokens(varuse("TARGET"), text(": "), varuse("SOURCES")),
- spaceBeforeComment: " ",
- hasComment: true,
- comment: " comment",
- })
-
- // A # starts a comment, except if it immediately follows a [.
- // This is done so that the length modifier :[#] can be written without
- // escaping the #.
- test("VAR=\t${OTHER:[#]} # comment",
- mkLineSplitResult{
- main: "VAR=\t${OTHER:[#]}",
- tokens: tokens(text("VAR=\t"), varuse("OTHER", "[#]")),
- spaceBeforeComment: " ",
- hasComment: true,
- comment: " comment",
- })
-
- // The # in the :[#] modifier may be escaped or not. Both forms are equivalent.
- test("VAR:=\t${VAR:M-*:[\\#]}",
- mkLineSplitResult{
- main: "VAR:=\t${VAR:M-*:[#]}",
- tokens: tokens(text("VAR:=\t"), varuse("VAR", "M-*", "[#]")),
- })
-
- // A backslash always escapes the next character, be it a # for a comment
- // or something else. This makes it difficult to write a literal \# in a
- // Makefile, but that's an edge case anyway.
- test("VAR0=\t#comment",
- mkLineSplitResult{
- main: "VAR0=",
- tokens: tokens(text("VAR0=")),
- // Later, when converting this result into a proper variable assignment,
- // this "space before comment" is reclassified as "space before the value",
- // in order to align the "#comment" with the other variable values.
- spaceBeforeComment: "\t",
- hasComment: true,
- comment: "comment",
- })
-
- test("VAR1=\t\\#no-comment",
- mkLineSplitResult{
- main: "VAR1=\t#no-comment",
- tokens: tokens(text("VAR1=\t#no-comment")),
- })
-
- test("VAR2=\t\\\\#comment",
- mkLineSplitResult{
- main: "VAR2=\t\\\\",
- tokens: tokens(text("VAR2=\t\\\\")),
- hasComment: true,
- comment: "comment",
- })
-
- // The backslash is only removed when it escapes a comment.
- // In particular, it cannot be used to escape a dollar that starts a
- // variable use.
- test("VAR0=\t$T",
- mkLineSplitResult{
- main: "VAR0=\t$T",
- tokens: tokens(text("VAR0=\t"), varuseText("$T", "T")),
- },
- "WARN: filename.mk:123: $T is ambiguous. Use ${T} if you mean a Make variable or $$T if you mean a shell variable.")
-
- test("VAR1=\t\\$T",
- mkLineSplitResult{
- main: "VAR1=\t\\$T",
- tokens: tokens(text("VAR1=\t\\"), varuseText("$T", "T")),
- },
- "WARN: filename.mk:123: $T is ambiguous. Use ${T} if you mean a Make variable or $$T if you mean a shell variable.")
-
- test("VAR2=\t\\\\$T",
- mkLineSplitResult{
- main: "VAR2=\t\\\\$T",
- tokens: tokens(text("VAR2=\t\\\\"), varuseText("$T", "T")),
- },
- "WARN: filename.mk:123: $T is ambiguous. Use ${T} if you mean a Make variable or $$T if you mean a shell variable.")
-
- // To escape a dollar, write it twice.
- test("$$shellvar $${shellvar} \\${MKVAR} [] \\x",
- mkLineSplitResult{
- main: "$$shellvar $${shellvar} \\${MKVAR} [] \\x",
- tokens: tokens(text("$$shellvar $${shellvar} \\"), varuse("MKVAR"), text(" [] \\x")),
- })
-
- // Parse errors are recorded in the rest return value.
- test("${UNCLOSED",
- mkLineSplitResult{
- main: "${UNCLOSED",
- tokens: tokens(varuseText("${UNCLOSED", "UNCLOSED")),
- },
- "WARN: filename.mk:123: Missing closing \"}\" for \"UNCLOSED\".")
-
- // Even if there is a parse error in the main part,
- // the comment is extracted.
- test("text before ${UNCLOSED# comment",
- mkLineSplitResult{
- main: "text before ${UNCLOSED",
- tokens: tokens(
- text("text before "),
- varuseText("${UNCLOSED", "UNCLOSED")),
- hasComment: true,
- comment: " comment",
- },
- "WARN: filename.mk:123: Missing closing \"}\" for \"UNCLOSED\".")
-
- // Even in case of parse errors, the space before the comment is parsed
- // correctly.
- test("text before ${UNCLOSED # comment",
- mkLineSplitResult{
- main: "text before ${UNCLOSED",
- tokens: tokens(
- text("text before "),
- // It's a bit inconsistent that the varname includes the space
- // but the text doesn't; anyway, it's an edge case.
- varuseText("${UNCLOSED", "UNCLOSED ")),
- spaceBeforeComment: " ",
- hasComment: true,
- comment: " comment",
- },
- "WARN: filename.mk:123: Missing closing \"}\" for \"UNCLOSED \".",
- "WARN: filename.mk:123: Invalid part \" \" after variable name \"UNCLOSED\".")
-
- // The dollar-space refers to a normal Make variable named " ".
- // The lonely dollar at the very end refers to the variable named "",
- // which is specially protected in bmake to always contain the empty string.
- // It is heavily used in .for loops in the form ${:Uvalue}.
- //
- // TODO: The rest of pkglint assumes that the empty string is not a valid
- // variable name, mainly because the empty variable name is not visible
- // outside of the bmake debugging mode.
- test("Lonely $ character $",
- mkLineSplitResult{
- main: "Lonely $ character $",
- tokens: tokens(
- text("Lonely "),
- varuseText("$ " /* instead of "${ }" */, " "),
- text("character "),
- text("$")),
- })
-
- // The character [ prevents the following # from starting a comment, even
- // outside of variable modifiers.
- test("COMMENT=\t[#] $$\\# $$# comment",
- mkLineSplitResult{
- main: "COMMENT=\t[#] $$# $$",
- tokens: tokens(text("COMMENT=\t[#] $$# $$")),
- hasComment: true,
- comment: " comment",
- })
-
- test("VAR2=\t\\\\#comment",
- mkLineSplitResult{
- main: "VAR2=\t\\\\",
- tokens: tokens(text("VAR2=\t\\\\")),
- hasComment: true,
- comment: "comment",
- })
-
- // At this stage, MkLine.split doesn't know that empty(...) takes
- // a variable use. Instead it just sees ordinary characters and
- // other uses of variables.
- test(".if empty(${VAR.${tool}}:C/\\:.*$//:M${pattern})",
- mkLineSplitResult{
- main: ".if empty(${VAR.${tool}}:C/\\:.*$//:M${pattern})",
- tokens: tokens(
- text(".if empty("),
- varuse("VAR.${tool}"),
- text(":C/\\:.*"),
- text("$"),
- text("//:M"),
- varuse("pattern"),
- text(")")),
- })
-
- test(" # comment after spaces",
- mkLineSplitResult{
- spaceBeforeComment: " ",
- hasComment: true,
- comment: " comment after spaces",
- })
-
- // FIXME: This theoretical edge case is interpreted differently
- // between bmake and pkglint. Pkglint treats the # as a comment,
- // while bmake interprets it as a regular character.
- test("\\[#",
- mkLineSplitResult{
- main: "\\[",
- tokens: tokens(text("\\[")),
- hasComment: true,
- })
-
- test("\\\\[#",
- mkLineSplitResult{
- main: "\\\\[#",
- tokens: tokens(text("\\\\[#")),
- })
-}
-
-func (s *Suite) Test_MkLineParser_split__unclosed_varuse(c *check.C) {
- t := s.Init(c)
-
- test := func(text string, expected mkLineSplitResult, diagnostics ...string) {
- line := t.NewLine("filename.mk", 123, text)
-
- data := MkLineParser{}.split(line, text)
-
- t.CheckDeepEquals(data, expected)
- t.CheckOutput(diagnostics)
- }
-
- test(
- "EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d",
-
- mkLineSplitResult{
- "EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d",
- []*MkToken{
- {"EGDIRS=\t", nil},
- {"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d",
- NewMkVarUse("EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d")}},
- "",
- false,
- "",
- },
-
- "WARN: filename.mk:123: Missing closing \"}\" for \"EGDIR/pam.d\".",
- "WARN: filename.mk:123: Invalid part \"/pam.d\" after variable name \"EGDIR\".",
- "WARN: filename.mk:123: Missing closing \"}\" for \"EGDIR/dbus-1/system.d ${EGDIR/pam.d\".",
- "WARN: filename.mk:123: Invalid part \"/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".",
- "WARN: filename.mk:123: Missing closing \"}\" for \"EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".",
- "WARN: filename.mk:123: Invalid part \"/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".")
-}
-
-func (s *Suite) Test_MkLineParser_parseDirective(c *check.C) {
- t := s.Init(c)
-
- test := func(input, expectedIndent, expectedDirective, expectedArgs, expectedComment string, diagnostics ...string) {
- line := t.NewLine("filename.mk", 123, input)
- data := MkLineParser{}.split(line, input)
- mkline := MkLineParser{}.parseDirective(line, data)
- if !c.Check(mkline, check.NotNil) {
- return
- }
-
- t.CheckDeepEquals(
- []interface{}{mkline.Indent(), mkline.Directive(), mkline.Args(), mkline.DirectiveComment()},
- []interface{}{expectedIndent, expectedDirective, expectedArgs, expectedComment})
- t.CheckOutput(diagnostics)
- }
-
- test(".if ${VAR} == value",
- "", "if", "${VAR} == value", "")
-
- test(".\tendif # comment",
- "\t", "endif", "", "comment")
-
- test(".if ${VAR} == \"#\"",
- "", "if", "${VAR} == \"", "\"")
-
- test(".if ${VAR:[#]}",
- "", "if", "${VAR:[#]}", "")
-
- test(".if ${VAR} == \\",
- "", "if", "${VAR} == \\", "")
-
- test(".if ${VAR",
- "", "if", "${VAR", "",
- "WARN: filename.mk:123: Missing closing \"}\" for \"VAR\".")
-}
-
func (s *Suite) Test_MatchMkInclude(c *check.C) {
t := s.Init(c)
- test := func(input, expectedIndent, expectedDirective, expectedFilename string) {
- m, indent, directive, args := MatchMkInclude(input)
+ test := func(input, expectedIndent, expectedDirective, expectedFilename, expectedComment string) {
+ splitResult := NewMkLineParser().split(nil, input, true)
+ m, indent, directive, args := MatchMkInclude(splitResult.main)
t.CheckDeepEquals(
- []interface{}{m, indent, directive, args},
- []interface{}{true, expectedIndent, expectedDirective, expectedFilename})
+ []interface{}{m, indent, directive, args, condStr(splitResult.hasComment, "#", "") + splitResult.comment},
+ []interface{}{true, expectedIndent, expectedDirective, expectedFilename, expectedComment})
}
testFail := func(input string) {
@@ -2689,16 +1636,16 @@ func (s *Suite) Test_MatchMkInclude(c *c
testFail(".include \"other.mk\" \"")
test(".include \"other.mk\"",
- "", "include", "other.mk")
+ "", "include", "other.mk", "")
test(".include \"other.mk\"\t",
- "", "include", "other.mk")
+ "", "include", "other.mk", "")
test(".include \"other.mk\"\t#",
- "", "include", "other.mk")
+ "", "include", "other.mk", "#")
test(".include \"other.mk\"\t# comment",
- "", "include", "other.mk")
+ "", "include", "other.mk", "# comment")
t.CheckOutputEmpty()
}
Index: pkgsrc/pkgtools/pkglint/files/mklinechecker.go
diff -u pkgsrc/pkgtools/pkglint/files/mklinechecker.go:1.47 pkgsrc/pkgtools/pkglint/files/mklinechecker.go:1.48
--- pkgsrc/pkgtools/pkglint/files/mklinechecker.go:1.47 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/mklinechecker.go Sat Oct 26 09:51:48 2019
@@ -131,7 +131,8 @@ func (ck MkLineChecker) checkInclude() {
mkline.Warnf("Please write \"USE_TOOLS+= intltool\" instead of this line.")
case hasSuffix(includedFile, "/builtin.mk"):
- if mkline.Basename != "hacks.mk" {
+ // TODO: mkline.HasRationale
+ if mkline.Basename != "hacks.mk" && !mkline.HasComment() {
fix := mkline.Autofix()
fix.Errorf("%s must not be included directly. Include \"%s/buildlink3.mk\" instead.", includedFile, path.Dir(includedFile))
fix.Replace("builtin.mk", "buildlink3.mk")
@@ -272,8 +273,8 @@ func (ck MkLineChecker) checkDirectiveIn
func (ck MkLineChecker) checkDependencyRule(allowedTargets map[string]bool) {
mkline := ck.MkLine
- targets := ck.MkLine.ValueFields(mkline.Targets())
- sources := ck.MkLine.ValueFields(mkline.Sources())
+ targets := mkline.ValueFields(mkline.Targets())
+ sources := mkline.ValueFields(mkline.Sources())
for _, source := range sources {
if source == ".PHONY" {
@@ -282,32 +283,38 @@ func (ck MkLineChecker) checkDependencyR
}
}
}
-
for _, target := range targets {
if target == ".PHONY" {
- for _, dep := range sources {
- allowedTargets[dep] = true
+ for _, source := range sources {
+ allowedTargets[source] = true
}
+ }
+ }
- } else if target == ".ORDER" {
- // TODO: Check for spelling mistakes.
-
- } else if hasPrefix(target, "${.CURDIR}/") {
- // This is deliberate, see the explanation below.
+ for _, target := range targets {
+ ck.checkDependencyTarget(target, allowedTargets)
+ }
+}
- } else if !allowedTargets[target] {
- mkline.Warnf("Undeclared target %q.", target)
- mkline.Explain(
- "To define a custom target in a package, declare it like this:",
- "",
- "\t.PHONY: my-target",
- "",
- "To define a custom target that creates a file (should be rarely needed),",
- "declare it like this:",
- "",
- "\t${.CURDIR}/my-file:")
- }
+func (ck MkLineChecker) checkDependencyTarget(target string, allowedTargets map[string]bool) {
+ if target == ".PHONY" ||
+ target == ".ORDER" ||
+ NewMkParser(nil, target).VarUse() != nil ||
+ allowedTargets[target] {
+ return
}
+
+ mkline := ck.MkLine
+ mkline.Warnf("Undeclared target %q.", target)
+ mkline.Explain(
+ "To define a custom target in a package, declare it like this:",
+ "",
+ "\t.PHONY: my-target",
+ "",
+ "To define a custom target that creates a file (should be rarely needed),",
+ "declare it like this:",
+ "",
+ "\t${.CURDIR}/my-file:")
}
// checkVarassignLeftPermissions checks the permissions for the left-hand side
@@ -450,7 +457,7 @@ func (ck MkLineChecker) checkVarassignLe
return
}
- if mkline.VarassignComment() != "" {
+ if mkline.HasComment() {
return
}
@@ -1048,7 +1055,7 @@ func (ck MkLineChecker) checkVarassignOp
case mkline.Op() != opAssignShell:
return
- case mkline.VarassignComment() != "":
+ case mkline.HasComment():
return
case mkline.Basename == "builtin.mk":
@@ -1094,7 +1101,7 @@ func (ck MkLineChecker) checkVarassignRi
varname := mkline.Varname()
op := mkline.Op()
value := mkline.Value()
- comment := mkline.VarassignComment()
+ comment := condStr(mkline.HasComment(), "#", "") + mkline.Comment()
if trace.Tracing {
defer trace.Call(varname, op, value)()
@@ -1254,7 +1261,7 @@ func (ck MkLineChecker) checkVarassignMi
ck.checkVarassignDecreasingVersions()
}
- if mkline.VarassignComment() == "# defined" && !hasSuffix(varname, "_MK") && !hasSuffix(varname, "_COMMON") {
+ if mkline.Comment() == " defined" && !hasSuffix(varname, "_MK") && !hasSuffix(varname, "_COMMON") {
mkline.Notef("Please use \"# empty\", \"# none\" or \"# yes\" instead of \"# defined\".")
mkline.Explain(
"The value #defined says something about the state of the variable,",
@@ -1378,7 +1385,7 @@ func (ck MkLineChecker) checkVarassignLe
}
switch {
- case mkline.VarassignComment() != "":
+ case mkline.HasComment():
// Assume that the comment contains a rationale for disabling
// this particular check.
@@ -1407,6 +1414,7 @@ func (ck MkLineChecker) checkVarassignLe
return true
}
+// comment is an empty string for no comment, or "#" + the actual comment otherwise.
func (ck MkLineChecker) checkVartype(varname string, op MkOperator, value, comment string) {
if trace.Tracing {
defer trace.Call(varname, op, value, comment)()
Index: pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go
diff -u pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go:1.43 pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go:1.44
--- pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go:1.43 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go Sat Oct 26 09:51:48 2019
@@ -395,6 +395,24 @@ func (s *Suite) Test_MkLineChecker_check
"Relative path \"../../category/package/nonexistent.mk\" does not exist.")
}
+func (s *Suite) Test_MkLineChecker_checkInclude__builtin_mk(c *check.C) {
+ t := s.Init(c)
+
+ t.SetUpPackage("category/package",
+ ".include \"../../category/package/builtin.mk\"",
+ ".include \"../../category/package/builtin.mk\" # ok")
+ t.CreateFileLines("category/package/builtin.mk",
+ MkCvsID)
+ t.FinishSetUp()
+
+ G.checkdirPackage(t.File("category/package"))
+
+ t.CheckOutputLines(
+ "ERROR: ~/category/package/Makefile:20: " +
+ "../../category/package/builtin.mk must not be included directly. " +
+ "Include \"../../category/package/buildlink3.mk\" instead.")
+}
+
func (s *Suite) Test_MkLineChecker__permissions_in_hacks_mk(c *check.C) {
t := s.Init(c)
@@ -606,7 +624,8 @@ func (s *Suite) Test_MkLineChecker_check
".ORDER: target-1 target-2",
"target-1:",
"target-2:",
- "target-3:")
+ "target-3:",
+ "${_COOKIE.test}:")
mklines.Check()
@@ -2107,6 +2126,7 @@ func (s *Suite) Test_MkLineChecker_check
func (s *Suite) Test_MkLineChecker_checkDirectiveCondCompareVarStr__no_tracing(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
t.SetUpVartypes()
mklines := t.NewMkLines("filename.mk",
@@ -2114,7 +2134,7 @@ func (s *Suite) Test_MkLineChecker_check
t.DisableTracing()
ck := MkLineChecker{mklines, mklines.mklines[0]}
- varUse := NewMkVarUse("DISTFILES", "Mpattern", "O", "u")
+ varUse := b.VarUse("DISTFILES", "Mpattern", "O", "u")
ck.checkDirectiveCondCompareVarStr(varUse, "==", "distfile-1.0.tar.gz")
t.CheckOutputEmpty()
@@ -2416,6 +2436,7 @@ func (s *Suite) Test_MkLineChecker_Check
// check that the variable names match exactly.
func (s *Suite) Test_MkLineChecker_CheckVaruse__varcanon(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
t.SetUpPkgsrc()
t.CreateFileLines("mk/sys-vars.mk",
@@ -2429,7 +2450,7 @@ func (s *Suite) Test_MkLineChecker_Check
ck := MkLineChecker{mklines, mklines.mklines[1]}
- ck.CheckVaruse(NewMkVarUse("CPPPATH.SunOS"), &VarUseContext{
+ ck.CheckVaruse(b.VarUse("CPPPATH.SunOS"), &VarUseContext{
vartype: &Vartype{
basicType: BtPathname,
options: Guessed,
Index: pkgsrc/pkgtools/pkglint/files/mklines.go
diff -u pkgsrc/pkgtools/pkglint/files/mklines.go:1.55 pkgsrc/pkgtools/pkglint/files/mklines.go:1.56
--- pkgsrc/pkgtools/pkglint/files/mklines.go:1.55 Sun Sep 8 22:47:47 2019
+++ pkgsrc/pkgtools/pkglint/files/mklines.go Sat Oct 26 09:51:48 2019
@@ -26,7 +26,7 @@ type MkLines struct {
func NewMkLines(lines *Lines) *MkLines {
mklines := make([]*MkLine, lines.Len())
for i, line := range lines.Lines {
- mklines[i] = MkLineParser{}.Parse(line)
+ mklines[i] = NewMkLineParser().Parse(line)
}
tools := NewTools()
@@ -578,5 +578,5 @@ func (mklines *MkLines) SaveAutofixChang
}
func (mklines *MkLines) EOFLine() *MkLine {
- return MkLineParser{}.Parse(mklines.lines.EOFLine())
+ return NewMkLineParser().Parse(mklines.lines.EOFLine())
}
Index: pkgsrc/pkgtools/pkglint/files/mktokenslexer_test.go
diff -u pkgsrc/pkgtools/pkglint/files/mktokenslexer_test.go:1.4 pkgsrc/pkgtools/pkglint/files/mktokenslexer_test.go:1.5
--- pkgsrc/pkgtools/pkglint/files/mktokenslexer_test.go:1.4 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/mktokenslexer_test.go Sat Oct 26 09:51:48 2019
@@ -16,8 +16,9 @@ func (s *Suite) Test_MkTokensLexer__empt
// A slice of a single token behaves like textproc.Lexer.
func (s *Suite) Test_MkTokensLexer__single_plain_text_token(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- lexer := NewMkTokensLexer([]*MkToken{{"\\# $$ [#] $V", nil}})
+ lexer := NewMkTokensLexer(b.Tokens(b.TextToken("\\# $$ [#] $V")))
t.CheckEquals(lexer.SkipByte('\\'), true)
t.CheckEquals(lexer.Rest(), "# $$ [#] $V")
@@ -36,8 +37,9 @@ func (s *Suite) Test_MkTokensLexer__sing
// text.
func (s *Suite) Test_MkTokensLexer__single_varuse_token(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- tokens := []*MkToken{{"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}}
+ tokens := b.Tokens(b.VaruseToken("VAR", "Mpattern"))
lexer := NewMkTokensLexer(tokens)
t.CheckEquals(lexer.EOF(), false)
@@ -47,10 +49,11 @@ func (s *Suite) Test_MkTokensLexer__sing
func (s *Suite) Test_MkTokensLexer__plain_then_varuse(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- tokens := []*MkToken{
- {"plain text", nil},
- {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}}
+ tokens := b.Tokens(
+ b.TextToken("plain text"),
+ b.VaruseToken("VAR", "Mpattern"))
lexer := NewMkTokensLexer(tokens)
t.CheckEquals(lexer.NextBytesSet(textproc.Digit.Inverse()), "plain text")
@@ -60,11 +63,12 @@ func (s *Suite) Test_MkTokensLexer__plai
func (s *Suite) Test_MkTokensLexer__varuse_varuse_varuse(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- tokens := []*MkToken{
- {"${dirs:O:u}", NewMkVarUse("dirs", "O", "u")},
- {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")},
- {"${.TARGET}", NewMkVarUse(".TARGET")}}
+ tokens := b.Tokens(
+ b.VaruseToken("dirs", "O", "u"),
+ b.VaruseToken("VAR", "Mpattern"),
+ b.VaruseToken(".TARGET"))
lexer := NewMkTokensLexer(tokens)
t.CheckDeepEquals(lexer.NextVarUse(), tokens[0])
@@ -75,11 +79,12 @@ func (s *Suite) Test_MkTokensLexer__varu
func (s *Suite) Test_MkTokensLexer__mark_reset_since_in_initial_state(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- tokens := []*MkToken{
- {"${dirs:O:u}", NewMkVarUse("dirs", "O", "u")},
- {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")},
- {"${.TARGET}", NewMkVarUse(".TARGET")}}
+ tokens := b.Tokens(
+ b.VaruseToken("dirs", "O", "u"),
+ b.VaruseToken("VAR", "Mpattern"),
+ b.VaruseToken(".TARGET"))
lexer := NewMkTokensLexer(tokens)
start := lexer.Mark()
@@ -94,11 +99,12 @@ func (s *Suite) Test_MkTokensLexer__mark
func (s *Suite) Test_MkTokensLexer__mark_reset_since_inside_plain_text(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- lexer := NewMkTokensLexer([]*MkToken{
- {"plain text", nil},
- {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")},
- {"rest", nil}})
+ lexer := NewMkTokensLexer(b.Tokens(
+ b.TextToken("plain text"),
+ b.VaruseToken("VAR", "Mpattern"),
+ b.TextToken("rest")))
start := lexer.Mark()
t.CheckEquals(lexer.NextBytesSet(textproc.Alpha), "plain")
@@ -112,11 +118,12 @@ func (s *Suite) Test_MkTokensLexer__mark
func (s *Suite) Test_MkTokensLexer__mark_reset_since_after_plain_text(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- lexer := NewMkTokensLexer([]*MkToken{
- {"plain text", nil},
- {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")},
- {"rest", nil}})
+ lexer := NewMkTokensLexer(b.Tokens(
+ b.TextToken("plain text"),
+ b.VaruseToken("VAR", "Mpattern"),
+ b.TextToken("rest")))
start := lexer.Mark()
t.CheckEquals(lexer.SkipString("plain text"), true)
@@ -130,10 +137,11 @@ func (s *Suite) Test_MkTokensLexer__mark
func (s *Suite) Test_MkTokensLexer__mark_reset_since_after_varuse(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- tokens := []*MkToken{
- {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")},
- {"rest", nil}}
+ tokens := b.Tokens(
+ b.VaruseToken("VAR", "Mpattern"),
+ b.TextToken("rest"))
lexer := NewMkTokensLexer(tokens)
start := lexer.Mark()
@@ -148,11 +156,12 @@ func (s *Suite) Test_MkTokensLexer__mark
func (s *Suite) Test_MkTokensLexer__multiple_marks_in_same_plain_text(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- lexer := NewMkTokensLexer([]*MkToken{
- {"plain text", nil},
- {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")},
- {"rest", nil}})
+ lexer := NewMkTokensLexer(b.Tokens(
+ b.TextToken("plain text"),
+ b.VaruseToken("VAR", "Mpattern"),
+ b.TextToken("rest")))
start := lexer.Mark()
t.CheckEquals(lexer.NextString("plain "), "plain ")
@@ -170,11 +179,12 @@ func (s *Suite) Test_MkTokensLexer__mult
func (s *Suite) Test_MkTokensLexer__multiple_marks_in_varuse(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- tokens := []*MkToken{
- {"${VAR1}", NewMkVarUse("VAR1")},
- {"${VAR2}", NewMkVarUse("VAR2")},
- {"${VAR3}", NewMkVarUse("VAR3")}}
+ tokens := b.Tokens(
+ b.VaruseToken("VAR1"),
+ b.VaruseToken("VAR2"),
+ b.VaruseToken("VAR3"))
lexer := NewMkTokensLexer(tokens)
start := lexer.Mark()
@@ -197,16 +207,18 @@ func (s *Suite) Test_MkTokensLexer__mult
func (s *Suite) Test_MkTokensLexer__EOF_before_plain_text(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- lexer := NewMkTokensLexer([]*MkToken{{"rest", nil}})
+ lexer := NewMkTokensLexer(b.Tokens(b.TextToken("rest")))
t.CheckEquals(lexer.EOF(), false)
}
func (s *Suite) Test_MkTokensLexer__EOF_before_varuse(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- lexer := NewMkTokensLexer([]*MkToken{{"${VAR}", NewMkVarUse("VAR")}})
+ lexer := NewMkTokensLexer(b.Tokens(b.VaruseToken("VAR")))
t.CheckEquals(lexer.EOF(), false)
}
@@ -223,25 +235,27 @@ func (s *Suite) Test_MkTokensLexer__EOF_
// bother to make this unnecessary copy and works on the shared slice.
func (s *Suite) Test_MkTokensLexer__constructor_uses_shared_array(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- tokens := []*MkToken{{"${VAR}", NewMkVarUse("VAR")}}
+ tokens := b.Tokens(b.VaruseToken("VAR"))
lexer := NewMkTokensLexer(tokens)
t.CheckEquals(lexer.Rest(), "${VAR}")
tokens[0].Text = "modified text"
- tokens[0].Varuse = NewMkVarUse("MODIFIED", "Mpattern")
+ tokens[0].Varuse = b.VarUse("MODIFIED", "Mpattern")
t.CheckEquals(lexer.Rest(), "modified text")
}
func (s *Suite) Test_MkTokensLexer__peek_after_varuse(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- tokens := []*MkToken{
- {"${VAR}", NewMkVarUse("VAR")},
- {"${VAR}", NewMkVarUse("VAR")},
- {"text", nil}}
+ tokens := b.Tokens(
+ b.VaruseToken("VAR"),
+ b.VaruseToken("VAR"),
+ b.TextToken("text"))
lexer := NewMkTokensLexer(tokens)
t.CheckDeepEquals(lexer.NextVarUse(), tokens[0])
@@ -253,8 +267,9 @@ func (s *Suite) Test_MkTokensLexer__peek
func (s *Suite) Test_MkTokensLexer__varuse_when_plain_text(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- lexer := NewMkTokensLexer([]*MkToken{{"text", nil}})
+ lexer := NewMkTokensLexer(b.Tokens(b.TextToken("text")))
t.Check(lexer.NextVarUse(), check.IsNil)
t.CheckEquals(lexer.NextString("te"), "te")
@@ -269,10 +284,11 @@ func (s *Suite) Test_MkTokensLexer__varu
// the beginning.
func (s *Suite) Test_MkTokensLexer__adjacent_plain_text(c *check.C) {
t := s.Init(c)
+ b := NewMkTokenBuilder()
- lexer := NewMkTokensLexer([]*MkToken{
- {"text1", nil},
- {"text2", nil}})
+ lexer := NewMkTokensLexer(b.Tokens(
+ b.TextToken("text1"),
+ b.TextToken("text2")))
// Returns false since the string is distributed over two separate tokens.
t.CheckEquals(lexer.SkipString("text1text2"), false)
Index: pkgsrc/pkgtools/pkglint/files/mktypes_test.go
diff -u pkgsrc/pkgtools/pkglint/files/mktypes_test.go:1.14 pkgsrc/pkgtools/pkglint/files/mktypes_test.go:1.15
--- pkgsrc/pkgtools/pkglint/files/mktypes_test.go:1.14 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/mktypes_test.go Sat Oct 26 09:51:48 2019
@@ -2,9 +2,36 @@ package pkglint
import (
"gopkg.in/check.v1"
+ "strings"
)
-func NewMkVarUse(varname string, modifiers ...string) *MkVarUse {
+type MkTokenBuilder struct{}
+
+func NewMkTokenBuilder() MkTokenBuilder { return MkTokenBuilder{} }
+
+func (b MkTokenBuilder) VaruseToken(varname string, modifiers ...string) *MkToken {
+ var text strings.Builder
+ text.WriteString("${")
+ text.WriteString(varname)
+ for _, modifier := range modifiers {
+ text.WriteString(":")
+ text.WriteString(modifier)
+ }
+ text.WriteString("}")
+ return &MkToken{Text: text.String(), Varuse: b.VarUse(varname, modifiers...)}
+}
+
+func (b MkTokenBuilder) VaruseTextToken(text, varname string, modifiers ...string) *MkToken {
+ return &MkToken{Text: text, Varuse: b.VarUse(varname, modifiers...)}
+}
+
+func (MkTokenBuilder) TextToken(text string) *MkToken {
+ return &MkToken{text, nil}
+}
+
+func (MkTokenBuilder) Tokens(tokens ...*MkToken) []*MkToken { return tokens }
+
+func (MkTokenBuilder) VarUse(varname string, modifiers ...string) *MkVarUse {
var mods []MkVarUseModifier
for _, modifier := range modifiers {
mods = append(mods, MkVarUseModifier{modifier})
Index: pkgsrc/pkgtools/pkglint/files/package.go
diff -u pkgsrc/pkgtools/pkglint/files/package.go:1.64 pkgsrc/pkgtools/pkglint/files/package.go:1.65
--- pkgsrc/pkgtools/pkglint/files/package.go:1.64 Fri Oct 11 23:30:02 2019
+++ pkgsrc/pkgtools/pkglint/files/package.go Sat Oct 26 09:51:48 2019
@@ -777,7 +777,7 @@ func (pkg *Package) checkGnuConfigureUse
continue
}
- if matches(mkline.VarassignComment(), `(?-i)\b(?:c|empty|none)\b`) {
+ if matches(mkline.Comment(), `(?-i)\b(?:c|empty|none)\b`) {
// Don't emit a warning since the comment probably contains a
// statement that C is really not needed.
return
@@ -836,7 +836,7 @@ func (pkg *Package) determineEffectivePk
}
if pkgnameLine != nil && (pkgname == distname || pkgname == "${DISTNAME}") {
- if pkgnameLine.VarassignComment() == "" {
+ if !pkgnameLine.HasComment() {
pkgnameLine.Notef("This assignment is probably redundant " +
"since PKGNAME is ${DISTNAME} by default.")
pkgnameLine.Explain(
@@ -1294,6 +1294,16 @@ func (pkg *Package) checkIncludeConditio
"%q is included unconditionally here "+
"and conditionally in %s (depending on %s).",
cleanpath(mkline.IncludedFile()), mkline.RefTo(other), strings.Join(other.ConditionalVars(), ", "))
+
+ if mkline.Basename == "buildlink3.mk" && containsStr(other.ConditionalVars(), "PKG_OPTIONS") {
+ mkline.Explain(
+ "When including a dependent file, the conditions in the",
+ "buildlink3.mk file should be the same as in options.mk",
+ "or the Makefile.",
+ "",
+ "To find out the PKG_OPTIONS of this package at build time,",
+ "have a look at mk/pkg-build-options.mk.")
+ }
}
}
Index: pkgsrc/pkgtools/pkglint/files/vartypecheck.go
diff -u pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.64 pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.65
--- pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.64 Fri Oct 11 23:30:02 2019
+++ pkgsrc/pkgtools/pkglint/files/vartypecheck.go Sat Oct 26 09:51:48 2019
@@ -614,7 +614,7 @@ func (cv *VartypeCheck) FetchURL() {
hasSuffix(fetchURL, "="),
hasSuffix(fetchURL, ":"),
hasPrefix(fetchURL, "-"),
- len(tokens) == 0 || tokens[len(tokens)-1].Varuse != nil:
+ tokens[len(tokens)-1].Varuse != nil:
break
default:
@@ -1401,7 +1401,7 @@ func (cv *VartypeCheck) VariableNamePatt
}
// TODO: sync with MkParser.Varname
- if matches(cv.Value, `^[*A-Z_][*0-9A-Z_]*(?:[.].*)?$`) {
+ if matches(cv.Value, `^[*A-Z_.][*0-9A-Z_]*(?:[.].*)?$`) {
return
}
Index: pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go
diff -u pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go:1.17 pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go:1.18
--- pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go:1.17 Sun Jul 14 21:25:47 2019
+++ pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go Sat Oct 26 09:51:48 2019
@@ -47,7 +47,7 @@ func (s *Suite) Test_ShTokenizer_ShAtom(
text += ":" + strings.Replace(strings.Replace(modifier, "\\", "\\\\", -1), ":", "\\:", -1)
}
text += "}"
- varuse := NewMkVarUse(varname, modifiers...)
+ varuse := NewMkTokenBuilder().VarUse(varname, modifiers...)
return &ShAtom{shtVaruse, text, shqPlain, varuse}
}
shvar := func(text, varname string) *ShAtom { return &ShAtom{shtShVarUse, text, shqPlain, varname} }
Index: pkgsrc/pkgtools/pkglint/files/substcontext.go
diff -u pkgsrc/pkgtools/pkglint/files/substcontext.go:1.28 pkgsrc/pkgtools/pkglint/files/substcontext.go:1.29
--- pkgsrc/pkgtools/pkglint/files/substcontext.go:1.28 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/substcontext.go Sat Oct 26 09:51:48 2019
@@ -267,7 +267,7 @@ func (ctx *SubstContext) suggestSubstVar
fix.Explain(
"Replacing @VAR@ with ${VAR} is such a typical pattern that pkgsrc has built-in support for it,",
"requiring only the variable name instead of the full sed command.")
- if mkline.VarassignComment() == "" && len(tokens) == 2 && tokens[0] == "-e" {
+ if !mkline.HasComment() && len(tokens) == 2 && tokens[0] == "-e" {
fix.Replace(mkline.Text, alignWith(varop, mkline.ValueAlign())+varname)
}
fix.Anyway()
Index: pkgsrc/pkgtools/pkglint/files/toplevel.go
diff -u pkgsrc/pkgtools/pkglint/files/toplevel.go:1.21 pkgsrc/pkgtools/pkglint/files/toplevel.go:1.22
--- pkgsrc/pkgtools/pkglint/files/toplevel.go:1.21 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/toplevel.go Sat Oct 26 09:51:48 2019
@@ -39,8 +39,7 @@ func (ctx *Toplevel) checkSubdir(mkline
subdir := mkline.Value()
if mkline.IsCommentedVarassign() {
- comment := mkline.VarassignComment()
- if comment == "" || comment == "#" {
+ if !mkline.HasComment() || mkline.Comment() == "" {
mkline.Warnf("%q commented out without giving a reason.", subdir)
}
}
Index: pkgsrc/pkgtools/pkglint/files/util.go
diff -u pkgsrc/pkgtools/pkglint/files/util.go:1.53 pkgsrc/pkgtools/pkglint/files/util.go:1.54
--- pkgsrc/pkgtools/pkglint/files/util.go:1.53 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/util.go Sat Oct 26 09:51:48 2019
@@ -73,6 +73,15 @@ func replaceAllFunc(s string, re regex.P
return G.res.Compile(re).ReplaceAllStringFunc(s, repl)
}
+func containsStr(slice []string, s string) bool {
+ for _, str := range slice {
+ if s == str {
+ return true
+ }
+ }
+ return false
+}
+
// intern returns an independent copy of the given string.
//
// It should be called when only a small substring of a large string
Index: pkgsrc/pkgtools/pkglint/files/varalignblock.go
diff -u pkgsrc/pkgtools/pkglint/files/varalignblock.go:1.5 pkgsrc/pkgtools/pkglint/files/varalignblock.go:1.6
--- pkgsrc/pkgtools/pkglint/files/varalignblock.go:1.5 Sun Sep 8 22:47:47 2019
+++ pkgsrc/pkgtools/pkglint/files/varalignblock.go Sat Oct 26 09:51:48 2019
@@ -118,7 +118,7 @@ func (va *VaralignBlock) processVarassig
// .include "../../mk/pkg-build-options.mk"
return
- case mkline.Value() == "" && mkline.VarassignComment() == "":
+ case mkline.Value() == "" && !mkline.HasComment():
// Multiple-inclusion guards usually appear in a block of
// their own and therefore do not need alignment.
//
Index: pkgsrc/pkgtools/pkglint/files/vargroups_test.go
diff -u pkgsrc/pkgtools/pkglint/files/vargroups_test.go:1.3 pkgsrc/pkgtools/pkglint/files/vargroups_test.go:1.4
--- pkgsrc/pkgtools/pkglint/files/vargroups_test.go:1.3 Tue Oct 1 21:37:59 2019
+++ pkgsrc/pkgtools/pkglint/files/vargroups_test.go Sat Oct 26 09:51:48 2019
@@ -222,6 +222,7 @@ func (s *Suite) Test_VargroupsChecker__i
"_VARGROUPS+=\t\tgroup",
"_IGN_VARS.group=\tPREFER_*",
"_IGN_VARS.group+=\t[",
+ "_IGN_VARS.group=\t.CURDIR",
"_UNDERSCORE=\t\t_", // This is not an isVargroups name.
"",
".if ${PREFER_PKGSRC:U} || ${WRKOBJDIR:U}",
@@ -231,11 +232,11 @@ func (s *Suite) Test_VargroupsChecker__i
t.CheckOutputLines(
"WARN: Makefile:5: \"[\" is not a valid variable name pattern.",
- "WARN: Makefile:6: Variable names starting with an underscore (_UNDERSCORE) "+
+ "WARN: Makefile:7: Variable names starting with an underscore (_UNDERSCORE) "+
"are reserved for internal pkgsrc use.",
- "WARN: Makefile:6: _UNDERSCORE is defined but not used.",
- "WARN: Makefile:6: Variable _UNDERSCORE is defined but not mentioned in the _VARGROUPS section.",
- "WARN: Makefile:8: Variable WRKOBJDIR is used but not mentioned in the _VARGROUPS section.")
+ "WARN: Makefile:7: _UNDERSCORE is defined but not used.",
+ "WARN: Makefile:7: Variable _UNDERSCORE is defined but not mentioned in the _VARGROUPS section.",
+ "WARN: Makefile:9: Variable WRKOBJDIR is used but not mentioned in the _VARGROUPS section.")
}
func (s *Suite) Test_VargroupsChecker__private_before_public(c *check.C) {
Index: pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go
diff -u pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go:1.56 pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go:1.57
--- pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go:1.56 Fri Oct 11 23:30:02 2019
+++ pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go Sat Oct 26 09:51:48 2019
@@ -1841,10 +1841,9 @@ func (vt *VartypeCheckTester) Values(val
test := func(mklines *MkLines, mkline *MkLine, value string) {
varname := vt.varname
- comment := ""
+ comment := condStr(mkline.HasComment(), "#", "") + mkline.Comment()
if mkline.IsVarassign() {
- mkline.Tokenize(value, true) // Produce some warnings as side-effects.
- comment = mkline.VarassignComment()
+ _ = mkline.Tokenize(value, true) // Produce some warnings as side-effects.
}
effectiveValue := value
Added files:
Index: pkgsrc/pkgtools/pkglint/files/mklineparser.go
diff -u /dev/null pkgsrc/pkgtools/pkglint/files/mklineparser.go:1.1
--- /dev/null Sat Oct 26 09:51:48 2019
+++ pkgsrc/pkgtools/pkglint/files/mklineparser.go Sat Oct 26 09:51:48 2019
@@ -0,0 +1,450 @@
+package pkglint
+
+import (
+ "netbsd.org/pkglint/textproc"
+ "strings"
+)
+
+type MkLineParser struct{}
+
+func NewMkLineParser() MkLineParser { return MkLineParser{} }
+
+// Parse parses the text of a Makefile line to see what kind of line
+// it is: variable assignment, include, comment, etc.
+//
+// See devel/bmake/parse.c:/^Parse_File/
+func (p MkLineParser) Parse(line *Line) *MkLine {
+ text := line.Text
+
+ // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing.
+ if hasPrefix(text, " ") && line.Basename != "bsd.buildlink3.mk" {
+ line.Warnf("Makefile lines should not start with space characters.")
+ line.Explain(
+ "If this line should be a shell command connected to a target, use a tab character for indentation.",
+ "Otherwise remove the leading whitespace.")
+ }
+
+ // Check for shell commands first because these cannot have comments
+ // at the end of the line.
+ if hasPrefix(text, "\t") {
+ lex := textproc.NewLexer(text)
+ for lex.SkipByte('\t') {
+ }
+
+ splitResult := p.split(line, lex.Rest(), false)
+ return p.parseShellcmd(line, splitResult)
+ }
+
+ splitResult := p.split(line, text, true)
+
+ if mkline := p.parseVarassign(line, text, splitResult); mkline != nil {
+ return mkline
+ }
+ if mkline := p.parseCommentOrEmpty(line, splitResult); mkline != nil {
+ return mkline
+ }
+ if mkline := p.parseDirective(line, splitResult); mkline != nil {
+ return mkline
+ }
+ if mkline := p.parseInclude(line, splitResult); mkline != nil {
+ return mkline
+ }
+ if mkline := p.parseSysinclude(line, splitResult); mkline != nil {
+ return mkline
+ }
+ if mkline := p.parseDependency(line, splitResult); mkline != nil {
+ return mkline
+ }
+ if mkline := p.parseMergeConflict(line, splitResult); mkline != nil {
+ return mkline
+ }
+
+ // The %q is deliberate here since it shows possible strange characters.
+ line.Errorf("Unknown Makefile line format: %q.", text)
+ return &MkLine{line, splitResult, nil}
+}
+
+func (p MkLineParser) parseVarassign(line *Line, text string, splitResult mkLineSplitResult) *MkLine {
+ m, a := p.MatchVarassign(line, text, &splitResult)
+ if !m {
+ return nil
+ }
+
+ if a.spaceAfterVarname != "" {
+ varname := a.varname
+ op := a.op
+ switch {
+ case hasSuffix(varname, "+") && (op == opAssign || op == opAssignAppend):
+ break
+ case matches(varname, `^[a-z]`) && op == opAssignEval:
+ break
+ default:
+ fix := line.Autofix()
+ fix.Notef("Unnecessary space after variable name %q.", varname)
+ fix.Replace(varname+a.spaceAfterVarname+op.String(), varname+op.String())
+ fix.Apply()
+ }
+ }
+
+ if splitResult.hasComment && a.value != "" && splitResult.spaceBeforeComment == "" {
+ line.Warnf("The # character starts a Makefile comment.")
+ line.Explain(
+ "In a variable assignment, an unescaped # starts a comment that",
+ "continues until the end of the line.",
+ "To escape the #, write \\#.",
+ "",
+ "If this # character intentionally starts a comment,",
+ "it should be preceded by a space in order to make it more visible.")
+ }
+
+ return &MkLine{line, splitResult, a}
+}
+
+func (p MkLineParser) MatchVarassign(line *Line, text string, splitResult *mkLineSplitResult) (bool, *mkLineAssign) {
+
+ // A commented variable assignment does not have leading whitespace.
+ // Otherwise line 1 of almost every Makefile fragment would need to
+ // be scanned for a variable assignment even though it only contains
+ // the $NetBSD CVS Id.
+ commented := splitResult.main == "" && splitResult.hasComment
+ if commented {
+ clex := textproc.NewLexer(splitResult.comment)
+ if clex.SkipHspace() || clex.EOF() {
+ return false, nil
+ }
+ *splitResult = p.split(nil, text[1:], true)
+ }
+
+ lexer := NewMkTokensLexer(splitResult.tokens)
+ mainStart := lexer.Mark()
+
+ for !commented && lexer.SkipByte(' ') {
+ }
+
+ varnameStart := lexer.Mark()
+ // TODO: duplicated code in MkParser.Varname
+ for lexer.NextBytesSet(VarbaseBytes) != "" || lexer.NextVarUse() != nil {
+ }
+ if lexer.SkipByte('.') || hasPrefix(splitResult.main, "SITES_") {
+ for lexer.NextBytesSet(VarparamBytes) != "" || lexer.NextVarUse() != nil {
+ }
+ }
+
+ varname := lexer.Since(varnameStart)
+
+ if varname == "" {
+ return false, nil
+ }
+
+ spaceAfterVarname := lexer.NextHspace()
+
+ opStart := lexer.Mark()
+ switch lexer.PeekByte() {
+ case '!', '+', ':', '?':
+ lexer.Skip(1)
+ }
+ if !lexer.SkipByte('=') {
+ return false, nil
+ }
+ op := NewMkOperator(lexer.Since(opStart))
+
+ if hasSuffix(varname, "+") && op == opAssign && spaceAfterVarname == "" {
+ varname = varname[:len(varname)-1]
+ op = opAssignAppend
+ }
+
+ lexer.SkipHspace()
+
+ value := trimHspace(lexer.Rest())
+ parsedValueAlign := condStr(commented, "#", "") + lexer.Since(mainStart)
+ valueAlign := p.getRawValueAlign(line.raw[0].orignl, parsedValueAlign)
+ if value == "" {
+ valueAlign += splitResult.spaceBeforeComment
+ splitResult.spaceBeforeComment = ""
+ }
+
+ return true, &mkLineAssign{
+ commented: commented,
+ varname: varname,
+ varcanon: varnameCanon(varname),
+ varparam: varnameParam(varname),
+ spaceAfterVarname: spaceAfterVarname,
+ op: op,
+ valueAlign: valueAlign,
+ value: value,
+ valueMk: nil, // filled in lazily
+ valueMkRest: "", // filled in lazily
+ fields: nil, // filled in lazily
+ }
+}
+
+func (p MkLineParser) parseShellcmd(line *Line, splitResult mkLineSplitResult) *MkLine {
+ return &MkLine{line, splitResult, mkLineShell{line.Text[1:]}}
+}
+
+func (p MkLineParser) parseCommentOrEmpty(line *Line, splitResult mkLineSplitResult) *MkLine {
+ trimmedText := trimHspace(line.Text)
+
+ if strings.HasPrefix(trimmedText, "#") {
+ return &MkLine{line, splitResult, mkLineComment{}}
+ }
+
+ if trimmedText == "" {
+ return &MkLine{line, splitResult, mkLineEmpty{}}
+ }
+
+ return nil
+}
+
+func (p MkLineParser) parseDirective(line *Line, splitResult mkLineSplitResult) *MkLine {
+ text := line.Text
+ if !hasPrefix(text, ".") { // TODO: use lexer instead
+ return nil
+ }
+
+ lexer := textproc.NewLexer(splitResult.main[1:])
+
+ indent := lexer.NextHspace()
+ directive := lexer.NextBytesSet(LowerDash)
+ switch directive {
+ case "if", "else", "elif", "endif",
+ "ifdef", "ifndef",
+ "for", "endfor", "undef",
+ "error", "warning", "info",
+ "export", "export-env", "unexport", "unexport-env":
+ break
+ default:
+ // Intentionally not supported are: ifmake ifnmake elifdef elifndef elifmake elifnmake.
+ return nil
+ }
+
+ lexer.SkipHspace()
+
+ args := lexer.Rest()
+
+ // In .if and .endif lines the space surrounding the comment is irrelevant.
+ // Especially for checking that the .endif comment matches the .if condition,
+ // it must be trimmed.
+ trimmedComment := trimHspace(splitResult.comment)
+
+ return &MkLine{line, splitResult, &mkLineDirective{indent, directive, args, trimmedComment, nil, nil, nil}}
+}
+
+func (p MkLineParser) parseInclude(line *Line, splitResult mkLineSplitResult) *MkLine {
+ m, indent, directive, includedFile := MatchMkInclude(splitResult.main)
+ if !m {
+ return nil
+ }
+
+ return &MkLine{line, splitResult, &mkLineInclude{directive == "include", false, indent, includedFile, nil}}
+}
+
+func (p MkLineParser) parseSysinclude(line *Line, splitResult mkLineSplitResult) *MkLine {
+ m, indent, directive, includedFile := match3(splitResult.main, `^\.([\t ]*)(s?include)[\t ]+<([^>]+)>$`)
+ if !m {
+ return nil
+ }
+
+ return &MkLine{line, splitResult, &mkLineInclude{directive == "include", true, indent, includedFile, nil}}
+}
+
+func (p MkLineParser) parseDependency(line *Line, splitResult mkLineSplitResult) *MkLine {
+ // XXX: Replace this regular expression with proper parsing.
+ // There might be a ${VAR:M*.c} in these variables, which the below regular expression cannot handle.
+ m, targets, whitespace, sources := match3(line.Text, `^([^\t :]+(?:[\t ]*[^\t :]+)*)([\t ]*):[\t ]*([^#]*?)(?:[\t ]*#.*)?$`)
+ if !m {
+ return nil
+ }
+
+ if whitespace != "" {
+ line.Notef("Space before colon in dependency line.")
+ }
+ return &MkLine{line, splitResult, mkLineDependency{targets, sources}}
+}
+
+func (p MkLineParser) parseMergeConflict(line *Line, splitResult mkLineSplitResult) *MkLine {
+ if !matches(line.Text, `^(<<<<<<<|=======|>>>>>>>)`) {
+ return nil
+ }
+
+ return &MkLine{line, splitResult, nil}
+}
+
+// split parses a logical line from a Makefile (that is, after joining
+// the lines that end in a backslash) into two parts: the main part and the
+// comment.
+//
+// This applies to all line types except those starting with a tab, which
+// contain the shell commands to be associated with make targets. These cannot
+// have comments.
+//
+// If line is given, it is used for logging parse errors and warnings
+// about round parentheses instead of curly braces, as well as ambiguous
+// variables of the form $v instead of ${v}.
+//
+// If trimComment is true, the main task of this method is to split the
+// text into tokens. The remaining space is placed into spaceBeforeComment,
+// but hasComment will always be false, and comment will always be empty.
+// This behavior is useful for shell commands (which are indented with a
+// single tab).
+func (MkLineParser) split(line *Line, text string, trimComment bool) mkLineSplitResult {
+ assert(!hasPrefix(text, "\t"))
+
+ var mainWithSpaces, comment string
+ if trimComment {
+ mainWithSpaces, comment = NewMkLineParser().unescapeComment(text)
+ } else {
+ mainWithSpaces = text
+ }
+
+ parser := NewMkParser(line, mainWithSpaces)
+ lexer := parser.lexer
+
+ parseOther := func() string {
+ var sb strings.Builder
+
+ for !lexer.EOF() {
+ if lexer.SkipString("$$") {
+ sb.WriteString("$$")
+ continue
+ }
+
+ other := lexer.NextBytesFunc(func(b byte) bool { return b != '$' })
+ if other == "" {
+ break
+ }
+
+ sb.WriteString(other)
+ }
+
+ return sb.String()
+ }
+
+ var tokens []*MkToken
+ for !lexer.EOF() {
+ mark := lexer.Mark()
+
+ if varUse := parser.VarUse(); varUse != nil {
+ tokens = append(tokens, &MkToken{lexer.Since(mark), varUse})
+
+ } else if other := parseOther(); other != "" {
+ tokens = append(tokens, &MkToken{other, nil})
+
+ } else {
+ assert(lexer.SkipByte('$'))
+ tokens = append(tokens, &MkToken{"$", nil})
+ }
+ }
+
+ hasComment := comment != ""
+ if hasComment {
+ comment = comment[1:]
+ }
+
+ mainTrimmed := rtrimHspace(mainWithSpaces)
+ spaceBeforeComment := mainWithSpaces[len(mainTrimmed):]
+ if spaceBeforeComment != "" {
+ tokenText := &tokens[len(tokens)-1].Text
+ *tokenText = rtrimHspace(*tokenText)
+ if *tokenText == "" {
+ if len(tokens) == 1 {
+ tokens = nil
+ } else {
+ tokens = tokens[:len(tokens)-1]
+ }
+ }
+ }
+
+ return mkLineSplitResult{mainTrimmed, tokens, spaceBeforeComment, hasComment, comment}
+}
+
+// unescapeComment takes a Makefile line, as written in a file, and splits
+// it into the main part and the comment.
+//
+// The comment starts at the first #. Except if it is preceded by an odd number
+// of backslashes. Or by an opening bracket.
+//
+// The main text is returned including leading and trailing whitespace.
+// Any escaped # is unescaped, that is, \# becomes #.
+//
+// The comment is returned including the leading "#", if any.
+// If the line has no comment, it is an empty string.
+func (MkLineParser) unescapeComment(text string) (main, comment string) {
+ var sb strings.Builder
+
+ lexer := textproc.NewLexer(text)
+
+again:
+ if plain := lexer.NextBytesSet(unescapeMkCommentSafeChars); plain != "" {
+ sb.WriteString(plain)
+ goto again
+ }
+
+ switch {
+ case lexer.SkipString("\\#"):
+ sb.WriteByte('#')
+
+ case lexer.PeekByte() == '\\' && len(lexer.Rest()) >= 2:
+ sb.WriteString(lexer.Rest()[:2])
+ lexer.Skip(2)
+
+ case lexer.SkipByte('\\'):
+ sb.WriteByte('\\')
+
+ case lexer.SkipString("[#"):
+ // See devel/bmake/files/parse.c:/as in modifier/
+ sb.WriteString("[#")
+
+ case lexer.SkipByte('['):
+ sb.WriteByte('[')
+
+ default:
+ main = sb.String()
+ if lexer.PeekByte() == '#' {
+ return main, lexer.Rest()
+ }
+
+ assert(lexer.EOF())
+ return main, ""
+ }
+
+ goto again
+}
+
+func (MkLineParser) getRawValueAlign(raw, parsed string) string {
+ r := textproc.NewLexer(raw)
+ p := textproc.NewLexer(parsed)
+ mark := r.Mark()
+
+ for !p.EOF() {
+ pch := p.PeekByte()
+ rch := r.PeekByte()
+
+ switch {
+ case pch == rch:
+ p.Skip(1)
+ r.Skip(1)
+
+ case pch == ' ', pch == '\t':
+ p.SkipHspace()
+ r.SkipHspace()
+
+ default:
+ assert(pch == '#')
+ assert(r.SkipString("\\#"))
+ p.Skip(1)
+ }
+ }
+
+ return r.Since(mark)
+}
+
+type mkLineSplitResult struct {
+ // The text of the line, without the comment at the end of the line,
+ // and with # signs unescaped.
+ main string
+ tokens []*MkToken
+ spaceBeforeComment string
+ hasComment bool
+ comment string
+}
Index: pkgsrc/pkgtools/pkglint/files/mklineparser_test.go
diff -u /dev/null pkgsrc/pkgtools/pkglint/files/mklineparser_test.go:1.1
--- /dev/null Sat Oct 26 09:51:48 2019
+++ pkgsrc/pkgtools/pkglint/files/mklineparser_test.go Sat Oct 26 09:51:48 2019
@@ -0,0 +1,1147 @@
+package pkglint
+
+import "gopkg.in/check.v1"
+
+// Exotic code examples from the pkgsrc infrastructure.
+// Hopefully, pkgsrc packages don't need such complicated code.
+// Still, pkglint needs to parse them correctly, or it would not
+// be able to parse and check the infrastructure files as well.
+//
+// See Pkgsrc.loadUntypedVars.
+func (s *Suite) Test_MkLineParser_Parse__infrastructure(c *check.C) {
+ t := s.Init(c)
+
+ mklines := t.NewMkLines("infra.mk",
+ MkCvsID,
+ " USE_BUILTIN.${_pkg_:S/^-//}:=no",
+ ".error \"Something went wrong\"",
+ ".export WRKDIR",
+ ".export",
+ ".unexport-env WRKDIR",
+ "",
+ ".ifmake target1", // Luckily, this is not used in the wild.
+ ".elifnmake target2", // Neither is this.
+ ".endif")
+
+ t.CheckEquals(mklines.mklines[1].Varcanon(), "USE_BUILTIN.*")
+ t.CheckEquals(mklines.mklines[2].Directive(), "error")
+ t.CheckEquals(mklines.mklines[3].Directive(), "export")
+
+ t.CheckOutputLines(
+ "WARN: infra.mk:2: Makefile lines should not start with space characters.",
+ "ERROR: infra.mk:8: Unknown Makefile line format: \".ifmake target1\".",
+ "ERROR: infra.mk:9: Unknown Makefile line format: \".elifnmake target2\".")
+
+ mklines.Check()
+
+ t.CheckOutputLines(
+ "WARN: infra.mk:2: USE_BUILTIN.${_pkg_:S/^-//} is defined but not used.",
+ "WARN: infra.mk:2: _pkg_ is used but not defined.",
+ "ERROR: infra.mk:5: \".export\" requires arguments.",
+ "NOTE: infra.mk:2: This variable value should be aligned to column 41.",
+ "ERROR: infra.mk:10: Unmatched .endif.")
+}
+
+// In variable assignments, a plain '#' introduces a line comment, unless
+// it is escaped by a backslash. In shell commands, on the other hand, it
+// is interpreted literally.
+func (s *Suite) Test_MkLineParser_Parse__comment_or_not(c *check.C) {
+ t := s.Init(c)
+
+ mklineVarassignEscaped := t.NewMkLine("filename.mk", 1, "SED_CMD=\t's,\\#,hash,g'")
+
+ t.CheckEquals(mklineVarassignEscaped.Varname(), "SED_CMD")
+ t.CheckEquals(mklineVarassignEscaped.Value(), "'s,#,hash,g'")
+
+ mklineCommandEscaped := t.NewMkLine("filename.mk", 1, "\tsed -e 's,\\#,hash,g'")
+
+ t.CheckEquals(mklineCommandEscaped.ShellCommand(), "sed -e 's,\\#,hash,g'")
+
+ // From shells/zsh/Makefile.common, rev. 1.78
+ mklineCommandUnescaped := t.NewMkLine("filename.mk", 1, "\t# $ sha1 patches/patch-ac")
+
+ t.CheckEquals(mklineCommandUnescaped.ShellCommand(), "# $ sha1 patches/patch-ac")
+ t.CheckOutputEmpty() // No warning about parsing the lonely dollar sign.
+
+ mklineVarassignUnescaped := t.NewMkLine("filename.mk", 1, "SED_CMD=\t's,#,hash,'")
+
+ t.CheckEquals(mklineVarassignUnescaped.Value(), "'s,")
+ t.CheckOutputLines(
+ "WARN: filename.mk:1: The # character starts a Makefile comment.")
+}
+
+func (s *Suite) Test_MkLineParser_Parse__commented_lines(c *check.C) {
+ t := s.Init(c)
+
+ test := func(text string) {
+ mkline := t.NewMkLines("filename.mk", text).mklines[0]
+ t.CheckEquals(mkline.HasComment(), true)
+ t.CheckEquals(mkline.Comment(), " the comment")
+ }
+
+ test("VAR=value # the comment")
+ test("# the comment")
+ test(".if 0 # the comment")
+ test(".include \"other.mk\" # the comment")
+ test(".include <other.mk> # the comment")
+ test("target: source # the comment")
+}
+
+func (s *Suite) Test_MkLineParser_parseVarassign(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101,
+ "VARNAME.param?=value # varassign comment")
+
+ t.CheckEquals(mkline.IsVarassign(), true)
+ t.CheckEquals(mkline.Varname(), "VARNAME.param")
+ t.CheckEquals(mkline.Varcanon(), "VARNAME.*")
+ t.CheckEquals(mkline.Varparam(), "param")
+ t.CheckEquals(mkline.Op(), opAssignDefault)
+ t.CheckEquals(mkline.Value(), "value")
+ t.CheckEquals(mkline.Comment(), " varassign comment")
+}
+
+func (s *Suite) Test_MkLineParser_parseVarassign__empty_multiline(c *check.C) {
+ t := s.Init(c)
+
+ mklines := t.NewMkLines("test.mk",
+ "VAR=\t\\",
+ "\t\\",
+ "\t\\",
+ "\t# nothing",
+ "",
+ "VAR=\t1\\",
+ "\t\\",
+ "\t\\",
+ "\t# a single letter")
+
+ // Bmake and pkglint agree that the variable value is an empty string.
+ // They don't agree on the exact whitespace in the line, though,
+ // but this doesn't matter in practice. To see the difference, run:
+ // bmake -dA 2>&1 | grep 'ParseReadLine.*VAR'
+ // See devel/bmake/files/parse.c:/non-comment, non-blank line/
+ t.CheckEquals(mklines.mklines[0].Text, "VAR= # nothing")
+ t.CheckEquals(mklines.mklines[2].Text, "VAR=\t1 # a single letter")
+
+ mkline := mklines.mklines[0]
+ t.CheckEquals(mkline.IsVarassign(), true)
+ t.CheckEquals(mkline.Varname(), "VAR")
+ t.CheckEquals(mkline.Op(), opAssign)
+ t.CheckEquals(mkline.Value(), "")
+ t.CheckEquals(mkline.Comment(), " nothing")
+}
+
+func (s *Suite) Test_MkLineParser_parseVarassign__leading_space(c *check.C) {
+ t := s.Init(c)
+
+ _ = t.NewMkLine("rubyversion.mk", 427, " _RUBYVER=\t2.15")
+ _ = t.NewMkLine("bsd.buildlink3.mk", 132, " ok:=yes")
+
+ // In mk/buildlink3/bsd.buildlink3.mk, the leading space is really helpful,
+ // therefore no warnings for that file.
+ t.CheckOutputLines(
+ "WARN: rubyversion.mk:427: Makefile lines should not start with space characters.")
+}
+
+func (s *Suite) Test_MkLineParser_parseVarassign__space_around_operator(c *check.C) {
+ t := s.Init(c)
+
+ t.SetUpCommandLine("--show-autofix", "--source")
+ t.NewMkLine("test.mk", 101,
+ "pkgbase = package")
+
+ t.CheckOutputLines(
+ "NOTE: test.mk:101: Unnecessary space after variable name \"pkgbase\".",
+ "AUTOFIX: test.mk:101: Replacing \"pkgbase =\" with \"pkgbase=\".",
+ "-\tpkgbase = package",
+ "+\tpkgbase= package")
+}
+
+func (s *Suite) Test_MkLineParser_parseVarassign__autofix_space_after_varname(c *check.C) {
+ t := s.Init(c)
+
+ t.SetUpCommandLine("-Wspace")
+ filename := t.CreateFileLines("Makefile",
+ MkCvsID,
+ "VARNAME +=\t${VARNAME}",
+ "VARNAME+ =\t${VARNAME+}",
+ "VARNAME+ +=\t${VARNAME+}",
+ "VARNAME+ ?=\t${VARNAME}",
+ "pkgbase := pkglint")
+
+ CheckFileMk(filename)
+
+ t.CheckOutputLines(
+ "NOTE: ~/Makefile:2: Unnecessary space after variable name \"VARNAME\".",
+
+ // The assignment operators other than = and += cannot lead to ambiguities.
+ "NOTE: ~/Makefile:5: Unnecessary space after variable name \"VARNAME+\".")
+
+ t.SetUpCommandLine("-Wspace", "--autofix")
+
+ CheckFileMk(filename)
+
+ t.CheckOutputLines(
+ "AUTOFIX: ~/Makefile:2: Replacing \"VARNAME +=\" with \"VARNAME+=\".",
+ "AUTOFIX: ~/Makefile:5: Replacing \"VARNAME+ ?=\" with \"VARNAME+?=\".")
+ t.CheckFileLines("Makefile",
+ MkCvsID+"",
+ "VARNAME+=\t${VARNAME}",
+ "VARNAME+ =\t${VARNAME+}",
+ "VARNAME+ +=\t${VARNAME+}",
+ "VARNAME+?=\t${VARNAME}",
+ "pkgbase := pkglint")
+}
+
+func (s *Suite) Test_MkLineParser_parseVarassign__append(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101,
+ "VARNAME+=value")
+
+ t.CheckEquals(mkline.IsVarassign(), true)
+ t.CheckEquals(mkline.Varname(), "VARNAME")
+ t.CheckEquals(mkline.Varcanon(), "VARNAME")
+ t.CheckEquals(mkline.Varparam(), "")
+}
+
+func (s *Suite) Test_MkLineParser_parseVarassign__varname_with_hash(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("Makefile", 123, "VARNAME.#=\tvalue")
+
+ // Parse error because the # starts a comment.
+ t.CheckEquals(mkline.IsVarassign(), false)
+
+ mkline2 := t.NewMkLine("Makefile", 124, "VARNAME.\\#=\tvalue")
+
+ t.CheckEquals(mkline2.IsVarassign(), true)
+ t.CheckEquals(mkline2.Varname(), "VARNAME.#")
+
+ t.CheckOutputLines(
+ "ERROR: Makefile:123: Unknown Makefile line format: \"VARNAME.#=\\tvalue\".")
+}
+
+// Ensures that pkglint parses escaped # characters in the same way as bmake.
+//
+// To check that bmake parses them the same, set a breakpoint after the t.NewMkLines
+// and look in t.tmpdir for the location of the file. Then run bmake with that file.
+func (s *Suite) Test_MkLineParser_parseVarassign__escaped_hash_in_value(c *check.C) {
+ t := s.Init(c)
+
+ mklines := t.SetUpFileMkLines("Makefile",
+ "VAR0=\tvalue#",
+ "VAR1=\tvalue\\#",
+ "VAR2=\tvalue\\\\#",
+ "VAR3=\tvalue\\\\\\#",
+ "VAR4=\tvalue\\\\\\\\#",
+ "",
+ "all:",
+ ".for var in VAR0 VAR1 VAR2 VAR3 VAR4",
+ "\t@printf '%s\\n' ${${var}}''",
+ ".endfor")
+ parsed := mklines.mklines
+
+ t.CheckEquals(parsed[0].Value(), "value")
+ t.CheckEquals(parsed[1].Value(), "value#")
+ t.CheckEquals(parsed[2].Value(), "value\\\\")
+ t.CheckEquals(parsed[3].Value(), "value\\\\#")
+ t.CheckEquals(parsed[4].Value(), "value\\\\\\\\")
+
+ t.CheckOutputLines(
+ "WARN: ~/Makefile:1: The # character starts a Makefile comment.",
+ "WARN: ~/Makefile:3: The # character starts a Makefile comment.",
+ "WARN: ~/Makefile:5: The # character starts a Makefile comment.")
+}
+
+func (s *Suite) Test_MkLineParser_MatchVarassign(c *check.C) {
+ t := s.Init(c)
+
+ testLine := func(line *Line, commented bool, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment string, diagnostics ...string) {
+ text := line.Text
+
+ parser := NewMkLineParser()
+ splitResult := parser.split(nil, text, true)
+ m, actual := parser.MatchVarassign(line, text, &splitResult)
+
+ assert(m)
+ expected := mkLineAssign{
+ commented: commented,
+ varname: varname,
+ varcanon: varnameCanon(varname),
+ varparam: varnameParam(varname),
+ spaceAfterVarname: spaceAfterVarname,
+ op: NewMkOperator(op),
+ valueAlign: align,
+ value: value,
+ valueMk: nil,
+ valueMkRest: "",
+ fields: nil,
+ }
+ t.CheckDeepEquals(*actual, expected)
+ t.CheckEquals(splitResult.spaceBeforeComment, spaceAfterValue)
+ t.CheckEquals(splitResult.hasComment, comment != "")
+ t.CheckEquals(condStr(splitResult.hasComment, "#", "")+splitResult.comment, comment)
+ t.CheckOutput(diagnostics)
+ }
+
+ test := func(text string, commented bool, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment string, diagnostics ...string) {
+ line := t.NewLine("filename.mk", 123, text)
+ testLine(line, commented, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment, diagnostics...)
+ }
+
+ testInvalid := func(text string, diagnostics ...string) {
+ line := t.NewLine("filename.mk", 123, text)
+ parser := NewMkLineParser()
+ splitResult := parser.split(nil, text, true)
+ m, _ := parser.MatchVarassign(line, text, &splitResult)
+ if m {
+ c.Errorf("Text %q matches variable assignment but shouldn't.", text)
+ }
+ t.CheckOutput(diagnostics)
+ }
+
+ lines := func(text ...string) *Line {
+ mklines := t.NewMkLines("filename.mk",
+ text...)
+ return mklines.mklines[0].Line
+ }
+
+ test("C++=c11", false, "C+", "", "+=", "C++=", "c11", "", "")
+ test("V=v", false, "V", "", "=", "V=", "v", "", "")
+ test("VAR=#comment", false, "VAR", "", "=", "VAR=", "", "", "#comment")
+ test("VAR=\\#comment", false, "VAR", "", "=", "VAR=", "#comment", "", "")
+ test("VAR=\\\\\\##comment", false, "VAR", "", "=", "VAR=", "\\\\#", "", "#comment")
+ test("VAR=\\", false, "VAR", "", "=", "VAR=", "\\", "", "")
+ test("VAR += value", false, "VAR", " ", "+=", "VAR += ", "value", "", "")
+ test(" VAR=value", false, "VAR", "", "=", " VAR=", "value", "", "")
+ test("VAR=value #comment", false, "VAR", "", "=", "VAR=", "value", " ", "#comment")
+ test("NFILES=${FILES:[#]}", false, "NFILES", "", "=", "NFILES=", "${FILES:[#]}", "", "")
+
+ // To humans, the base variable name seems to be SITES_, being parameterized
+ // with distfile-1.0.tar.gz. For pkglint though, the base variable name is
+ // SITES_distfile-1.
+ test("SITES_distfile-1.0.tar.gz=https://example.org/";,
+ false,
+ "SITES_distfile-1.0.tar.gz",
+ "",
+ "=",
+ "SITES_distfile-1.0.tar.gz=",
+ "https://example.org/";,
+ "",
+ "")
+
+ test("SITES_${distfile}=https://example.org/";,
+ false,
+ "SITES_${distfile}",
+ "",
+ "=",
+ "SITES_${distfile}=",
+ "https://example.org/";,
+ "",
+ "")
+
+ t.ExpectAssert(func() { testInvalid("\tVAR=value") })
+ testInvalid("?=value")
+ testInvalid("<=value")
+ testInvalid("#")
+ testInvalid("VAR.$$=value")
+
+ // A commented variable assignment must start immediately after the comment character.
+ // There must be no additional whitespace before the variable name.
+ test("#VAR=value", true, "VAR", "", "=", "#VAR=", "value", "", "")
+
+ // A single space is typically used for writing documentation, not for commenting out code.
+ // Therefore this line doesn't count as commented variable assignment.
+ testInvalid("# VAR=value")
+
+ // Ensure that the alignment for the variable value is correct.
+ test("BUILD_DIRS=\tdir1 dir2",
+ false,
+ "BUILD_DIRS",
+ "",
+ "=",
+ "BUILD_DIRS=\t",
+ "dir1 dir2",
+ "",
+ "")
+
+ // Ensure that the alignment for the variable value is correct,
+ // even if the whole line is commented.
+ test("#BUILD_DIRS=\tdir1 dir2",
+ true,
+ "BUILD_DIRS",
+ "",
+ "=",
+ "#BUILD_DIRS=\t",
+ "dir1 dir2",
+ "",
+ "")
+
+ test("MASTER_SITES=\t#none",
+ false,
+ "MASTER_SITES",
+ "",
+ "=",
+ "MASTER_SITES=\t",
+ "",
+ "",
+ "#none")
+
+ test("MASTER_SITES=\t# none",
+ false,
+ "MASTER_SITES",
+ "",
+ "=",
+ "MASTER_SITES=\t",
+ "",
+ "",
+ "# none")
+
+ test("EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d",
+
+ false,
+ "EGDIRS",
+ "",
+ "=",
+ "EGDIRS=\t",
+ "${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d",
+ "",
+ "")
+
+ test("VAR:=\t${VAR:M-*:[\\#]}",
+ false,
+ "VAR",
+ "",
+ ":=",
+ "VAR:=\t",
+ "${VAR:M-*:[#]}",
+ "",
+ "")
+
+ test("#VAR=value",
+ true, "VAR", "", "=", "#VAR=", "value", "", "")
+
+ testInvalid("# VAR=value")
+ testInvalid("#\tVAR=value")
+ testInvalid(MkCvsID)
+
+ testLine(
+ lines(
+ "VAR=\t\t\t\\",
+ "\tvalue"),
+ false,
+ "VAR",
+ "",
+ "=",
+ "VAR=\t\t\t",
+ "value",
+ "",
+ "")
+
+ testLine(
+ lines(
+ "#VAR=\t\t\t\\",
+ "#\tvalue"),
+ true,
+ "VAR",
+ "",
+ "=",
+ "#VAR=\t\t\t",
+ "value",
+ "",
+ "")
+}
+
+func (s *Suite) Test_MkLineParser_parseShellcmd(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101,
+ "\tshell command # shell comment")
+
+ t.CheckEquals(mkline.IsShellCommand(), true)
+ t.CheckEquals(mkline.ShellCommand(), "shell command # shell comment")
+}
+
+func (s *Suite) Test_MkLineParser_parseCommentOrEmpty__comment(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101,
+ "# whole line comment")
+
+ t.CheckEquals(mkline.IsComment(), true)
+}
+
+func (s *Suite) Test_MkLineParser_parseCommentOrEmpty__empty(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101, "")
+
+ t.CheckEquals(mkline.IsEmpty(), true)
+}
+
+func (s *Suite) Test_MkLineParser_parseDirective(c *check.C) {
+ t := s.Init(c)
+
+ test := func(input, expectedIndent, expectedDirective, expectedArgs, expectedComment string, diagnostics ...string) {
+ line := t.NewLine("filename.mk", 123, input)
+ parser := NewMkLineParser()
+ splitResult := parser.split(line, input, true)
+ mkline := parser.parseDirective(line, splitResult)
+ if !c.Check(mkline, check.NotNil) {
+ return
+ }
+
+ t.CheckDeepEquals(
+ []interface{}{mkline.Indent(), mkline.Directive(), mkline.Args(), mkline.DirectiveComment()},
+ []interface{}{expectedIndent, expectedDirective, expectedArgs, expectedComment})
+ t.CheckOutput(diagnostics)
+ }
+
+ test(".if ${VAR} == value",
+ "", "if", "${VAR} == value", "")
+
+ test(".\tendif # comment",
+ "\t", "endif", "", "comment")
+
+ test(".if ${VAR} == \"#\"",
+ "", "if", "${VAR} == \"", "\"")
+
+ test(".if ${VAR:[#]}",
+ "", "if", "${VAR:[#]}", "")
+
+ test(".if ${VAR} == \\",
+ "", "if", "${VAR} == \\", "")
+
+ test(".if ${VAR",
+ "", "if", "${VAR", "",
+ "WARN: filename.mk:123: Missing closing \"}\" for \"VAR\".")
+}
+
+func (s *Suite) Test_MkLineParser_parseDirective__escaped_hash(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101,
+ ". if !empty(PKGNAME:M*-*) && ${RUBY_RAILS_SUPPORTED:[\\#]} == 1 # directive comment")
+
+ t.CheckEquals(mkline.IsDirective(), true)
+ t.CheckEquals(mkline.Indent(), " ")
+ t.CheckEquals(mkline.Directive(), "if")
+ t.CheckEquals(mkline.Args(), "!empty(PKGNAME:M*-*) && ${RUBY_RAILS_SUPPORTED:[#]} == 1")
+ t.CheckEquals(mkline.DirectiveComment(), "directive comment")
+}
+
+func (s *Suite) Test_MkLineParser_parseInclude(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101,
+ ". include \"../../mk/bsd.prefs.mk\" # include comment")
+
+ t.CheckEquals(mkline.IsInclude(), true)
+ t.CheckEquals(mkline.Indent(), " ")
+ t.CheckEquals(mkline.MustExist(), true)
+ t.CheckEquals(mkline.IncludedFile(), "../../mk/bsd.prefs.mk")
+
+ t.CheckEquals(mkline.IsSysinclude(), false)
+}
+
+func (s *Suite) Test_MkLineParser_parseSysinclude(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101,
+ ". include <subdir.mk> # sysinclude comment")
+
+ t.CheckEquals(mkline.IsSysinclude(), true)
+ t.CheckEquals(mkline.Indent(), " ")
+ t.CheckEquals(mkline.MustExist(), true)
+ t.CheckEquals(mkline.IncludedFile(), "subdir.mk")
+
+ t.CheckEquals(mkline.IsInclude(), false)
+}
+
+func (s *Suite) Test_MkLineParser_parseDependency(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101,
+ "target1 target2: source1 source2")
+
+ t.CheckEquals(mkline.IsDependency(), true)
+ t.CheckEquals(mkline.Targets(), "target1 target2")
+ t.CheckEquals(mkline.Sources(), "source1 source2")
+}
+
+func (s *Suite) Test_MkLineParser_parseDependency__space(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101,
+ "target : source")
+
+ t.CheckEquals(mkline.Targets(), "target")
+ t.CheckEquals(mkline.Sources(), "source")
+ t.CheckOutputLines(
+ "NOTE: test.mk:101: Space before colon in dependency line.")
+}
+
+func (s *Suite) Test_MkLineParser_parseMergeConflict(c *check.C) {
+ t := s.Init(c)
+
+ mkline := t.NewMkLine("test.mk", 101,
+ "<<<<<<<<<<<<<<<<<")
+
+ // Merge conflicts are of neither type.
+ t.CheckEquals(mkline.IsVarassign(), false)
+ t.CheckEquals(mkline.IsDirective(), false)
+ t.CheckEquals(mkline.IsInclude(), false)
+ t.CheckEquals(mkline.IsEmpty(), false)
+ t.CheckEquals(mkline.IsComment(), false)
+ t.CheckEquals(mkline.IsDependency(), false)
+ t.CheckEquals(mkline.IsShellCommand(), false)
+ t.CheckEquals(mkline.IsSysinclude(), false)
+}
+
+func (s *Suite) Test_MkLineParser_split(c *check.C) {
+ t := s.Init(c)
+ b := NewMkTokenBuilder()
+
+ varuse := b.VaruseToken
+ varuseText := b.VaruseTextToken
+ text := b.TextToken
+ tokens := b.Tokens
+
+ test := func(text string, expected mkLineSplitResult, diagnostics ...string) {
+ line := t.NewLine("filename.mk", 123, text)
+ actual := NewMkLineParser().split(line, text, true)
+
+ t.CheckOutput(diagnostics)
+ t.CheckDeepEquals([]interface{}{text, actual}, []interface{}{text, expected})
+ }
+
+ t.Use(text, varuse, varuseText, tokens)
+
+ test(
+ "",
+ mkLineSplitResult{})
+
+ test(
+ "text",
+ mkLineSplitResult{
+ main: "text",
+ tokens: tokens(text("text")),
+ })
+
+ // Leading space is always kept.
+ test(
+ " text",
+ mkLineSplitResult{
+ main: " text",
+ tokens: tokens(text(" text")),
+ })
+
+ // Trailing space does not end up in the tokens since it is usually
+ // ignored.
+ test(
+ "text\t",
+ mkLineSplitResult{
+ main: "text",
+ tokens: tokens(text("text")),
+ spaceBeforeComment: "\t",
+ })
+
+ test(
+ "text\t# intended comment",
+ mkLineSplitResult{
+ main: "text",
+ tokens: tokens(text("text")),
+ spaceBeforeComment: "\t",
+ hasComment: true,
+ comment: " intended comment",
+ })
+
+ // Trailing space is saved in a separate field to detect accidental
+ // unescaped # in the middle of a word, like the URL fragment in this
+ // example.
+ test(
+ "url#fragment",
+ mkLineSplitResult{
+ main: "url",
+ tokens: tokens(text("url")),
+ hasComment: true,
+ comment: "fragment",
+ })
+
+ // The leading space from the comment is preserved to make parsing as exact
+ // as possible.
+ //
+ // The difference between "#defined" and "# defined" is relevant in a few
+ // cases, such as the API documentation of the infrastructure files.
+ test("# comment",
+ mkLineSplitResult{
+ hasComment: true,
+ comment: " comment",
+ })
+
+ test("#\tcomment",
+ mkLineSplitResult{
+ hasComment: true,
+ comment: "\tcomment",
+ })
+
+ test("# comment",
+ mkLineSplitResult{
+ hasComment: true,
+ comment: " comment",
+ })
+
+ test(
+ "#VAR=#value",
+ mkLineSplitResult{
+ hasComment: true,
+ comment: "VAR=#value"})
+
+ // When parsing a commented variable assignment, the code assumes that
+ // the whole comment is left uninterpreted.
+ test(
+ "#VAR=\\#value",
+ mkLineSplitResult{
+ hasComment: true,
+ comment: "VAR=\\#value"})
+
+ // Other than in the shell, # also starts a comment in the middle of a word.
+ test("COMMENT=\tThe C# compiler",
+ mkLineSplitResult{
+ main: "COMMENT=\tThe C",
+ tokens: tokens(text("COMMENT=\tThe C")),
+ hasComment: true,
+ comment: " compiler",
+ })
+
+ test("COMMENT=\tThe C\\# compiler",
+ mkLineSplitResult{
+ main: "COMMENT=\tThe C# compiler",
+ tokens: tokens(text("COMMENT=\tThe C# compiler")),
+ hasComment: false,
+ comment: "",
+ })
+
+ test("${TARGET}: ${SOURCES} # comment",
+ mkLineSplitResult{
+ main: "${TARGET}: ${SOURCES}",
+ tokens: tokens(varuse("TARGET"), text(": "), varuse("SOURCES")),
+ spaceBeforeComment: " ",
+ hasComment: true,
+ comment: " comment",
+ })
+
+ // A # starts a comment, except if it immediately follows a [.
+ // This is done so that the length modifier :[#] can be written without
+ // escaping the #.
+ test("VAR=\t${OTHER:[#]} # comment",
+ mkLineSplitResult{
+ main: "VAR=\t${OTHER:[#]}",
+ tokens: tokens(text("VAR=\t"), varuse("OTHER", "[#]")),
+ spaceBeforeComment: " ",
+ hasComment: true,
+ comment: " comment",
+ })
+
+ // The # in the :[#] modifier may be escaped or not. Both forms are equivalent.
+ test("VAR:=\t${VAR:M-*:[\\#]}",
+ mkLineSplitResult{
+ main: "VAR:=\t${VAR:M-*:[#]}",
+ tokens: tokens(text("VAR:=\t"), varuse("VAR", "M-*", "[#]")),
+ })
+
+ // A backslash always escapes the next character, be it a # for a comment
+ // or something else. This makes it difficult to write a literal \# in a
+ // Makefile, but that's an edge case anyway.
+ test("VAR0=\t#comment",
+ mkLineSplitResult{
+ main: "VAR0=",
+ tokens: tokens(text("VAR0=")),
+ // Later, when converting this result into a proper variable assignment,
+ // this "space before comment" is reclassified as "space before the value",
+ // in order to align the "#comment" with the other variable values.
+ spaceBeforeComment: "\t",
+ hasComment: true,
+ comment: "comment",
+ })
+
+ test("VAR1=\t\\#no-comment",
+ mkLineSplitResult{
+ main: "VAR1=\t#no-comment",
+ tokens: tokens(text("VAR1=\t#no-comment")),
+ })
+
+ test("VAR2=\t\\\\#comment",
+ mkLineSplitResult{
+ main: "VAR2=\t\\\\",
+ tokens: tokens(text("VAR2=\t\\\\")),
+ hasComment: true,
+ comment: "comment",
+ })
+
+ // The backslash is only removed when it escapes a comment.
+ // In particular, it cannot be used to escape a dollar that starts a
+ // variable use.
+ test("VAR0=\t$T",
+ mkLineSplitResult{
+ main: "VAR0=\t$T",
+ tokens: tokens(text("VAR0=\t"), varuseText("$T", "T")),
+ },
+ "WARN: filename.mk:123: $T is ambiguous. Use ${T} if you mean a Make variable or $$T if you mean a shell variable.")
+
+ test("VAR1=\t\\$T",
+ mkLineSplitResult{
+ main: "VAR1=\t\\$T",
+ tokens: tokens(text("VAR1=\t\\"), varuseText("$T", "T")),
+ },
+ "WARN: filename.mk:123: $T is ambiguous. Use ${T} if you mean a Make variable or $$T if you mean a shell variable.")
+
+ test("VAR2=\t\\\\$T",
+ mkLineSplitResult{
+ main: "VAR2=\t\\\\$T",
+ tokens: tokens(text("VAR2=\t\\\\"), varuseText("$T", "T")),
+ },
+ "WARN: filename.mk:123: $T is ambiguous. Use ${T} if you mean a Make variable or $$T if you mean a shell variable.")
+
+ // To escape a dollar, write it twice.
+ test("$$shellvar $${shellvar} \\${MKVAR} [] \\x",
+ mkLineSplitResult{
+ main: "$$shellvar $${shellvar} \\${MKVAR} [] \\x",
+ tokens: tokens(text("$$shellvar $${shellvar} \\"), varuse("MKVAR"), text(" [] \\x")),
+ })
+
+ // Parse errors are recorded in the rest return value.
+ test("${UNCLOSED",
+ mkLineSplitResult{
+ main: "${UNCLOSED",
+ tokens: tokens(varuseText("${UNCLOSED", "UNCLOSED")),
+ },
+ "WARN: filename.mk:123: Missing closing \"}\" for \"UNCLOSED\".")
+
+ // Even if there is a parse error in the main part,
+ // the comment is extracted.
+ test("text before ${UNCLOSED# comment",
+ mkLineSplitResult{
+ main: "text before ${UNCLOSED",
+ tokens: tokens(
+ text("text before "),
+ varuseText("${UNCLOSED", "UNCLOSED")),
+ hasComment: true,
+ comment: " comment",
+ },
+ "WARN: filename.mk:123: Missing closing \"}\" for \"UNCLOSED\".")
+
+ // Even in case of parse errors, the space before the comment is parsed
+ // correctly.
+ test("text before ${UNCLOSED # comment",
+ mkLineSplitResult{
+ main: "text before ${UNCLOSED",
+ tokens: tokens(
+ text("text before "),
+ // It's a bit inconsistent that the varname includes the space
+ // but the text doesn't; anyway, it's an edge case.
+ varuseText("${UNCLOSED", "UNCLOSED ")),
+ spaceBeforeComment: " ",
+ hasComment: true,
+ comment: " comment",
+ },
+ "WARN: filename.mk:123: Missing closing \"}\" for \"UNCLOSED \".",
+ "WARN: filename.mk:123: Invalid part \" \" after variable name \"UNCLOSED\".")
+
+ // The dollar-space refers to a normal Make variable named " ".
+ // The lonely dollar at the very end refers to the variable named "",
+ // which is specially protected in bmake to always contain the empty string.
+ // It is heavily used in .for loops in the form ${:Uvalue}.
+ //
+ // TODO: The rest of pkglint assumes that the empty string is not a valid
+ // variable name, mainly because the empty variable name is not visible
+ // outside of the bmake debugging mode.
+ test("Lonely $ character $",
+ mkLineSplitResult{
+ main: "Lonely $ character $",
+ tokens: tokens(
+ text("Lonely "),
+ varuseText("$ " /* instead of "${ }" */, " "),
+ text("character "),
+ text("$")),
+ })
+
+ // The character [ prevents the following # from starting a comment, even
+ // outside of variable modifiers.
+ test("COMMENT=\t[#] $$\\# $$# comment",
+ mkLineSplitResult{
+ main: "COMMENT=\t[#] $$# $$",
+ tokens: tokens(text("COMMENT=\t[#] $$# $$")),
+ hasComment: true,
+ comment: " comment",
+ })
+
+ test("VAR2=\t\\\\#comment",
+ mkLineSplitResult{
+ main: "VAR2=\t\\\\",
+ tokens: tokens(text("VAR2=\t\\\\")),
+ hasComment: true,
+ comment: "comment",
+ })
+
+ // At this stage, MkLine.split doesn't know that empty(...) takes
+ // a variable use. Instead it just sees ordinary characters and
+ // other uses of variables.
+ test(".if empty(${VAR.${tool}}:C/\\:.*$//:M${pattern})",
+ mkLineSplitResult{
+ main: ".if empty(${VAR.${tool}}:C/\\:.*$//:M${pattern})",
+ tokens: tokens(
+ text(".if empty("),
+ varuse("VAR.${tool}"),
+ text(":C/\\:.*"),
+ text("$"),
+ text("//:M"),
+ varuse("pattern"),
+ text(")")),
+ })
+
+ test(" # comment after spaces",
+ mkLineSplitResult{
+ spaceBeforeComment: " ",
+ hasComment: true,
+ comment: " comment after spaces",
+ })
+
+ // FIXME: This theoretical edge case is interpreted differently
+ // between bmake and pkglint. Pkglint treats the # as a comment,
+ // while bmake interprets it as a regular character.
+ test("\\[#",
+ mkLineSplitResult{
+ main: "\\[",
+ tokens: tokens(text("\\[")),
+ hasComment: true,
+ })
+
+ test("\\\\[#",
+ mkLineSplitResult{
+ main: "\\\\[#",
+ tokens: tokens(text("\\\\[#")),
+ })
+}
+
+func (s *Suite) Test_MkLineParser_split__preserve_comment(c *check.C) {
+ t := s.Init(c)
+ b := NewMkTokenBuilder()
+
+ tokens := b.Tokens
+ text := b.TextToken
+ varUse := b.VaruseToken
+
+ test := func(text string, expected mkLineSplitResult, diagnostics ...string) {
+ line := t.NewLine("filename.mk", 123, text)
+ actual := NewMkLineParser().split(line, text, false)
+
+ t.CheckDeepEquals(actual, expected)
+ t.CheckOutput(diagnostics)
+ }
+
+ test(
+ "text\t# no comment",
+ mkLineSplitResult{
+ main: "text\t# no comment",
+ tokens: tokens(text("text\t# no comment"))})
+
+ test(
+ "url#fragment",
+ mkLineSplitResult{
+ main: "url#fragment",
+ tokens: tokens(text("url#fragment"))})
+
+ test("# no comment",
+ mkLineSplitResult{
+ main: "# no comment",
+ tokens: tokens(text("# no comment"))})
+
+ // Other than in the shell, # also starts a comment in the middle of a word.
+ test("The C# compiler",
+ mkLineSplitResult{
+ main: "The C# compiler",
+ tokens: tokens(text("The C# compiler"))})
+
+ test("The C\\# compiler",
+ mkLineSplitResult{
+ main: "The C\\# compiler",
+ tokens: tokens(text("The C\\# compiler"))})
+
+ test("# ${VAR}",
+ mkLineSplitResult{
+ main: "# ${VAR}",
+ tokens: tokens(text("# "), varUse("VAR"))})
+
+ test("# ",
+ mkLineSplitResult{
+ main: "#",
+ tokens: tokens(text("#")),
+ spaceBeforeComment: " "})
+}
+
+func (s *Suite) Test_MkLineParser_split__unclosed_varuse(c *check.C) {
+ t := s.Init(c)
+ b := NewMkTokenBuilder()
+
+ test := func(text string, expected mkLineSplitResult, diagnostics ...string) {
+ line := t.NewLine("filename.mk", 123, text)
+
+ splitResult := NewMkLineParser().split(line, text, true)
+
+ t.CheckDeepEquals(splitResult, expected)
+ t.CheckOutput(diagnostics)
+ }
+
+ test(
+ "EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d",
+
+ mkLineSplitResult{
+ "EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d",
+ b.Tokens(
+ b.TextToken("EGDIRS=\t"),
+ b.VaruseTextToken(
+ "${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d",
+ "EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d")),
+ "",
+ false,
+ "",
+ },
+
+ "WARN: filename.mk:123: Missing closing \"}\" for \"EGDIR/pam.d\".",
+ "WARN: filename.mk:123: Invalid part \"/pam.d\" after variable name \"EGDIR\".",
+ "WARN: filename.mk:123: Missing closing \"}\" for \"EGDIR/dbus-1/system.d ${EGDIR/pam.d\".",
+ "WARN: filename.mk:123: Invalid part \"/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".",
+ "WARN: filename.mk:123: Missing closing \"}\" for \"EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".",
+ "WARN: filename.mk:123: Invalid part \"/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".")
+}
+
+func (s *Suite) Test_MkLineParser_unescapeComment(c *check.C) {
+ t := s.Init(c)
+
+ test := func(text string, main, comment string) {
+ aMain, aComment := NewMkLineParser().unescapeComment(text)
+ t.CheckDeepEquals(
+ []interface{}{text, aMain, aComment},
+ []interface{}{text, main, comment})
+ }
+
+ test("",
+ "",
+ "")
+ test("text",
+ "text",
+ "")
+
+ // The leading space from the comment is preserved to make parsing as exact
+ // as possible.
+ //
+ // The difference between "#defined" and "# defined" is relevant in a few
+ // cases, such as the API documentation of the infrastructure files.
+ test("# comment",
+ "",
+ "# comment")
+ test("#\tcomment",
+ "",
+ "#\tcomment")
+ test("# comment",
+ "",
+ "# comment")
+
+ // Other than in the shell, # also starts a comment in the middle of a word.
+ test("COMMENT=\tThe C# compiler",
+ "COMMENT=\tThe C",
+ "# compiler")
+ test("COMMENT=\tThe C\\# compiler",
+ "COMMENT=\tThe C# compiler",
+ "")
+
+ test("${TARGET}: ${SOURCES} # comment",
+ "${TARGET}: ${SOURCES} ",
+ "# comment")
+
+ // A # starts a comment, except if it immediately follows a [.
+ // This is done so that the length modifier :[#] can be written without
+ // escaping the #.
+ test("VAR=\t${OTHER:[#]} # comment",
+ "VAR=\t${OTHER:[#]} ",
+ "# comment")
+
+ // The # in the :[#] modifier may be escaped or not. Both forms are equivalent.
+ test("VAR:=\t${VAR:M-*:[\\#]}",
+ "VAR:=\t${VAR:M-*:[#]}",
+ "")
+
+ // The character [ prevents the following # from starting a comment, even
+ // outside of variable modifiers.
+ test("COMMENT=\t[#] $$\\# $$# comment",
+ "COMMENT=\t[#] $$# $$",
+ "# comment")
+
+ // A backslash always escapes the next character, be it a # for a comment
+ // or something else. This makes it difficult to write a literal \# in a
+ // Makefile, but that's an edge case anyway.
+ test("VAR0=\t#comment",
+ "VAR0=\t",
+ "#comment")
+ test("VAR1=\t\\#no-comment",
+ "VAR1=\t#no-comment",
+ "")
+ test("VAR2=\t\\\\#comment",
+ "VAR2=\t\\\\",
+ "#comment")
+
+ // The backslash is only removed when it escapes a comment.
+ // In particular, it cannot be used to escape a dollar that starts a
+ // variable use.
+ test("VAR0=\t$T",
+ "VAR0=\t$T",
+ "")
+ test("VAR1=\t\\$T",
+ "VAR1=\t\\$T",
+ "")
+ test("VAR2=\t\\\\$T",
+ "VAR2=\t\\\\$T",
+ "")
+
+ // To escape a dollar, write it twice.
+ test("$$shellvar $${shellvar} \\${MKVAR} [] \\x",
+ "$$shellvar $${shellvar} \\${MKVAR} [] \\x",
+ "")
+
+ // Parse errors are recorded in the rest return value.
+ test("${UNCLOSED",
+ "${UNCLOSED",
+ "")
+
+ // In this early phase of parsing, unfinished variable uses are not
+ // interpreted and do not influence the detection of the comment start.
+ test("text before ${UNCLOSED # comment",
+ "text before ${UNCLOSED ",
+ "# comment")
+
+ // The dollar-space refers to a normal Make variable named " ".
+ // The lonely dollar at the very end refers to the variable named "",
+ // which is specially protected in bmake to always contain the empty string.
+ // It is heavily used in .for loops in the form ${:Uvalue}.
+ test("Lonely $ character $",
+ "Lonely $ character $",
+ "")
+
+ // An even number of backslashes does not escape the #.
+ // Therefore it starts a comment here.
+ test("VAR2=\t\\\\#comment",
+ "VAR2=\t\\\\",
+ "#comment")
+}
+
+func (s *Suite) Test_MkLineParser_getRawValueAlign__assertion(c *check.C) {
+ t := s.Init(c)
+
+ var p MkLineParser
+
+ // This is unrealistic; just for code coverage of the assertion.
+ t.ExpectAssert(func() { p.getRawValueAlign("a", "b") })
+}
Home |
Main Index |
Thread Index |
Old Index