diff --git a/Units/parser-puppetManifest.r/node.d/expected.tags b/Units/parser-puppetManifest.r/node.d/expected.tags index 1c727d8d48..2b86eab13c 100644 --- a/Units/parser-puppetManifest.r/node.d/expected.tags +++ b/Units/parser-puppetManifest.r/node.d/expected.tags @@ -1,3 +1,4 @@ -www1.example.com input.pp /^node 'www1.example.com' {$/;" node line:2 language:PuppetManifest -www2.example.com input.pp /^node 'www2.example.com', 'www3.example.com' {$/;" node line:12 language:PuppetManifest -www3.example.com input.pp /^node 'www2.example.com', 'www3.example.com' {$/;" node line:12 language:PuppetManifest +default input.pp /^node 'default' {}$/;" node line:2 language:PuppetManifest +www1.example.com input.pp /^node 'www1.example.com' {$/;" node line:3 language:PuppetManifest +www2.example.com input.pp /^node 'www2.example.com', 'www3.example.com' {$/;" node line:13 language:PuppetManifest +www3.example.com input.pp /^node 'www2.example.com', 'www3.example.com' {$/;" node line:13 language:PuppetManifest diff --git a/Units/parser-puppetManifest.r/node.d/input.pp b/Units/parser-puppetManifest.r/node.d/input.pp index 98d09f6eed..6f61e912d6 100644 --- a/Units/parser-puppetManifest.r/node.d/input.pp +++ b/Units/parser-puppetManifest.r/node.d/input.pp @@ -1,4 +1,5 @@ /* Taken from https://docs.puppet.com/puppet/5.1/lang_node_definitions.html */ +node 'default' {} node 'www1.example.com' { include common include apache diff --git a/Units/parser-puppetManifest.r/puppet-append.d/validator b/Units/parser-puppetManifest.r/puppet-append.d/validator new file mode 100644 index 0000000000..514326a073 --- /dev/null +++ b/Units/parser-puppetManifest.r/puppet-append.d/validator @@ -0,0 +1,12 @@ +# +# The puppet validator reports this input as "invalid" because +# += operator is obsoleted in puppet5. +# @ahakanbaba in #1909 reported and proposed a fix for this already. +# +# However, @masatake kept this as is for following reasons: +# +# - valid in puppet4, +# - testing KNOWN-INVALIDATION special validator, and +# - testing '#' comment in a validator file. +# +KNOWN-INVALIDATION diff --git a/Units/parser-puppetManifest.r/puppet-collection_within_virtual_definitions.d/expected.tags b/Units/parser-puppetManifest.r/puppet-collection_within_virtual_definitions.d/expected.tags index eeb4594a92..6d17894ae6 100644 --- a/Units/parser-puppetManifest.r/puppet-collection_within_virtual_definitions.d/expected.tags +++ b/Units/parser-puppetManifest.r/puppet-collection_within_virtual_definitions.d/expected.tags @@ -1,5 +1,5 @@ -test input.pp /^define test($name) {$/;" definition line:1 language:PuppetManifest end:6 -/tmp/collection_within_virtual_definitions1_$name.txt input.pp /^ file {"\/tmp\/collection_within_virtual_definitions1_$name.txt":$/;" resource line:2 language:PuppetManifest scope:definition:test end:4 +test input.pp /^define test($my_name) {$/;" definition line:1 language:PuppetManifest end:6 +/tmp/collection_within_virtual_definitions1_$my_name.txt input.pp /^ file {"\/tmp\/collection_within_virtual_definitions1_$my_name.txt":$/;" resource line:2 language:PuppetManifest scope:definition:test end:4 test2 input.pp /^define test2() {$/;" definition line:8 language:PuppetManifest end:12 /tmp/collection_within_virtual_definitions2_$name.txt input.pp /^ file {"\/tmp\/collection_within_virtual_definitions2_$name.txt":$/;" resource line:9 language:PuppetManifest scope:definition:test2 end:11 foo input.pp /^ @test {"foo":$/;" resource line:15 language:PuppetManifest end:17 diff --git a/Units/parser-puppetManifest.r/puppet-collection_within_virtual_definitions.d/input.pp b/Units/parser-puppetManifest.r/puppet-collection_within_virtual_definitions.d/input.pp index 3c21468b07..a31df7bc89 100644 --- a/Units/parser-puppetManifest.r/puppet-collection_within_virtual_definitions.d/input.pp +++ b/Units/parser-puppetManifest.r/puppet-collection_within_virtual_definitions.d/input.pp @@ -1,6 +1,6 @@ -define test($name) { - file {"/tmp/collection_within_virtual_definitions1_$name.txt": - content => "File name $name\n" +define test($my_name) { + file {"/tmp/collection_within_virtual_definitions1_$my_name.txt": + content => "File name $my_name\n" } Test2 <||> } @@ -13,7 +13,7 @@ node default { @test {"foo": - name => "foo" + my_name => "foo" } @test2 {"foo2": } Test <||> diff --git a/Units/parser-puppetManifest.r/puppet-emptyif.d/args.ctags b/Units/parser-puppetManifest.r/puppet-emptyif.d/args.ctags new file mode 100644 index 0000000000..fb884c1049 --- /dev/null +++ b/Units/parser-puppetManifest.r/puppet-emptyif.d/args.ctags @@ -0,0 +1,2 @@ +--sort=no +--fields=+KZlne diff --git a/Units/parser-puppetManifest.r/puppet-emptyif.d/expected.tags b/Units/parser-puppetManifest.r/puppet-emptyif.d/expected.tags new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Units/parser-puppetManifest.r/puppet-emptyif.d/input.pp b/Units/parser-puppetManifest.r/puppet-emptyif.d/input.pp new file mode 100644 index 0000000000..88f8f059f6 --- /dev/null +++ b/Units/parser-puppetManifest.r/puppet-emptyif.d/input.pp @@ -0,0 +1,4 @@ + +if true { + # still nothing +} diff --git a/Units/parser-puppetManifest.r/puppet-emptyifelse.d/input.pp b/Units/parser-puppetManifest.r/puppet-emptyifelse.d/input.pp index 598b486acb..744e2cf81c 100644 --- a/Units/parser-puppetManifest.r/puppet-emptyifelse.d/input.pp +++ b/Units/parser-puppetManifest.r/puppet-emptyifelse.d/input.pp @@ -4,6 +4,3 @@ # nothing here } -if true { - # still nothing -} diff --git a/Units/parser-puppetManifest.r/puppet-multipleclass.d/expected.tags b/Units/parser-puppetManifest.r/puppet-multipleclass.d/expected.tags index 58478ed458..3f2b405c73 100644 --- a/Units/parser-puppetManifest.r/puppet-multipleclass.d/expected.tags +++ b/Units/parser-puppetManifest.r/puppet-multipleclass.d/expected.tags @@ -1,4 +1,4 @@ one input.pp /^class one {$/;" class line:1 language:PuppetManifest end:3 /tmp/multipleclassone input.pp /^ file { "\/tmp\/multipleclassone": content => "one" }$/;" resource line:2 language:PuppetManifest scope:class:one end:2 -one input.pp /^class one {$/;" class line:5 language:PuppetManifest end:7 -/tmp/multipleclasstwo input.pp /^ file { "\/tmp\/multipleclasstwo": content => "two" }$/;" resource line:6 language:PuppetManifest scope:class:one end:6 +two input.pp /^class two {$/;" class line:5 language:PuppetManifest end:7 +/tmp/multipleclasstwo input.pp /^ file { "\/tmp\/multipleclasstwo": content => "two" }$/;" resource line:6 language:PuppetManifest scope:class:two end:6 diff --git a/Units/parser-puppetManifest.r/puppet-multipleclass.d/input.pp b/Units/parser-puppetManifest.r/puppet-multipleclass.d/input.pp index ae02edc382..4f4ed0eaba 100644 --- a/Units/parser-puppetManifest.r/puppet-multipleclass.d/input.pp +++ b/Units/parser-puppetManifest.r/puppet-multipleclass.d/input.pp @@ -2,7 +2,7 @@ file { "/tmp/multipleclassone": content => "one" } } -class one { +class two { file { "/tmp/multipleclasstwo": content => "two" } } diff --git a/Units/parser-puppetManifest.r/puppet-scopetest.d/expected.tags b/Units/parser-puppetManifest.r/puppet-scopetest.d/expected.tags index fffc89e363..7044a1aaf0 100644 --- a/Units/parser-puppetManifest.r/puppet-scopetest.d/expected.tags +++ b/Units/parser-puppetManifest.r/puppet-scopetest.d/expected.tags @@ -1,5 +1,5 @@ -mode input.pp /^$mode = 640$/;" variable line:2 language:PuppetManifest +mode input.pp /^$mode = "640"$/;" variable line:2 language:PuppetManifest thing input.pp /^define thing {$/;" definition line:4 language:PuppetManifest end:6 /tmp/$name input.pp /^ file { "\/tmp\/$name": ensure => file, mode => $mode }$/;" resource line:5 language:PuppetManifest scope:definition:thing end:5 testing input.pp /^class testing {$/;" class line:8 language:PuppetManifest end:11 -mode input.pp /^ $mode = 755$/;" variable line:9 language:PuppetManifest scope:class:testing +mode input.pp /^ $mode = "755"$/;" variable line:9 language:PuppetManifest scope:class:testing diff --git a/Units/parser-puppetManifest.r/puppet-scopetest.d/input.pp b/Units/parser-puppetManifest.r/puppet-scopetest.d/input.pp index 331491766c..8ebd7c31ff 100644 --- a/Units/parser-puppetManifest.r/puppet-scopetest.d/input.pp +++ b/Units/parser-puppetManifest.r/puppet-scopetest.d/input.pp @@ -1,12 +1,12 @@ -$mode = 640 +$mode = "640" define thing { file { "/tmp/$name": ensure => file, mode => $mode } } class testing { - $mode = 755 + $mode = "755" thing {scopetest: } } diff --git a/Units/parser-puppetManifest.r/puppet-selectorvalues.d/input.pp b/Units/parser-puppetManifest.r/puppet-selectorvalues.d/input.pp index d80d26c36d..c11e055de7 100644 --- a/Units/parser-puppetManifest.r/puppet-selectorvalues.d/input.pp +++ b/Units/parser-puppetManifest.r/puppet-selectorvalues.d/input.pp @@ -6,37 +6,37 @@ $test = "yay" $mode1 = $value1 ? { - "" => 755, - default => 644 + "" => "755", + default => "644" } $mode2 = $value2 ? { - true => 755, - default => 644 + true => "755", + default => "644" } $mode3 = $value3 ? { - false => 755, - default => 644 + false => "755", + default => "644" } $mode4 = $value4 ? { - $test => 755, - default => 644 + $test => "755", + default => "644" } $mode5 = yay ? { - $test => 755, - default => 644 + $test => "755", + default => "644" } $mode6 = $mode5 ? { - 755 => 755 + "755" => "755" } $mode7 = "test regex" ? { - /regex$/ => 755, - default => 644 + /regex$/ => "755", + default => "644" } diff --git a/Units/parser-puppetManifest.r/puppet-simpleselector.d/expected.tags b/Units/parser-puppetManifest.r/puppet-simpleselector.d/expected.tags index bb9ef51518..993a1a6f9d 100644 --- a/Units/parser-puppetManifest.r/puppet-simpleselector.d/expected.tags +++ b/Units/parser-puppetManifest.r/puppet-simpleselector.d/expected.tags @@ -3,5 +3,5 @@ var input.pp /^$var = "value"$/;" variable line:3 language:PuppetManifest /tmp/snippetselectbtest input.pp /^file { "\/tmp\/snippetselectbtest":$/;" resource line:13 language:PuppetManifest end:19 othervar input.pp /^$othervar = "complex value"$/;" variable line:21 language:PuppetManifest /tmp/snippetselectctest input.pp /^file { "\/tmp\/snippetselectctest":$/;" resource line:23 language:PuppetManifest end:29 -anothervar input.pp /^$anothervar = Yayness$/;" variable line:30 language:PuppetManifest +anothervar input.pp /^$anothervar = "Yayness"$/;" variable line:30 language:PuppetManifest /tmp/snippetselectdtest input.pp /^file { "\/tmp\/snippetselectdtest":$/;" resource line:32 language:PuppetManifest end:38 diff --git a/Units/parser-puppetManifest.r/puppet-simpleselector.d/input.pp b/Units/parser-puppetManifest.r/puppet-simpleselector.d/input.pp index 8b9bc72923..d1a943903e 100644 --- a/Units/parser-puppetManifest.r/puppet-simpleselector.d/input.pp +++ b/Units/parser-puppetManifest.r/puppet-simpleselector.d/input.pp @@ -5,16 +5,16 @@ file { "/tmp/snippetselectatest": ensure => file, mode => $var ? { - nottrue => 641, - value => 755 + nottrue => "641", + value => "755" } } file { "/tmp/snippetselectbtest": ensure => file, mode => $var ? { - nottrue => 644, - default => 755 + nottrue => "644", + default => "755" } } @@ -23,16 +23,16 @@ file { "/tmp/snippetselectctest": ensure => file, mode => $othervar ? { - "complex value" => 755, - default => 644 + "complex value" => "755", + default => "644" } } -$anothervar = Yayness +$anothervar = "Yayness" file { "/tmp/snippetselectdtest": ensure => file, mode => $anothervar ? { - Yayness => 755, - default => 644 + "Yayness" => "755", + default => "644" } } diff --git a/Units/parser-puppetManifest.r/puppet-singleselector.d/input.pp b/Units/parser-puppetManifest.r/puppet-singleselector.d/input.pp index 520a14017a..ab0477be2c 100644 --- a/Units/parser-puppetManifest.r/puppet-singleselector.d/input.pp +++ b/Units/parser-puppetManifest.r/puppet-singleselector.d/input.pp @@ -6,15 +6,15 @@ $test = "yay" $mode1 = $value1 ? { - "" => 755 + "" => "755" } $mode2 = $value2 ? { - true => 755 + true => "755" } $mode3 = $value3 ? { - default => 755 + default => "755" } file { "/tmp/singleselector1": ensure => file, mode => $mode1 } diff --git a/Units/parser-puppetManifest.r/unless.d/expected.tags b/Units/parser-puppetManifest.r/unless.d/expected.tags index e1c7d5612b..77511af38d 100644 --- a/Units/parser-puppetManifest.r/unless.d/expected.tags +++ b/Units/parser-puppetManifest.r/unless.d/expected.tags @@ -1 +1,2 @@ -/tmp/x input.pp /^ file { "\/tmp\/x": }$/;" resource line:2 language:PuppetManifest end:2 +array input.pp /^$array = [ 3, 5, 7 ]$/;" variable line:1 language:PuppetManifest +/tmp/x input.pp /^ file { "\/tmp\/x": }$/;" resource line:3 language:PuppetManifest end:3 diff --git a/Units/parser-puppetManifest.r/unless.d/input.pp b/Units/parser-puppetManifest.r/unless.d/input.pp index 689cf7a0b7..ebb53974b5 100644 --- a/Units/parser-puppetManifest.r/unless.d/input.pp +++ b/Units/parser-puppetManifest.r/unless.d/input.pp @@ -1,3 +1,4 @@ +$array = [ 3, 5, 7 ] unless $array[0] > 5 { file { "/tmp/x": } } diff --git a/Units/parser-puppetManifest.r/validator b/Units/parser-puppetManifest.r/validator new file mode 100644 index 0000000000..17cb753bb8 --- /dev/null +++ b/Units/parser-puppetManifest.r/validator @@ -0,0 +1 @@ +puppet diff --git a/circle.yml b/circle.yml index be789395db..a8c87f3aca 100644 --- a/circle.yml +++ b/circle.yml @@ -14,7 +14,7 @@ jobs: name: Install build tools command: | dnf -y install gcc automake autoconf pkgconfig bmake aspell-devel aspell-en libxml2-devel jansson-devel libyaml-devel findutils || : - dnf -y install jq + dnf -y install jq puppet || : - run: name: Build command: | diff --git a/docs/input-validation.rst b/docs/input-validation.rst new file mode 100644 index 0000000000..d5a91a0ad6 --- /dev/null +++ b/docs/input-validation.rst @@ -0,0 +1,129 @@ +Input validation for *Units* +--------------------------------------------------------------------- + +:Maintainer: Masatake YAMATO + +---- + +We have to maintain parsers for languages that we don't know well. We +don't have enough time to learn the languages. + +*Units* test cases help us not introduce wrong changes to a parser. + +However, there is still an issue; a developer who doesn't know a +target language well may write a broken test input file for the +language. Here comes "Input validation." + +You can validate the test input files of *Units* with *validate-input* +make target if a validator for a language is defined. + +How to run and an example session +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here is an example validating an input file for JSON. + +.. code-block:: console + + $ make validate-input VALIDATORS=jq + ... + Category: ROOT + ------------------------------------------------------------ + simple-json.d/input.json with jq valid + + Summary + ------------------------------------------------------------ + #valid: 1 + #invalid: 0 + #skipped (known invalidation) 0 + #skipped (validator unavailable) 0 + + +This example shows validating *simple-json.d/input.json* as an input +file with *jq* validator. With VALIDATORS variable passed via +command-line, you can specify validators to run. Multiple validators +can be specified using a comma-separated list. If you don't give +VALIDATORS, the make target tries to use all available validators. + +The meanings of "valid" and "invalid" in "Summary" are apparent. In +two cases, the target skips validating input files: + +#skipped (known invalidation) + + A test case specifies KNOWN-INVALIDATION in its *validator* file. + +#skipped (validator unavailable) + + A command for a validator is not available. + +*validator* file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*validator* file in a *Units* test directory specifies which +validator the make target should use. + +.. code-block:: console + + $ cat Units/simple-json.d/validator + jq + +If you put *validator* file to a category directory (a directory +having *.r* suffix), the make target uses the validator specified in +the file as default. The default validator can be overridden with a +*validator* file in a subdirectory. + +.. code-block:: console + + $ cat Units/parser-puppetManifest.r/validator + puppet + # cat Units/parser-puppetManifest.r/puppet-append.d/validator + KNOWN-INVALIDATION + +In the example, the make target uses *puppet* validator for validating +the most of all input files under *Units/parser-puppetManifest.r* +directory. An exception is an input file under +*Units/parser-puppetManifest.r/puppet-append.d* directory. The +directory has its specific *validator* file. + +If a *Unit* test case doesn't have *expected.tags* file, the make +target doesn't run the validator on the file even if a default +validator is given in its category directory. + +If a *Unit* test case specifies KNOWN-INVALIDATION in its *validator* +file, the make target just increments "#skipped (known invalidation)" +counter. The target reports the counter at the end of execution. + +validator command +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A validator specified in a *validator* file is a command file put +under *misc/validators* directory. The command must have "validator-" +as prefix in its file name. For an example, +*misc/validators/validator-jq* is the command for "jq". + +The command file must be an executable. *validate-input* make target +runs the command in two ways. + +*is_runnable* method + + Before running the command as a validator, the target runs + the command with "is_runnable" as the first argument. + A validator command can let the target know whether the + validator command is runnable or not with exit status. + 0 means ready to run. Non-zero means not ready to run. + + The make target never runs the validator command for + validation purpose if the exit status is non-zero. + + For an example, *misc/validators/validator-jq* command uses *jq* + command as its backend. If *jq* command is not available on a + system, *validator-jq* can do nothing. If such case, + *is_runnable* method of *validator-jq* command should exit with + non-zero value. + +*validate* method + + The make target runs the command with "validate* and an input + file name for validating the input file. The command exits + non-zero if the input file contains invalid syntax. This method + will never run if *is_runnable* method of the command exits with + non-zero. diff --git a/docs/testing.rst b/docs/testing.rst index 0c8df29cf9..544d9d2911 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -21,3 +21,4 @@ ctags in limited resources. tmain.rst tinst.rst cspell.rst + input-validation.rst diff --git a/misc/validators/validator-puppet b/misc/validators/validator-puppet new file mode 100755 index 0000000000..36447391a0 --- /dev/null +++ b/misc/validators/validator-puppet @@ -0,0 +1,60 @@ +# -*- sh -*- +# validator-puppet - validating puppetManifest input files +# +# Copyright (c) 2018, Hakan Baba +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# +# NOTE about puppet version +# ------------------------------------------------------------------------ +# This driver script does not specify the used puppet +# version. Instead, puppet version will depend on the platform +# runtime. As a result, you may see some warnings on some platforms +# even tough the validation has passed. +# +# As of writing this comment, puppet5 is becoming popular. In #1909, +# @ahakanbaba has fixed much of test cases to be valid puppet5 +# files. Even tough that PR exists, we are still sticking with puppet4 +# in our automated tests. One of our test platforms is Fedora28 and +# its distribution has puppet4 at this time. As the note about +# warnings above mentions, in validation runs using puppet4 you may +# see the following Errors prompts that do not fail the validation. +# +# puppet-simpleselector.d/input.pp with puppet valid +# Error: Could not write report for e22e2d5d26ca.ec2.internal at \ +# /var/lib/puppet/reports/e22e2d5d26ca.ec2.internal/201810241144.yaml: \ +# undefined method `split' for :ensure:Symbol +# Error: Could not send report: undefined method `split' for :ensure:Symbol +# ... +# +# So ideally, we should specify the version of puppet used +# in this script. @ahakanbaba shows the way to do so in #1926. +# However, because I (@masatake) don't want to take much time to write +# portable puppet validator, I keep this script as is. +# +action=$1 +input=$2 +case "$action" in + is_runnable) + type puppet > /dev/null 2>&1 + exit $? + ;; + validate) + puppet apply --noop "$input" > /dev/null + exit $? + ;; +esac