Merge pull request #1926 from masatake/verify-input-pr

Introduce puppet verifier (derived from #1909)

Units/parser-puppetManifest.r/node.d/expected.tags

@@ -1,3 +1,4 @@
-www1.example.com	input.pp	/^node 'www1.example.com' {$/;"	node	line:2	language:PuppetManifest
-www2.example.com	input.pp	/^node 'www2.example.com', 'www3.example.com' {$/;"	node	line:12	language:PuppetManifest
-www3.example.com	input.pp	/^node 'www2.example.com', 'www3.example.com' {$/;"	node	line:12	language:PuppetManifest
+default	input.pp	/^node 'default' {}$/;"	node	line:2	language:PuppetManifest
+www1.example.com	input.pp	/^node 'www1.example.com' {$/;"	node	line:3	language:PuppetManifest
+www2.example.com	input.pp	/^node 'www2.example.com', 'www3.example.com' {$/;"	node	line:13	language:PuppetManifest
+www3.example.com	input.pp	/^node 'www2.example.com', 'www3.example.com' {$/;"	node	line:13	language:PuppetManifest

Units/parser-puppetManifest.r/node.d/input.pp

@@ -1,4 +1,5 @@
 /* Taken from https://docs.puppet.com/puppet/5.1/lang_node_definitions.html */
+node 'default' {}
 node 'www1.example.com' {
   include common
   include apache

Units/parser-puppetManifest.r/puppet-append.d/validator

@@ -0,0 +1,12 @@
+#
+# The puppet validator reports this input as "invalid" because
+# += operator is obsoleted in puppet5.
+# @ahakanbaba in #1909 reported and proposed a fix for this already.
+#
+# However, @masatake kept this as is for following reasons:
+#
+# - valid in puppet4,
+# - testing KNOWN-INVALIDATION special validator, and
+# - testing '#' comment in a validator file.
+#
+KNOWN-INVALIDATION

Units/parser-puppetManifest.r/puppet-collection_within_virtual_definitions.d/expected.tags

@@ -1,5 +1,5 @@
-test	input.pp	/^define test($name) {$/;"	definition	line:1	language:PuppetManifest	end:6
-/tmp/collection_within_virtual_definitions1_$name.txt	input.pp	/^    file {"\/tmp\/collection_within_virtual_definitions1_$name.txt":$/;"	resource	line:2	language:PuppetManifest	scope:definition:test	end:4
+test	input.pp	/^define test($my_name) {$/;"	definition	line:1	language:PuppetManifest	end:6
+/tmp/collection_within_virtual_definitions1_$my_name.txt	input.pp	/^    file {"\/tmp\/collection_within_virtual_definitions1_$my_name.txt":$/;"	resource	line:2	language:PuppetManifest	scope:definition:test	end:4
 test2	input.pp	/^define test2() {$/;"	definition	line:8	language:PuppetManifest	end:12
 /tmp/collection_within_virtual_definitions2_$name.txt	input.pp	/^    file {"\/tmp\/collection_within_virtual_definitions2_$name.txt":$/;"	resource	line:9	language:PuppetManifest	scope:definition:test2	end:11
 foo	input.pp	/^    @test {"foo":$/;"	resource	line:15	language:PuppetManifest	end:17

Units/parser-puppetManifest.r/puppet-collection_within_virtual_definitions.d/input.pp

@@ -1,6 +1,6 @@
-define test($name) {
-    file {"/tmp/collection_within_virtual_definitions1_$name.txt":
-        content => "File name $name\n"
+define test($my_name) {
+    file {"/tmp/collection_within_virtual_definitions1_$my_name.txt":
+        content => "File name $my_name\n"
     }
     Test2 <||>
 }
@@ -13,7 +13,7 @@
 
 node default {
     @test {"foo":
-        name => "foo"
+        my_name => "foo"
     }
     @test2 {"foo2": }
     Test <||>

Units/parser-puppetManifest.r/puppet-emptyif.d/args.ctags

@@ -0,0 +1,2 @@
+--sort=no
+--fields=+KZlne

Units/parser-puppetManifest.r/puppet-emptyif.d/input.pp

@@ -0,0 +1,4 @@
+
+if true {
+  # still nothing
+}

Units/parser-puppetManifest.r/puppet-emptyifelse.d/input.pp

@@ -4,6 +4,3 @@
   # nothing here
 }
 
-if true {
-  # still nothing
-}

Units/parser-puppetManifest.r/puppet-multipleclass.d/expected.tags

@@ -1,4 +1,4 @@
 one	input.pp	/^class one {$/;"	class	line:1	language:PuppetManifest	end:3
 /tmp/multipleclassone	input.pp	/^    file { "\/tmp\/multipleclassone": content => "one" }$/;"	resource	line:2	language:PuppetManifest	scope:class:one	end:2
-one	input.pp	/^class one {$/;"	class	line:5	language:PuppetManifest	end:7
-/tmp/multipleclasstwo	input.pp	/^    file { "\/tmp\/multipleclasstwo": content => "two" }$/;"	resource	line:6	language:PuppetManifest	scope:class:one	end:6
+two	input.pp	/^class two {$/;"	class	line:5	language:PuppetManifest	end:7
+/tmp/multipleclasstwo	input.pp	/^    file { "\/tmp\/multipleclasstwo": content => "two" }$/;"	resource	line:6	language:PuppetManifest	scope:class:two	end:6

Units/parser-puppetManifest.r/puppet-multipleclass.d/input.pp

@@ -2,7 +2,7 @@
     file { "/tmp/multipleclassone": content => "one" }
 }
 
-class one {
+class two {
     file { "/tmp/multipleclasstwo": content => "two" }
 }
 

Units/parser-puppetManifest.r/puppet-scopetest.d/expected.tags

@@ -1,5 +1,5 @@
-mode	input.pp	/^$mode = 640$/;"	variable	line:2	language:PuppetManifest
+mode	input.pp	/^$mode = "640"$/;"	variable	line:2	language:PuppetManifest
 thing	input.pp	/^define thing {$/;"	definition	line:4	language:PuppetManifest	end:6
 /tmp/$name	input.pp	/^    file { "\/tmp\/$name": ensure => file, mode => $mode }$/;"	resource	line:5	language:PuppetManifest	scope:definition:thing	end:5
 testing	input.pp	/^class testing {$/;"	class	line:8	language:PuppetManifest	end:11
-mode	input.pp	/^    $mode = 755$/;"	variable	line:9	language:PuppetManifest	scope:class:testing
+mode	input.pp	/^    $mode = "755"$/;"	variable	line:9	language:PuppetManifest	scope:class:testing

Units/parser-puppetManifest.r/puppet-scopetest.d/input.pp

@@ -1,12 +1,12 @@
 
-$mode = 640
+$mode = "640"
 
 define thing {
     file { "/tmp/$name": ensure => file, mode => $mode }
 }
 
 class testing {
-    $mode = 755
+    $mode = "755"
     thing {scopetest: }
 }
 

Units/parser-puppetManifest.r/puppet-selectorvalues.d/input.pp

@@ -6,37 +6,37 @@
 $test = "yay"
 
 $mode1 = $value1 ? {
-    "" => 755,
-    default => 644
+    "" => "755",
+    default => "644"
 }
 
 $mode2 = $value2 ? {
-    true => 755,
-    default => 644
+    true => "755",
+    default => "644"
 }
 
 $mode3 = $value3 ? {
-    false => 755,
-    default => 644
+    false => "755",
+    default => "644"
 }
 
 $mode4 = $value4 ? {
-    $test => 755,
-    default => 644
+    $test => "755",
+    default => "644"
 }
 
 $mode5 = yay ? {
-    $test => 755,
-    default => 644
+    $test => "755",
+    default => "644"
 }
 
 $mode6 = $mode5 ? {
-    755 => 755
+    "755" => "755"
 }
 
 $mode7 = "test regex" ? {
-    /regex$/ => 755,
-    default => 644
+    /regex$/ => "755",
+    default => "644"
 }
 
 

Units/parser-puppetManifest.r/puppet-simpleselector.d/expected.tags

@@ -3,5 +3,5 @@ var	input.pp	/^$var = "value"$/;"	variable	line:3	language:PuppetManifest
 /tmp/snippetselectbtest	input.pp	/^file { "\/tmp\/snippetselectbtest":$/;"	resource	line:13	language:PuppetManifest	end:19
 othervar	input.pp	/^$othervar = "complex value"$/;"	variable	line:21	language:PuppetManifest
 /tmp/snippetselectctest	input.pp	/^file { "\/tmp\/snippetselectctest":$/;"	resource	line:23	language:PuppetManifest	end:29
-anothervar	input.pp	/^$anothervar = Yayness$/;"	variable	line:30	language:PuppetManifest
+anothervar	input.pp	/^$anothervar = "Yayness"$/;"	variable	line:30	language:PuppetManifest
 /tmp/snippetselectdtest	input.pp	/^file { "\/tmp\/snippetselectdtest":$/;"	resource	line:32	language:PuppetManifest	end:38

Units/parser-puppetManifest.r/puppet-simpleselector.d/input.pp

@@ -5,16 +5,16 @@
 file { "/tmp/snippetselectatest":
     ensure => file,
     mode => $var ? {
-        nottrue => 641,
-        value => 755
+        nottrue => "641",
+        value => "755"
     }
 }
 
 file { "/tmp/snippetselectbtest":
     ensure => file,
     mode => $var ? {
-        nottrue => 644,
-        default => 755
+        nottrue => "644",
+        default => "755"
     }
 }
 
@@ -23,16 +23,16 @@
 file { "/tmp/snippetselectctest":
     ensure => file,
     mode => $othervar ? {
-        "complex value" => 755,
-        default => 644
+        "complex value" => "755",
+        default => "644"
     }
 }
-$anothervar = Yayness
+$anothervar = "Yayness"
 
 file { "/tmp/snippetselectdtest":
     ensure => file,
     mode => $anothervar ? {
-        Yayness => 755,
-        default => 644
+        "Yayness" => "755",
+        default => "644"
     }
 }

Units/parser-puppetManifest.r/puppet-singleselector.d/input.pp

@@ -6,15 +6,15 @@
 $test = "yay"
 
 $mode1 = $value1 ? {
-    "" => 755
+    "" => "755"
 }
 
 $mode2 = $value2 ? {
-    true => 755
+    true => "755"
 }
 
 $mode3 = $value3 ? {
-    default => 755
+    default => "755"
 }
 
 file { "/tmp/singleselector1": ensure => file, mode => $mode1 }

Units/parser-puppetManifest.r/unless.d/expected.tags

@@ -1 +1,2 @@
-/tmp/x	input.pp	/^       file { "\/tmp\/x": }$/;"	resource	line:2	language:PuppetManifest	end:2
+array	input.pp	/^$array = [ 3, 5, 7 ]$/;"	variable	line:1	language:PuppetManifest
+/tmp/x	input.pp	/^       file { "\/tmp\/x": }$/;"	resource	line:3	language:PuppetManifest	end:3

Units/parser-puppetManifest.r/unless.d/input.pp

@@ -1,3 +1,4 @@
+$array = [ 3, 5, 7 ]
 unless $array[0] > 5 {
        file { "/tmp/x": }
 }

Units/parser-puppetManifest.r/validator

@@ -0,0 +1 @@
+puppet

circle.yml

@@ -14,7 +14,7 @@ jobs:
            name: Install build tools
            command: |
              dnf -y install gcc automake autoconf pkgconfig bmake aspell-devel aspell-en libxml2-devel jansson-devel libyaml-devel findutils || :
-             dnf -y install jq
+             dnf -y install jq puppet || :
        - run:
            name: Build
            command: |

docs/input-validation.rst

@@ -0,0 +1,129 @@
+Input validation for *Units*
+---------------------------------------------------------------------
+
+:Maintainer: Masatake YAMATO <yamato@redhat.com>
+
+----
+
+We have to maintain parsers for languages that we don't know well.  We
+don't have enough time to learn the languages.
+
+*Units* test cases help us not introduce wrong changes to a parser.
+
+However, there is still an issue; a developer who doesn't know a
+target language well may write a broken test input file for the
+language.  Here comes "Input validation."
+
+You can validate the test input files of *Units* with *validate-input*
+make target if a validator for a language is defined.
+
+How to run and an example session
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Here is an example validating an input file for JSON.
+
+.. code-block:: console
+
+  $ make validate-input VALIDATORS=jq
+  ...
+  Category: ROOT
+  ------------------------------------------------------------
+  simple-json.d/input.json with jq                                 valid
+
+  Summary
+  ------------------------------------------------------------
+    #valid:                                 1
+    #invalid:                               0
+    #skipped (known invalidation)           0
+    #skipped (validator unavailable)        0
+
+
+This example shows validating *simple-json.d/input.json* as an input
+file with *jq* validator. With VALIDATORS variable passed via
+command-line, you can specify validators to run. Multiple validators
+can be specified using a comma-separated list.  If you don't give
+VALIDATORS, the make target tries to use all available validators.
+
+The meanings of "valid" and "invalid" in "Summary" are apparent.  In
+two cases, the target skips validating input files:
+
+#skipped (known invalidation)
+
+    A test case specifies KNOWN-INVALIDATION in its *validator* file.
+
+#skipped (validator unavailable)
+
+    A command for a validator is not available.
+
+*validator* file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*validator* file in a *Units* test directory specifies which
+validator the make target should use.
+
+.. code-block:: console
+
+  $ cat Units/simple-json.d/validator
+  jq
+
+If you put *validator* file to a category directory (a directory
+having *.r* suffix), the make target uses the validator specified in
+the file as default.  The default validator can be overridden with a
+*validator* file in a subdirectory.
+
+.. code-block:: console
+
+  $ cat Units/parser-puppetManifest.r/validator
+  puppet
+  # cat Units/parser-puppetManifest.r/puppet-append.d/validator
+  KNOWN-INVALIDATION
+
+In the example, the make target uses *puppet* validator for validating
+the most of all input files under *Units/parser-puppetManifest.r*
+directory. An exception is an input file under
+*Units/parser-puppetManifest.r/puppet-append.d* directory.  The
+directory has its specific *validator* file.
+
+If a *Unit* test case doesn't have *expected.tags* file, the make
+target doesn't run the validator on the file even if a default
+validator is given in its category directory.
+
+If a *Unit* test case specifies KNOWN-INVALIDATION in its *validator*
+file, the make target just increments "#skipped (known invalidation)"
+counter. The target reports the counter at the end of execution.
+
+validator command
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A validator specified in a *validator* file is a command file put
+under *misc/validators* directory.  The command must have "validator-"
+as prefix in its file name. For an example,
+*misc/validators/validator-jq* is the command for "jq".
+
+The command file must be an executable. *validate-input* make target
+runs the command in two ways.
+
+*is_runnable* method
+
+    Before running the command as a validator, the target runs
+    the command with "is_runnable" as the first argument.
+    A validator command can let the target know whether the
+    validator command is runnable or not with exit status.
+    0 means ready to run. Non-zero means not ready to run.
+
+    The make target never runs the validator command for
+    validation purpose if the exit status is non-zero.
+
+    For an example, *misc/validators/validator-jq* command uses *jq*
+    command as its backend. If *jq* command is not available on a
+    system, *validator-jq* can do nothing.  If such case,
+    *is_runnable* method of *validator-jq* command should exit with
+    non-zero value.
+
+*validate* method
+
+    The make target runs the command with "validate* and an input
+    file name for validating the input file.  The command exits
+    non-zero if the input file contains invalid syntax. This method
+	will never run if *is_runnable* method of the command exits with
+	non-zero.

docs/testing.rst

@@ -21,3 +21,4 @@ ctags in limited resources.
 	tmain.rst
 	tinst.rst
 	cspell.rst
+	input-validation.rst