From 2462704c5e55fdd0bfa41dfa7dc86db4c6abe934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reigota?= Date: Fri, 5 Nov 2021 17:56:50 +0000 Subject: [PATCH] feat(yaml): Added support to ignore lines by comments to yaml files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: João Reigota --- pkg/model/comment_yaml.go | 170 ++++++++++ pkg/model/comment_yaml_test.go | 557 +++++++++++++++++++++++++++++++++ pkg/model/model_yaml.go | 1 + 3 files changed, 728 insertions(+) create mode 100644 pkg/model/comment_yaml.go create mode 100644 pkg/model/comment_yaml_test.go diff --git a/pkg/model/comment_yaml.go b/pkg/model/comment_yaml.go new file mode 100644 index 00000000000..3fda88e4e3c --- /dev/null +++ b/pkg/model/comment_yaml.go @@ -0,0 +1,170 @@ +package model + +import ( + "strings" + + "gopkg.in/yaml.v3" +) + +// comment is a struct that holds the comment +type comment string + +// Ignore is a struct that holds the lines to ignore +type Ignore struct { + // Lines is the lines to ignore + Lines []int +} + +var ( + // NewIgnore is the ignore struct + NewIgnore = &Ignore{} +) + +// Build builds the ignore struct +func (i *Ignore) Build(lines []int) { + i.Lines = append(i.Lines, lines...) +} + +// GetLines returns the lines to ignore +func (i *Ignore) GetLines() []int { + return removeDuplicates(i.Lines) +} + +// Reset resets the ignore struct +func (i *Ignore) Reset() { + i.Lines = make([]int, 0) +} + +// ignoreCommentsYAML sets the lines to ignore for a yaml file +func ignoreCommentsYAML(node *yaml.Node) { + linesIgnore := make([]int, 0) + if node.HeadComment != "" { + // Squence Node - Head Comment comes in root node + linesIgnore = append(linesIgnore, processCommentYAML((*comment)(&node.HeadComment), 0, node, node.Kind)...) + NewIgnore.Build(linesIgnore) + return + } + // check if comment is in the content + for i, content := range node.Content { + if content.HeadComment == "" { + continue + } + linesIgnore = append(linesIgnore, processCommentYAML((*comment)(&content.HeadComment), i, node, node.Kind)...) + } + + NewIgnore.Build(linesIgnore) +} + +// processCommentYAML returns the lines to ignore +func processCommentYAML(comment *comment, position int, content *yaml.Node, kind yaml.Kind) (linesIgnore []int) { + linesIgnore = make([]int, 0) + switch com := (*comment).value(); com { + case IgnoreLine: + linesIgnore = append(linesIgnore, processLine(kind, content, position)...) + case IgnoreBlock: + linesIgnore = append(linesIgnore, processBlock(kind, content.Content, position)...) + } + + return +} + +// processLine returns the lines to ignore for a line +func processLine(kind yaml.Kind, content *yaml.Node, position int) (linesIgnore []int) { + linesIgnore = make([]int, 0) + var nodeToIgnore *yaml.Node + if kind == yaml.ScalarNode { + nodeToIgnore = content + } else { + nodeToIgnore = content.Content[position] + } + linesIgnore = append(linesIgnore, nodeToIgnore.Line-1, nodeToIgnore.Line) + return +} + +// processBlock returns the lines to ignore for a block +func processBlock(kind yaml.Kind, content []*yaml.Node, position int) (linesIgnore []int) { + linesIgnore = make([]int, 0) + var contentToIgnore []*yaml.Node + if kind == yaml.SequenceNode { + contentToIgnore = content[position].Content + } else if position == 0 { + contentToIgnore = content + } else { + contentToIgnore = content[position+1].Content + } + + linesIgnore = append(linesIgnore, content[position].Line, content[position].Line-1) + linesIgnore = append(linesIgnore, serviceRange(contentToIgnore[0].Line, + getNodeLastLine(contentToIgnore[len(contentToIgnore)-1]))...) + return +} + +// serviceRange returns the range of the services lines +func serviceRange(startLine, endLine int) (lines []int) { + lines = make([]int, 0) + for i := startLine; i <= endLine; i++ { + lines = append(lines, i) + } + + return +} + +// getNodeLastLine returns the last line of a node +func getNodeLastLine(node *yaml.Node) (lastLine int) { + lastLine = node.Line + for _, content := range node.Content { + if content.Line > lastLine { + lastLine = content.Line + } + if lineContent := getNodeLastLine(content); lineContent > lastLine { + lastLine = lineContent + } + } + + return +} + +// removeDuplicates removes duplicates from a slice +func removeDuplicates(linesIgnore []int) (list []int) { + keys := make(map[int]bool) + list = make([]int, 0) + for _, entry := range linesIgnore { + if _, value := keys[entry]; !value { + keys[entry] = true + list = append(list, entry) + } + } + + return +} + +// value returns the value of the comment +func (c *comment) value() (value CommentCommand) { + comment := strings.ToLower(string(*c)) + + // check if we are working with kics command + if KICSCommentRgxp.MatchString(comment) { + comment = KICSCommentRgxp.ReplaceAllString(comment, "") + commands := strings.Split(strings.Trim(comment, "\n"), " ") + value = processCommands(commands) + return + } + + return CommentCommand(comment) +} + +// processCommands goes over kics commands in a line and returns the type of command given +func processCommands(commands []string) CommentCommand { + for _, command := range commands { + switch com := CommentCommand(command); com { + case IgnoreLine: + return IgnoreLine + case IgnoreBlock: + return IgnoreBlock + default: + continue + } + } + + return CommentCommand(commands[0]) +} diff --git a/pkg/model/comment_yaml_test.go b/pkg/model/comment_yaml_test.go new file mode 100644 index 00000000000..311302cd795 --- /dev/null +++ b/pkg/model/comment_yaml_test.go @@ -0,0 +1,557 @@ +package model + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func Test_ignoreCommentsYAML(t *testing.T) { + type args struct { + node *yaml.Node + } + tests := []struct { + name string + args args + want []int + }{ + { + name: "test_1: ignore-block", + want: []int{2, 1, 3, 4, 5, 6, 7, 8, 9}, + args: args{ + &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "key", + Line: 1, + }, + { + Kind: yaml.ScalarNode, + Value: "false", + Tag: "!!bool", + }, + { + Kind: yaml.ScalarNode, + HeadComment: "# kics ignore-block", + Value: "key_object", + Line: 2, + }, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "null_object", + Line: 3, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!null", + }, + { + Kind: yaml.ScalarNode, + Value: "int_object", + Line: 4, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: "24", + }, + { + Kind: yaml.ScalarNode, + Value: "seq_object", + Line: 5, + }, + { + Kind: yaml.SequenceNode, + Content: []*yaml.Node{ + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "key_seq", + Line: 6, + }, + { + Kind: yaml.ScalarNode, + Value: "key_val", + Tag: " !!str", + }, + { + Kind: yaml.ScalarNode, + Value: "key_seq_2", + Line: 7, + }, + { + Kind: yaml.ScalarNode, + Value: "key_val_2", + Tag: " !!str", + }, + }, + }, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "second_key", + Line: 8, + }, + { + Kind: yaml.ScalarNode, + Value: "second_val", + Tag: " !!str", + }, + { + Kind: yaml.ScalarNode, + Value: "second_key_2", + Line: 9, + }, + { + Kind: yaml.ScalarNode, + Value: "second_val_2", + Tag: " !!str", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "test_2: ignore-line", + want: []int{0, 1}, + args: args{ + &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + HeadComment: "# kics ignore-line", + Value: "key", + Line: 1, + }, + { + Kind: yaml.ScalarNode, + Value: "false", + Tag: "!!bool", + }, + { + Kind: yaml.ScalarNode, + Value: "key_object", + Line: 2, + }, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "null_object", + Line: 3, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!null", + }, + { + Kind: yaml.ScalarNode, + Value: "int_object", + Line: 4, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: "24", + }, + { + Kind: yaml.ScalarNode, + Value: "seq_object", + Line: 5, + }, + { + Kind: yaml.SequenceNode, + Content: []*yaml.Node{ + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "key_seq", + Line: 6, + }, + { + Kind: yaml.ScalarNode, + Value: "key_val", + Tag: " !!str", + }, + { + Kind: yaml.ScalarNode, + Value: "key_seq_2", + Line: 7, + }, + { + Kind: yaml.ScalarNode, + Value: "key_val_2", + Tag: " !!str", + }, + }, + }, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "second_key", + Line: 8, + }, + { + Kind: yaml.ScalarNode, + Value: "second_val", + Tag: " !!str", + }, + { + Kind: yaml.ScalarNode, + Value: "second_key_2", + Line: 9, + }, + { + Kind: yaml.ScalarNode, + Value: "second_val_2", + Tag: " !!str", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "test_3: regular-comment", + want: []int{}, + args: args{ + &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + HeadComment: "# kics regular comment", + Value: "key", + Line: 1, + }, + { + Kind: yaml.ScalarNode, + Value: "false", + Tag: "!!bool", + }, + { + Kind: yaml.ScalarNode, + Value: "key_object", + Line: 2, + }, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "null_object", + Line: 3, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!null", + }, + { + Kind: yaml.ScalarNode, + Value: "int_object", + Line: 4, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: "24", + }, + { + Kind: yaml.ScalarNode, + Value: "seq_object", + Line: 5, + }, + { + Kind: yaml.SequenceNode, + Content: []*yaml.Node{ + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "key_seq", + Line: 6, + }, + { + Kind: yaml.ScalarNode, + Value: "key_val", + Tag: " !!str", + }, + { + Kind: yaml.ScalarNode, + Value: "key_seq_2", + Line: 7, + }, + { + Kind: yaml.ScalarNode, + Value: "key_val_2", + Tag: " !!str", + }, + }, + }, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "second_key", + Line: 8, + }, + { + Kind: yaml.ScalarNode, + Value: "second_val", + Tag: " !!str", + }, + { + Kind: yaml.ScalarNode, + Value: "second_key_2", + Line: 9, + }, + { + Kind: yaml.ScalarNode, + Value: "second_val_2", + Tag: " !!str", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "test_4: ignore-all", + want: []int{1, 0, 2, 3, 4, 5, 6, 7, 8, 9}, + args: args{ + &yaml.Node{ + Kind: yaml.MappingNode, + HeadComment: "# kics ignore-block", + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "key", + Line: 1, + }, + { + Kind: yaml.ScalarNode, + Value: "false", + Tag: "!!bool", + }, + { + Kind: yaml.ScalarNode, + Value: "key_object", + Line: 2, + }, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "null_object", + Line: 3, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!null", + }, + { + Kind: yaml.ScalarNode, + Value: "int_object", + Line: 4, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: "24", + }, + { + Kind: yaml.ScalarNode, + Value: "seq_object", + Line: 5, + }, + { + Kind: yaml.SequenceNode, + Content: []*yaml.Node{ + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "key_seq", + Line: 6, + }, + { + Kind: yaml.ScalarNode, + Value: "key_val", + Tag: " !!str", + }, + { + Kind: yaml.ScalarNode, + Value: "key_seq_2", + Line: 7, + }, + { + Kind: yaml.ScalarNode, + Value: "key_val_2", + Tag: " !!str", + }, + }, + }, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "second_key", + Line: 8, + }, + { + Kind: yaml.ScalarNode, + Value: "second_val", + Tag: " !!str", + }, + { + Kind: yaml.ScalarNode, + Value: "second_key_2", + Line: 9, + }, + { + Kind: yaml.ScalarNode, + Value: "second_val_2", + Tag: " !!str", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "test_5: ignore-seq", + want: []int{5, 4, 6, 7, 8, 9}, + args: args{ + &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "key", + Line: 1, + }, + { + Kind: yaml.ScalarNode, + Value: "false", + Tag: "!!bool", + }, + { + Kind: yaml.ScalarNode, + HeadComment: "# kics ignore-block", + Value: "seq_object", + Line: 5, + }, + { + Kind: yaml.SequenceNode, + Line: 6, + Content: []*yaml.Node{ + { + Kind: yaml.MappingNode, + Line: 6, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "key_seq", + Line: 6, + }, + { + Kind: yaml.ScalarNode, + Value: "key_val", + Tag: " !!str", + Line: 6, + }, + { + Kind: yaml.ScalarNode, + Value: "key_seq_2", + Line: 7, + }, + { + Kind: yaml.ScalarNode, + Value: "key_val_2", + Tag: " !!str", + Line: 7, + }, + }, + }, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "second_key", + Line: 8, + }, + { + Kind: yaml.ScalarNode, + Value: "second_val", + Tag: " !!str", + }, + { + Kind: yaml.ScalarNode, + Value: "second_key_2", + Line: 9, + }, + { + Kind: yaml.ScalarNode, + Value: "second_val_2", + Tag: " !!str", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ignoreCommentsYAML(tt.args.node) + require.Equal(t, tt.want, NewIgnore.GetLines()) + NewIgnore.Reset() + }) + } +} diff --git a/pkg/model/model_yaml.go b/pkg/model/model_yaml.go index 3a8a6df7dfa..910dfa37133 100644 --- a/pkg/model/model_yaml.go +++ b/pkg/model/model_yaml.go @@ -36,6 +36,7 @@ func (m *Document) UnmarshalYAML(value *yaml.Node) error { // to place their line information in the payload func unmarshal(val *yaml.Node) interface{} { tmp := make(map[string]interface{}) + ignoreCommentsYAML(val) // if Yaml Node is an Array than we are working with ansible // which need to be placed inside "playbooks"