From c4010d5fdbe189ea076e4af9674480785d1168b9 Mon Sep 17 00:00:00 2001 From: Jeroen Bobbeldijk Date: Tue, 24 May 2022 09:43:53 +0200 Subject: [PATCH 1/2] Allow file patterns for all analyzers --- analyzer/all/import.go | 1 + analyzer/analyzer.go | 47 ++++++++++++++-- analyzer/analyzer_test.go | 62 +++++++++++++++++++-- analyzer/config/all/import.go | 8 +++ analyzer/config/config.go | 47 ---------------- analyzer/config/config_test.go | 35 +++++------- analyzer/config/dockerfile/docker.go | 27 +++------ analyzer/config/dockerfile/docker_test.go | 23 +++----- analyzer/config/json/json.go | 27 +++------ analyzer/config/json/json_test.go | 23 +++----- analyzer/config/terraform/terraform.go | 19 +++---- analyzer/config/terraform/terraform_test.go | 7 +-- analyzer/config/yaml/yaml.go | 27 +++------ analyzer/config/yaml/yaml_test.go | 23 +++----- artifact/artifact.go | 2 + artifact/image/image.go | 14 ++--- artifact/image/image_test.go | 1 + artifact/local/fs.go | 13 ++--- artifact/local/fs_test.go | 2 + artifact/remote/git_test.go | 1 + cache/key.go | 5 +- cache/key_test.go | 18 +++--- external/config_scan.go | 2 + go.sum | 4 -- 24 files changed, 216 insertions(+), 222 deletions(-) create mode 100644 analyzer/config/all/import.go diff --git a/analyzer/all/import.go b/analyzer/all/import.go index 14ec251ab..d94d2bfbc 100644 --- a/analyzer/all/import.go +++ b/analyzer/all/import.go @@ -3,6 +3,7 @@ package all import ( _ "github.com/aquasecurity/fanal/analyzer/buildinfo" _ "github.com/aquasecurity/fanal/analyzer/command/apk" + _ "github.com/aquasecurity/fanal/analyzer/config/all" _ "github.com/aquasecurity/fanal/analyzer/language/dotnet/nuget" _ "github.com/aquasecurity/fanal/analyzer/language/golang/binary" _ "github.com/aquasecurity/fanal/analyzer/language/golang/mod" diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go index 7107335a1..ec4896fa7 100644 --- a/analyzer/analyzer.go +++ b/analyzer/analyzer.go @@ -5,6 +5,7 @@ import ( "errors" "io/fs" "os" + "regexp" "sort" "strings" "sync" @@ -81,6 +82,7 @@ type Opener func() (dio.ReadSeekCloserAt, error) type AnalyzerGroup struct { analyzers []analyzer configAnalyzers []configAnalyzer + filePatterns map[Type][]*regexp.Regexp } type AnalysisResult struct { @@ -233,12 +235,37 @@ func belongToGroup(groupName Group, analyzerType Type, disabledAnalyzers []Type, return true } -func NewAnalyzerGroup(groupName Group, disabledAnalyzers []Type) AnalyzerGroup { +const separator = ":" + +func NewAnalyzerGroup(groupName Group, disabledAnalyzers []Type, filePatterns []string) (AnalyzerGroup, error) { if groupName == "" { groupName = GroupBuiltin } - var group AnalyzerGroup + group := AnalyzerGroup{ + filePatterns: map[Type][]*regexp.Regexp{}, + } + + for _, p := range filePatterns { + // e.g. "dockerfile:my_dockerfile_*" + s := strings.SplitN(p, separator, 2) + if len(s) != 2 { + return group, xerrors.Errorf("invalid file pattern (%s)", p) + } + + fileType, pattern := s[0], s[1] + r, err := regexp.Compile(pattern) + if err != nil { + return group, xerrors.Errorf("invalid file regexp (%s): %w", p, err) + } + + if _, ok := group.filePatterns[Type(fileType)]; !ok { + group.filePatterns[Type(fileType)] = []*regexp.Regexp{} + } + + group.filePatterns[Type(fileType)] = append(group.filePatterns[Type(fileType)], r) + } + for analyzerType, a := range analyzers { if !belongToGroup(groupName, analyzerType, disabledAnalyzers, a) { continue @@ -253,7 +280,7 @@ func NewAnalyzerGroup(groupName Group, disabledAnalyzers []Type) AnalyzerGroup { group.configAnalyzers = append(group.configAnalyzers, a) } - return group + return group, nil } // AnalyzerVersions returns analyzer version identifier used for cache keys. @@ -280,14 +307,24 @@ func (ag AnalyzerGroup) AnalyzeFile(ctx context.Context, wg *sync.WaitGroup, lim return nil } + // filepath extracted from tar file doesn't have the prefix "/" + cleanPath := strings.TrimLeft(filePath, "/") + for _, a := range ag.analyzers { // Skip disabled analyzers if slices.Contains(disabled, a.Type()) { continue } - // filepath extracted from tar file doesn't have the prefix "/" - if !a.Required(strings.TrimLeft(filePath, "/"), info) { + filePatternMatch := false + for _, pattern := range ag.filePatterns[a.Type()] { + if pattern.MatchString(cleanPath) { + filePatternMatch = true + break + } + } + + if !filePatternMatch && !a.Required(cleanPath, info) { continue } rc, err := opener() diff --git a/analyzer/analyzer_test.go b/analyzer/analyzer_test.go index 9e46aa8a4..2ef054537 100644 --- a/analyzer/analyzer_test.go +++ b/analyzer/analyzer_test.go @@ -281,6 +281,7 @@ func TestAnalyzeFile(t *testing.T) { filePath string testFilePath string disabledAnalyzers []analyzer.Type + filePatterns []string } tests := []struct { name string @@ -377,6 +378,28 @@ func TestAnalyzeFile(t *testing.T) { }, want: &analyzer.AnalysisResult{}, }, + { + name: "happy path with library analyzer file pattern regex", + args: args{ + filePath: "/app/Gemfile-dev.lock", + testFilePath: "testdata/app/Gemfile.lock", + filePatterns: []string{"bundler:Gemfile(-.*)?\\.lock"}, + }, + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: "bundler", + FilePath: "/app/Gemfile-dev.lock", + Libraries: []types.Package{ + { + Name: "actioncable", + Version: "5.2.3", + }, + }, + }, + }, + }, + }, { name: "ignore permission error", args: args{ @@ -393,6 +416,24 @@ func TestAnalyzeFile(t *testing.T) { }, wantErr: "unable to open /lib/apk/db/installed", }, + { + name: "sad path with broken file pattern regex", + args: args{ + filePath: "/app/Gemfile-dev.lock", + testFilePath: "testdata/app/Gemfile.lock", + filePatterns: []string{"bundler:Gemfile(-.*?\\.lock"}, + }, + wantErr: "error parsing regexp", + }, + { + name: "sad path with broken file pattern", + args: args{ + filePath: "/app/Gemfile-dev.lock", + testFilePath: "testdata/app/Gemfile.lock", + filePatterns: []string{"Gemfile(-.*)?\\.lock"}, + }, + wantErr: "invalid file pattern", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -400,7 +441,13 @@ func TestAnalyzeFile(t *testing.T) { limit := semaphore.NewWeighted(3) got := new(analyzer.AnalysisResult) - a := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.args.disabledAnalyzers) + a, err := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.args.disabledAnalyzers, tt.args.filePatterns) + if err != nil && tt.wantErr != "" { + require.NotNil(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) info, err := os.Stat(tt.args.testFilePath) require.NoError(t, err) @@ -440,6 +487,7 @@ func TestAnalyzeConfig(t *testing.T) { targetOS types.OS configBlob []byte disabledAnalyzers []analyzer.Type + filePatterns []string } tests := []struct { name string @@ -482,7 +530,9 @@ func TestAnalyzeConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - a := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.args.disabledAnalyzers) + a, err := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.args.disabledAnalyzers, tt.args.filePatterns) + require.NoError(t, err) + got := a.AnalyzeImageConfig(tt.args.targetOS, tt.args.configBlob) assert.Equal(t, tt.want, got) }) @@ -517,7 +567,9 @@ func TestAnalyzer_AnalyzerVersions(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - a := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.disabled) + a, err := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.disabled, nil) + require.NoError(t, err) + got := a.AnalyzerVersions() fmt.Printf("%v\n", got) assert.Equal(t, tt.want, got) @@ -549,7 +601,9 @@ func TestAnalyzer_ImageConfigAnalyzerVersions(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - a := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.disabled) + a, err := analyzer.NewAnalyzerGroup(analyzer.GroupBuiltin, tt.disabled, nil) + require.NoError(t, err) + got := a.ImageConfigAnalyzerVersions() assert.Equal(t, tt.want, got) }) diff --git a/analyzer/config/all/import.go b/analyzer/config/all/import.go new file mode 100644 index 000000000..0da76345c --- /dev/null +++ b/analyzer/config/all/import.go @@ -0,0 +1,8 @@ +package all + +import ( + _ "github.com/aquasecurity/fanal/analyzer/config/dockerfile" + _ "github.com/aquasecurity/fanal/analyzer/config/json" + _ "github.com/aquasecurity/fanal/analyzer/config/terraform" + _ "github.com/aquasecurity/fanal/analyzer/config/yaml" +) diff --git a/analyzer/config/config.go b/analyzer/config/config.go index 767561b82..74dc8bd49 100644 --- a/analyzer/config/config.go +++ b/analyzer/config/config.go @@ -1,22 +1,9 @@ package config import ( - "regexp" "sort" - "strings" - - "golang.org/x/xerrors" - - "github.com/aquasecurity/fanal/analyzer" - "github.com/aquasecurity/fanal/analyzer/config/dockerfile" - "github.com/aquasecurity/fanal/analyzer/config/json" - "github.com/aquasecurity/fanal/analyzer/config/terraform" - "github.com/aquasecurity/fanal/analyzer/config/yaml" - "github.com/aquasecurity/fanal/types" ) -const separator = ":" - type ScannerOption struct { Trace bool RegoOnly bool @@ -33,37 +20,3 @@ func (o *ScannerOption) Sort() { sort.Strings(o.PolicyPaths) sort.Strings(o.DataPaths) } - -func RegisterConfigAnalyzers(filePatterns []string) error { - var dockerRegexp, jsonRegexp, yamlRegexp *regexp.Regexp - for _, p := range filePatterns { - // e.g. "dockerfile:my_dockerfile_*" - s := strings.SplitN(p, separator, 2) - if len(s) != 2 { - return xerrors.Errorf("invalid file pattern (%s)", p) - } - fileType, pattern := s[0], s[1] - r, err := regexp.Compile(pattern) - if err != nil { - return xerrors.Errorf("invalid file regexp (%s): %w", p, err) - } - - switch fileType { - case types.Dockerfile: - dockerRegexp = r - case types.JSON: - jsonRegexp = r - case types.YAML: - yamlRegexp = r - default: - return xerrors.Errorf("unknown file type: %s, pattern: %s", fileType, pattern) - } - } - - analyzer.RegisterAnalyzer(dockerfile.NewConfigAnalyzer(dockerRegexp)) - analyzer.RegisterAnalyzer(terraform.NewConfigAnalyzer()) - analyzer.RegisterAnalyzer(json.NewConfigAnalyzer(jsonRegexp)) - analyzer.RegisterAnalyzer(yaml.NewConfigAnalyzer(yamlRegexp)) - - return nil -} diff --git a/analyzer/config/config_test.go b/analyzer/config/config_test.go index 1abc388a5..9b818c59a 100644 --- a/analyzer/config/config_test.go +++ b/analyzer/config/config_test.go @@ -10,10 +10,9 @@ import ( func TestScannerOption_Sort(t *testing.T) { type fields struct { - Namespaces []string - FilePatterns []string - PolicyPaths []string - DataPaths []string + Namespaces []string + PolicyPaths []string + DataPaths []string } tests := []struct { name string @@ -23,25 +22,22 @@ func TestScannerOption_Sort(t *testing.T) { { name: "happy path", fields: fields{ - Namespaces: []string{"main", "custom", "default"}, - FilePatterns: []string{"dockerfile:foo*", "yaml:yml_*"}, - PolicyPaths: []string{"policy"}, - DataPaths: []string{"data/b", "data/c", "data/a"}, + Namespaces: []string{"main", "custom", "default"}, + PolicyPaths: []string{"policy"}, + DataPaths: []string{"data/b", "data/c", "data/a"}, }, want: config.ScannerOption{ - Namespaces: []string{"custom", "default", "main"}, - FilePatterns: []string{"dockerfile:foo*", "yaml:yml_*"}, - PolicyPaths: []string{"policy"}, - DataPaths: []string{"data/a", "data/b", "data/c"}, + Namespaces: []string{"custom", "default", "main"}, + PolicyPaths: []string{"policy"}, + DataPaths: []string{"data/a", "data/b", "data/c"}, }, }, { name: "missing some fields", fields: fields{ - Namespaces: []string{"main"}, - FilePatterns: nil, - PolicyPaths: nil, - DataPaths: nil, + Namespaces: []string{"main"}, + PolicyPaths: nil, + DataPaths: nil, }, want: config.ScannerOption{ Namespaces: []string{"main"}, @@ -51,10 +47,9 @@ func TestScannerOption_Sort(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { o := config.ScannerOption{ - Namespaces: tt.fields.Namespaces, - FilePatterns: tt.fields.FilePatterns, - PolicyPaths: tt.fields.PolicyPaths, - DataPaths: tt.fields.DataPaths, + Namespaces: tt.fields.Namespaces, + PolicyPaths: tt.fields.PolicyPaths, + DataPaths: tt.fields.DataPaths, } o.Sort() diff --git a/analyzer/config/dockerfile/docker.go b/analyzer/config/dockerfile/docker.go index 372f02d15..f8bb6c74a 100644 --- a/analyzer/config/dockerfile/docker.go +++ b/analyzer/config/dockerfile/docker.go @@ -5,7 +5,6 @@ import ( "io" "os" "path/filepath" - "regexp" "strings" "golang.org/x/xerrors" @@ -14,21 +13,17 @@ import ( "github.com/aquasecurity/fanal/types" ) +func init() { + analyzer.RegisterAnalyzer(&dockerConfigAnalyzer{}) +} + const version = 1 var requiredFiles = []string{"Dockerfile", "Containerfile"} -type ConfigAnalyzer struct { - filePattern *regexp.Regexp -} - -func NewConfigAnalyzer(filePattern *regexp.Regexp) ConfigAnalyzer { - return ConfigAnalyzer{ - filePattern: filePattern, - } -} +type dockerConfigAnalyzer struct{} -func (s ConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { +func (s dockerConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { b, err := io.ReadAll(input.Content) if err != nil { return nil, xerrors.Errorf("failed to read %s: %w", input.FilePath, err) @@ -50,11 +45,7 @@ func (s ConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) // Required does a case-insensitive check for filePath and returns true if // filePath equals/startsWith/hasExtension requiredFiles -func (s ConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { - if s.filePattern != nil && s.filePattern.MatchString(filePath) { - return true - } - +func (s dockerConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { base := filepath.Base(filePath) ext := filepath.Ext(base) for _, file := range requiredFiles { @@ -69,10 +60,10 @@ func (s ConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { return false } -func (s ConfigAnalyzer) Type() analyzer.Type { +func (s dockerConfigAnalyzer) Type() analyzer.Type { return analyzer.TypeDockerfile } -func (s ConfigAnalyzer) Version() int { +func (s dockerConfigAnalyzer) Version() int { return version } diff --git a/analyzer/config/dockerfile/docker_test.go b/analyzer/config/dockerfile/docker_test.go index 4e0cbd64d..7c2d056b3 100644 --- a/analyzer/config/dockerfile/docker_test.go +++ b/analyzer/config/dockerfile/docker_test.go @@ -1,16 +1,14 @@ -package dockerfile_test +package dockerfile import ( "context" "os" - "regexp" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/aquasecurity/fanal/analyzer" - "github.com/aquasecurity/fanal/analyzer/config/dockerfile" "github.com/aquasecurity/fanal/types" ) @@ -68,7 +66,7 @@ COPY --from=build /bar /bar require.NoError(t, err) defer f.Close() - a := dockerfile.NewConfigAnalyzer(nil) + a := dockerConfigAnalyzer{} ctx := context.Background() got, err := a.Analyze(ctx, analyzer.AnalysisInput{ FilePath: tt.inputFile, @@ -88,10 +86,9 @@ COPY --from=build /bar /bar func Test_dockerConfigAnalyzer_Required(t *testing.T) { tests := []struct { - name string - filePattern *regexp.Regexp - filePath string - want bool + name string + filePath string + want bool }{ { name: "dockerfile", @@ -143,16 +140,10 @@ func Test_dockerConfigAnalyzer_Required(t *testing.T) { filePath: "deployment.json", want: false, }, - { - name: "file pattern", - filePattern: regexp.MustCompile(`foo*`), - filePath: "foo_file", - want: true, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := dockerfile.NewConfigAnalyzer(tt.filePattern) + s := dockerConfigAnalyzer{} got := s.Required(tt.filePath, nil) assert.Equal(t, tt.want, got) }) @@ -160,7 +151,7 @@ func Test_dockerConfigAnalyzer_Required(t *testing.T) { } func Test_dockerConfigAnalyzer_Type(t *testing.T) { - s := dockerfile.NewConfigAnalyzer(nil) + s := dockerConfigAnalyzer{} want := analyzer.TypeDockerfile got := s.Type() assert.Equal(t, want, got) diff --git a/analyzer/config/json/json.go b/analyzer/config/json/json.go index 4d3d9d120..aefabe642 100644 --- a/analyzer/config/json/json.go +++ b/analyzer/config/json/json.go @@ -5,7 +5,6 @@ import ( "io" "os" "path/filepath" - "regexp" "golang.org/x/xerrors" @@ -13,6 +12,10 @@ import ( "github.com/aquasecurity/fanal/types" ) +func init() { + analyzer.RegisterAnalyzer(&jsonConfigAnalyzer{}) +} + const version = 1 var ( @@ -20,17 +23,9 @@ var ( excludedFiles = []string{types.NpmPkgLock, types.NuGetPkgsLock, types.NuGetPkgsConfig} ) -type ConfigAnalyzer struct { - filePattern *regexp.Regexp -} - -func NewConfigAnalyzer(filePattern *regexp.Regexp) ConfigAnalyzer { - return ConfigAnalyzer{ - filePattern: filePattern, - } -} +type jsonConfigAnalyzer struct{} -func (a ConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { +func (a jsonConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { b, err := io.ReadAll(input.Content) if err != nil { return nil, xerrors.Errorf("failed to read %s: %w", input.FilePath, err) @@ -50,11 +45,7 @@ func (a ConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) }, nil } -func (a ConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { - if a.filePattern != nil && a.filePattern.MatchString(filePath) { - return true - } - +func (a jsonConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { filename := filepath.Base(filePath) for _, excludedFile := range excludedFiles { if filename == excludedFile { @@ -65,10 +56,10 @@ func (a ConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { return filepath.Ext(filePath) == requiredExt } -func (ConfigAnalyzer) Type() analyzer.Type { +func (jsonConfigAnalyzer) Type() analyzer.Type { return analyzer.TypeJSON } -func (ConfigAnalyzer) Version() int { +func (jsonConfigAnalyzer) Version() int { return version } diff --git a/analyzer/config/json/json_test.go b/analyzer/config/json/json_test.go index 4ed079285..14eacc599 100644 --- a/analyzer/config/json/json_test.go +++ b/analyzer/config/json/json_test.go @@ -1,16 +1,14 @@ -package json_test +package json import ( "context" "os" - "regexp" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/aquasecurity/fanal/analyzer" - "github.com/aquasecurity/fanal/analyzer/config/json" "github.com/aquasecurity/fanal/types" ) @@ -133,7 +131,7 @@ func Test_jsonConfigAnalyzer_Analyze(t *testing.T) { require.NoError(t, err) defer f.Close() - s := json.NewConfigAnalyzer(nil) + s := &jsonConfigAnalyzer{} ctx := context.Background() got, err := s.Analyze(ctx, analyzer.AnalysisInput{ @@ -154,10 +152,9 @@ func Test_jsonConfigAnalyzer_Analyze(t *testing.T) { func Test_jsonConfigAnalyzer_Required(t *testing.T) { tests := []struct { - name string - filePattern *regexp.Regexp - filePath string - want bool + name string + filePath string + want bool }{ { name: "json", @@ -174,16 +171,10 @@ func Test_jsonConfigAnalyzer_Required(t *testing.T) { filePath: "package-lock.json", want: false, }, - { - name: "file pattern", - filePattern: regexp.MustCompile(`foo*`), - filePath: "foo_file", - want: true, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := json.NewConfigAnalyzer(tt.filePattern) + s := &jsonConfigAnalyzer{} got := s.Required(tt.filePath, nil) assert.Equal(t, tt.want, got) @@ -192,7 +183,7 @@ func Test_jsonConfigAnalyzer_Required(t *testing.T) { } func Test_jsonConfigAnalyzer_Type(t *testing.T) { - s := json.NewConfigAnalyzer(nil) + s := &jsonConfigAnalyzer{} want := analyzer.TypeJSON got := s.Type() diff --git a/analyzer/config/terraform/terraform.go b/analyzer/config/terraform/terraform.go index bb08786bc..e5015d328 100644 --- a/analyzer/config/terraform/terraform.go +++ b/analyzer/config/terraform/terraform.go @@ -13,19 +13,18 @@ import ( "github.com/aquasecurity/fanal/types" ) +func init() { + analyzer.RegisterAnalyzer(&terraformConfigAnalyzer{}) +} + const version = 1 var requiredExts = []string{".tf", ".tf.json"} -type ConfigAnalyzer struct { -} - -func NewConfigAnalyzer() ConfigAnalyzer { - return ConfigAnalyzer{} -} +type terraformConfigAnalyzer struct{} // Analyze returns a name of Terraform file -func (a ConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { +func (a terraformConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { b, err := io.ReadAll(input.Content) if err != nil { return nil, xerrors.Errorf("read error (%s): %w", input.FilePath, err) @@ -44,14 +43,14 @@ func (a ConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) }, nil } -func (a ConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a terraformConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { return slices.Contains(requiredExts, filepath.Ext(filePath)) } -func (ConfigAnalyzer) Type() analyzer.Type { +func (terraformConfigAnalyzer) Type() analyzer.Type { return analyzer.TypeTerraform } -func (ConfigAnalyzer) Version() int { +func (terraformConfigAnalyzer) Version() int { return version } diff --git a/analyzer/config/terraform/terraform_test.go b/analyzer/config/terraform/terraform_test.go index a7e7eeba9..d17a59a50 100644 --- a/analyzer/config/terraform/terraform_test.go +++ b/analyzer/config/terraform/terraform_test.go @@ -1,4 +1,4 @@ -package terraform_test +package terraform import ( "bytes" @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/require" "github.com/aquasecurity/fanal/analyzer" - "github.com/aquasecurity/fanal/analyzer/config/terraform" "github.com/aquasecurity/fanal/types" ) @@ -41,7 +40,7 @@ func TestConfigAnalyzer_Analyze(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - a := terraform.ConfigAnalyzer{} + a := &terraformConfigAnalyzer{} ctx := context.Background() got, err := a.Analyze(ctx, tt.input) @@ -75,7 +74,7 @@ func TestConfigAnalyzer_Required(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - a := terraform.ConfigAnalyzer{} + a := &terraformConfigAnalyzer{} got := a.Required(tt.filePath, nil) assert.Equal(t, tt.want, got) }) diff --git a/analyzer/config/yaml/yaml.go b/analyzer/config/yaml/yaml.go index fad4485dc..e124ae787 100644 --- a/analyzer/config/yaml/yaml.go +++ b/analyzer/config/yaml/yaml.go @@ -5,7 +5,6 @@ import ( "io" "os" "path/filepath" - "regexp" "golang.org/x/xerrors" @@ -13,21 +12,17 @@ import ( "github.com/aquasecurity/fanal/types" ) +func init() { + analyzer.RegisterAnalyzer(&yamlConfigAnalyzer{}) +} + const version = 1 var requiredExts = []string{".yaml", ".yml"} -type ConfigAnalyzer struct { - filePattern *regexp.Regexp -} - -func NewConfigAnalyzer(filePattern *regexp.Regexp) ConfigAnalyzer { - return ConfigAnalyzer{ - filePattern: filePattern, - } -} +type yamlConfigAnalyzer struct{} -func (a ConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { +func (a yamlConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { b, err := io.ReadAll(input.Content) if err != nil { return nil, xerrors.Errorf("failed to read %s: %w", input.FilePath, err) @@ -47,11 +42,7 @@ func (a ConfigAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) }, nil } -func (a ConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { - if a.filePattern != nil && a.filePattern.MatchString(filePath) { - return true - } - +func (a yamlConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { ext := filepath.Ext(filePath) for _, required := range requiredExts { if ext == required { @@ -61,10 +52,10 @@ func (a ConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool { return false } -func (ConfigAnalyzer) Type() analyzer.Type { +func (yamlConfigAnalyzer) Type() analyzer.Type { return analyzer.TypeYaml } -func (ConfigAnalyzer) Version() int { +func (yamlConfigAnalyzer) Version() int { return version } diff --git a/analyzer/config/yaml/yaml_test.go b/analyzer/config/yaml/yaml_test.go index 24b132617..9aa94bc6b 100644 --- a/analyzer/config/yaml/yaml_test.go +++ b/analyzer/config/yaml/yaml_test.go @@ -1,16 +1,14 @@ -package yaml_test +package yaml import ( "context" "os" - "regexp" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/aquasecurity/fanal/analyzer" - "github.com/aquasecurity/fanal/analyzer/config/yaml" "github.com/aquasecurity/fanal/types" ) @@ -163,7 +161,7 @@ spec: require.NoError(t, err) defer f.Close() - a := yaml.NewConfigAnalyzer(nil) + a := &yamlConfigAnalyzer{} ctx := context.Background() got, err := a.Analyze(ctx, analyzer.AnalysisInput{ FilePath: tt.inputFile, @@ -183,10 +181,9 @@ spec: func Test_yamlConfigAnalyzer_Required(t *testing.T) { tests := []struct { - name string - filePattern *regexp.Regexp - filePath string - want bool + name string + filePath string + want bool }{ { name: "yaml", @@ -203,16 +200,10 @@ func Test_yamlConfigAnalyzer_Required(t *testing.T) { filePath: "deployment.json", want: false, }, - { - name: "file pattern", - filePattern: regexp.MustCompile(`foo*`), - filePath: "foo_file", - want: true, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := yaml.NewConfigAnalyzer(tt.filePattern) + s := &yamlConfigAnalyzer{} got := s.Required(tt.filePath, nil) assert.Equal(t, tt.want, got) @@ -221,7 +212,7 @@ func Test_yamlConfigAnalyzer_Required(t *testing.T) { } func Test_yamlConfigAnalyzer_Type(t *testing.T) { - s := yaml.NewConfigAnalyzer(nil) + s := &yamlConfigAnalyzer{} want := analyzer.TypeYaml got := s.Type() diff --git a/artifact/artifact.go b/artifact/artifact.go index 11fe5b526..6f4c2296b 100644 --- a/artifact/artifact.go +++ b/artifact/artifact.go @@ -16,6 +16,7 @@ type Option struct { DisabledHandlers []types.HandlerType SkipFiles []string SkipDirs []string + FilePatterns []string NoProgress bool Offline bool InsecureSkipTLS bool @@ -30,6 +31,7 @@ func (o *Option) Sort() { }) sort.Strings(o.SkipFiles) sort.Strings(o.SkipDirs) + sort.Strings(o.FilePatterns) } type Artifact interface { diff --git a/artifact/image/image.go b/artifact/image/image.go index a0b5dd7f5..ef5692258 100644 --- a/artifact/image/image.go +++ b/artifact/image/image.go @@ -15,7 +15,6 @@ import ( "golang.org/x/xerrors" "github.com/aquasecurity/fanal/analyzer" - "github.com/aquasecurity/fanal/analyzer/config" "github.com/aquasecurity/fanal/analyzer/secret" "github.com/aquasecurity/fanal/artifact" "github.com/aquasecurity/fanal/cache" @@ -40,12 +39,6 @@ type Artifact struct { } func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (artifact.Artifact, error) { - misconf := opt.MisconfScannerOption - // Register config analyzers - if err := config.RegisterConfigAnalyzers(misconf.FilePatterns); err != nil { - return nil, xerrors.Errorf("config scanner error: %w", err) - } - // Initialize handlers handlerManager, err := handler.NewManager(opt) if err != nil { @@ -57,11 +50,16 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a return nil, xerrors.Errorf("secret scanner error: %w", err) } + a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerGroup, opt.DisabledAnalyzers, opt.FilePatterns) + if err != nil { + return nil, xerrors.Errorf("analyzer group error: %w", err) + } + return Artifact{ image: img, cache: c, walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs), - analyzer: analyzer.NewAnalyzerGroup(opt.AnalyzerGroup, opt.DisabledAnalyzers), + analyzer: a, handlerManager: handlerManager, artifactOption: opt, diff --git a/artifact/image/image_test.go b/artifact/image/image_test.go index 64fd4b57a..254258578 100644 --- a/artifact/image/image_test.go +++ b/artifact/image/image_test.go @@ -19,6 +19,7 @@ import ( "github.com/aquasecurity/fanal/types" _ "github.com/aquasecurity/fanal/analyzer/command/apk" + _ "github.com/aquasecurity/fanal/analyzer/config/all" _ "github.com/aquasecurity/fanal/analyzer/language/php/composer" _ "github.com/aquasecurity/fanal/analyzer/language/ruby/bundler" _ "github.com/aquasecurity/fanal/analyzer/os/alpine" diff --git a/artifact/local/fs.go b/artifact/local/fs.go index fd498ee42..e3bd88a33 100644 --- a/artifact/local/fs.go +++ b/artifact/local/fs.go @@ -14,7 +14,6 @@ import ( "golang.org/x/xerrors" "github.com/aquasecurity/fanal/analyzer" - "github.com/aquasecurity/fanal/analyzer/config" "github.com/aquasecurity/fanal/analyzer/secret" "github.com/aquasecurity/fanal/artifact" "github.com/aquasecurity/fanal/cache" @@ -38,11 +37,6 @@ type Artifact struct { } func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (artifact.Artifact, error) { - // Register config analyzers - if err := config.RegisterConfigAnalyzers(opt.MisconfScannerOption.FilePatterns); err != nil { - return nil, xerrors.Errorf("config analyzer error: %w", err) - } - handlerManager, err := handler.NewManager(opt) if err != nil { return nil, xerrors.Errorf("handler initialize error: %w", err) @@ -53,11 +47,16 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a return nil, xerrors.Errorf("secret scanner error: %w", err) } + a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerGroup, opt.DisabledAnalyzers, opt.FilePatterns) + if err != nil { + return nil, xerrors.Errorf("analyzer group error: %w", err) + } + return Artifact{ rootPath: filepath.Clean(rootPath), cache: c, walker: walker.NewFS(buildAbsPaths(rootPath, opt.SkipFiles), buildAbsPaths(rootPath, opt.SkipDirs)), - analyzer: analyzer.NewAnalyzerGroup(opt.AnalyzerGroup, opt.DisabledAnalyzers), + analyzer: a, handlerManager: handlerManager, artifactOption: opt, diff --git a/artifact/local/fs_test.go b/artifact/local/fs_test.go index 74f281563..b229467b3 100644 --- a/artifact/local/fs_test.go +++ b/artifact/local/fs_test.go @@ -15,9 +15,11 @@ import ( "github.com/aquasecurity/fanal/cache" "github.com/aquasecurity/fanal/types" + _ "github.com/aquasecurity/fanal/analyzer/config/all" _ "github.com/aquasecurity/fanal/analyzer/language/python/pip" _ "github.com/aquasecurity/fanal/analyzer/os/alpine" _ "github.com/aquasecurity/fanal/analyzer/pkg/apk" + _ "github.com/aquasecurity/fanal/handler/misconf" _ "github.com/aquasecurity/fanal/handler/sysfile" ) diff --git a/artifact/remote/git_test.go b/artifact/remote/git_test.go index 313f8b37d..d218e6901 100644 --- a/artifact/remote/git_test.go +++ b/artifact/remote/git_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + _ "github.com/aquasecurity/fanal/analyzer/config/all" "github.com/aquasecurity/fanal/artifact" "github.com/aquasecurity/fanal/cache" "github.com/aquasecurity/fanal/types" diff --git a/cache/key.go b/cache/key.go index 156869ecd..9b602f985 100644 --- a/cache/key.go +++ b/cache/key.go @@ -18,14 +18,15 @@ func CalcKey(id string, analyzerVersions, hookVersions map[string]int, artifactO h := sha256.New() - // Write ID, analyzer/handler versions, and skipped files/dirs + // Write ID, analyzer/handler versions, skipped files/dirs and file patterns keyBase := struct { ID string AnalyzerVersions map[string]int HookVersions map[string]int SkipFiles []string SkipDirs []string - }{id, analyzerVersions, hookVersions, artifactOpt.SkipFiles, artifactOpt.SkipDirs} + FilePatterns []string `json:",omitempty"` + }{id, analyzerVersions, hookVersions, artifactOpt.SkipFiles, artifactOpt.SkipDirs, artifactOpt.FilePatterns} if err := json.NewEncoder(h).Encode(keyBase); err != nil { return "", xerrors.Errorf("json encode error: %w", err) diff --git a/cache/key_test.go b/cache/key_test.go index 3032db62e..b04240fc9 100644 --- a/cache/key_test.go +++ b/cache/key_test.go @@ -78,7 +78,7 @@ func TestCalcKey(t *testing.T) { }, patterns: []string{""}, }, - want: "sha256:d69f13df33f4c159b4ea54c1967384782fcefb5e2a19af35f4cd6d2896e9285e", + want: "sha256:9b81e0bf3aa7809a0f41bc696f353fca5645bcb63b975ab30e23d81886df2e61", }, { name: "with single non empty string in file patterns", @@ -90,7 +90,7 @@ func TestCalcKey(t *testing.T) { }, patterns: []string{"test"}, }, - want: "sha256:d69f13df33f4c159b4ea54c1967384782fcefb5e2a19af35f4cd6d2896e9285e", + want: "sha256:7d91b2623ae4b5641a1f36efa59c774231efe8c28c27a03869894fd49b047fe8", }, { name: "with non empty followed by empty string in file patterns", @@ -102,7 +102,7 @@ func TestCalcKey(t *testing.T) { }, patterns: []string{"test", ""}, }, - want: "sha256:d69f13df33f4c159b4ea54c1967384782fcefb5e2a19af35f4cd6d2896e9285e", + want: "sha256:5c7f1555e95fc60cdaa7e92e99aee15ee7be356fad9e83f1c24a3be06713a5a8", }, { name: "with non empty preceded by empty string in file patterns", @@ -114,7 +114,7 @@ func TestCalcKey(t *testing.T) { }, patterns: []string{"", "test"}, }, - want: "sha256:d69f13df33f4c159b4ea54c1967384782fcefb5e2a19af35f4cd6d2896e9285e", + want: "sha256:5c7f1555e95fc60cdaa7e92e99aee15ee7be356fad9e83f1c24a3be06713a5a8", }, { name: "with policy", @@ -158,13 +158,13 @@ func TestCalcKey(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { artifactOpt := artifact.Option{ - SkipFiles: tt.args.skipFiles, - SkipDirs: tt.args.skipDirs, + SkipFiles: tt.args.skipFiles, + SkipDirs: tt.args.skipDirs, + FilePatterns: tt.args.patterns, MisconfScannerOption: config.ScannerOption{ - FilePatterns: tt.args.patterns, - PolicyPaths: tt.args.policy, - DataPaths: tt.args.data, + PolicyPaths: tt.args.policy, + DataPaths: tt.args.data, }, } got, err := CalcKey(tt.args.key, tt.args.analyzerVersions, tt.args.hookVersions, artifactOpt) diff --git a/external/config_scan.go b/external/config_scan.go index e90fb6deb..ecaa317dd 100644 --- a/external/config_scan.go +++ b/external/config_scan.go @@ -11,6 +11,8 @@ import ( "github.com/aquasecurity/fanal/artifact/local" "github.com/aquasecurity/fanal/cache" "github.com/aquasecurity/fanal/types" + + _ "github.com/aquasecurity/fanal/analyzer/config/all" ) type ConfigScanner struct { diff --git a/go.sum b/go.sum index a23b8ca04..0bd887552 100644 --- a/go.sum +++ b/go.sum @@ -165,10 +165,6 @@ github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:o github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= -github.com/aquasecurity/defsec v0.57.7-0.20220516140531-8f1cdbae2fe3 h1:+fBBePykXtITMGU6bB/1D9NkC8mFamGo1ySiLTZQMMc= -github.com/aquasecurity/defsec v0.57.7-0.20220516140531-8f1cdbae2fe3/go.mod h1:42FxKif2itz+MHFlJ3TJjdroL9Jzj3THoexlueBTU5w= -github.com/aquasecurity/defsec v0.57.7-0.20220516142130-832318963b8d h1:qzkx724H8LUhIwH0n/KRTUhDB5CyRz+/YHJY7xohqEo= -github.com/aquasecurity/defsec v0.57.7-0.20220516142130-832318963b8d/go.mod h1:42FxKif2itz+MHFlJ3TJjdroL9Jzj3THoexlueBTU5w= github.com/aquasecurity/defsec v0.57.7 h1:Y5D9YOUuU5oEtOQ6a+gjfKS5AAhsOK9gMcXUyiq20tY= github.com/aquasecurity/defsec v0.57.7/go.mod h1:42FxKif2itz+MHFlJ3TJjdroL9Jzj3THoexlueBTU5w= github.com/aquasecurity/go-dep-parser v0.0.0-20220503151658-d316f5cc2cff h1:YNlzRYB0n4mZtfuWx6AWaGEjnLVNekchyoFDlYFZegs= From 8eef4bd04ddb87a774358a27b890f6e6d45172bd Mon Sep 17 00:00:00 2001 From: Jeroen Bobbeldijk Date: Mon, 20 Jun 2022 13:20:41 +0200 Subject: [PATCH 2/2] Fix up Helm test --- analyzer/config/helm/helm_test.go | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/analyzer/config/helm/helm_test.go b/analyzer/config/helm/helm_test.go index 3e4ffcfbc..0b677ce26 100644 --- a/analyzer/config/helm/helm_test.go +++ b/analyzer/config/helm/helm_test.go @@ -3,7 +3,6 @@ package helm import ( "context" "os" - "regexp" "testing" "github.com/aquasecurity/fanal/analyzer" @@ -339,7 +338,7 @@ affinity: {} info, err := os.Stat(tt.inputFile) require.NoError(t, err) - a := NewConfigAnalyzer(nil) + a := helmConfigAnalyzer{} ctx := context.Background() got, err := a.Analyze(ctx, analyzer.AnalysisInput{ FilePath: tt.inputFile, @@ -360,10 +359,9 @@ affinity: {} func Test_helmConfigAnalyzer_Required(t *testing.T) { tests := []struct { - name string - filePattern *regexp.Regexp - filePath string - want bool + name string + filePath string + want bool }{ { name: "yaml", @@ -405,17 +403,10 @@ func Test_helmConfigAnalyzer_Required(t *testing.T) { filePath: "testdata/nope.tgz", want: true, // its a tarball after all }, - - { - name: "file pattern", - filePattern: regexp.MustCompile(`foo*`), - filePath: "foo_file", - want: true, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewConfigAnalyzer(tt.filePattern) + s := helmConfigAnalyzer{} info, _ := os.Stat(tt.filePath) @@ -426,7 +417,7 @@ func Test_helmConfigAnalyzer_Required(t *testing.T) { } func Test_helmConfigAnalyzer_Type(t *testing.T) { - s := NewConfigAnalyzer(nil) + s := helmConfigAnalyzer{} want := analyzer.TypeHelm got := s.Type()