From 0236608b01c2b138efaaad848081b2af1325390f Mon Sep 17 00:00:00 2001 From: JoaoCxMartins Date: Tue, 2 Apr 2024 15:24:02 +0100 Subject: [PATCH 1/4] improve the possible dockerfile detection --- pkg/utils/get_extension.go | 27 ++++++++++- pkg/utils/get_extension_test.go | 5 +++ .../CW671X02_EBM_EVENT_RULE | 45 +++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/negative_dockerfile/CW671X02_EBM_EVENT_RULE diff --git a/pkg/utils/get_extension.go b/pkg/utils/get_extension.go index 690d04839ed..081280f5fa1 100644 --- a/pkg/utils/get_extension.go +++ b/pkg/utils/get_extension.go @@ -1,9 +1,11 @@ package utils import ( + "bufio" "bytes" "os" "path/filepath" + "strings" "github.com/rs/zerolog/log" "golang.org/x/tools/godoc/util" @@ -21,13 +23,36 @@ func GetExtension(path string) string { if Contains(base, targets) { ext = base } else if isTextFile(path) { - ext = "possibleDockerfile" + if readPossibleDockerFile(path) { + ext = "possibleDockerfile" + } } } return ext } +func readPossibleDockerFile(path string) bool { + file, err := os.Open(path) + if err != nil { + return false + } + defer file.Close() + // Create a scanner to read the file line by line + scanner := bufio.NewScanner(file) + // Read lines from the file + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "FROM") { + return true + } else if strings.HasPrefix(scanner.Text(), "#") { + continue + } else { + return false + } + } + return false +} + func isTextFile(path string) bool { info, err := os.Stat(path) if err != nil { diff --git a/pkg/utils/get_extension_test.go b/pkg/utils/get_extension_test.go index b2133af93a0..aaf51a6410d 100644 --- a/pkg/utils/get_extension_test.go +++ b/pkg/utils/get_extension_test.go @@ -27,6 +27,11 @@ func TestGetExtension(t *testing.T) { want: ".tf", filePath: "../../test/fixtures/all_auth_users_get_read_access/test/positive.tf", }, + { + name: "Get empty extension from a file not named as Dockerfile and without extension defined", + want: "", + filePath: "../../test/fixtures/negative_dockerfile/CW671X02_EBM_EVENT_RULE", + }, } for _, test := range tests { diff --git a/test/fixtures/negative_dockerfile/CW671X02_EBM_EVENT_RULE b/test/fixtures/negative_dockerfile/CW671X02_EBM_EVENT_RULE new file mode 100644 index 00000000000..d0b995cc882 --- /dev/null +++ b/test/fixtures/negative_dockerfile/CW671X02_EBM_EVENT_RULE @@ -0,0 +1,45 @@ +#!/bin/ksh93 +. ${PROC_DIR:=${0%/*}}/CWPARAM +export UDB_DB=${WAREHOUSE_DB} +. ${PROC_DIR}/DB2COMMON + +db2 -f ${UDB_CMD} + +TABLE=TECW.EBM_EVENT_RULE +CURSOR_NAME=TSTG_${RUN_TYPE}_EBM_EVENT_RULE +QUERY=" +SELECT + RUN_ID + , RPT_CASE_ID + , EVENT + , RPT_RULE_ID + , CHAR('SYMMETRY') + , MBR_MPI_ID + , CAST(TASKNBR as CHAR(3)) + , RULE_TYPE_NBR + , RESULT_FLG + , CASE WHEN EBM_FLG = '0' AND COMPLIANCE_FLG = '0' THEN '0' + ELSE '1' + END AS EBM_QUAL_IND + , EBM_FLG + , COMPLIANCE_FLG + , CURRENT_TIMESTAMP + , CURRENT_TIMESTAMP + , '${SCHED_JOB_NAME}' +FROM TSTG.${RUN_TYPE}_SYMM_EBM_SUMMARY_RESULT_FL +" + +db2 "declare ${CURSOR_NAME} cursor for ${QUERY}" +EXIT_CODE=$? + +if [[ ${EXIT_CODE} -ne 0 ]] +then + echo "Unable to declare cursor. RC:$EXIT_CODE}" + exit ${EXIT_CODE} +fi + +db2 "load from ${CURSOR_NAME} of cursor insert into ${TABLE} nonrecoverable" +EXIT_CODE=$? + +exit $EXIT_CODE + From 34aba83594d3bd32cfe27863b8bfe3868b5febc2 Mon Sep 17 00:00:00 2001 From: JoaoCxMartins Date: Tue, 2 Apr 2024 15:33:17 +0100 Subject: [PATCH 2/4] clean path --- pkg/utils/get_extension.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/utils/get_extension.go b/pkg/utils/get_extension.go index 081280f5fa1..d108dd22183 100644 --- a/pkg/utils/get_extension.go +++ b/pkg/utils/get_extension.go @@ -33,6 +33,7 @@ func GetExtension(path string) string { } func readPossibleDockerFile(path string) bool { + path = filepath.Clean(path) file, err := os.Open(path) if err != nil { return false From df648e11d909db813e33b909f6f4eccecc659d97 Mon Sep 17 00:00:00 2001 From: JoaoCxMartins Date: Tue, 2 Apr 2024 17:00:52 +0100 Subject: [PATCH 3/4] fix the gitignore case --- pkg/analyzer/analyzer_test.go | 8 ++------ pkg/utils/get_extension.go | 3 +++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/analyzer/analyzer_test.go b/pkg/analyzer/analyzer_test.go index 3b45f736635..98cdf146a9c 100644 --- a/pkg/analyzer/analyzer_test.go +++ b/pkg/analyzer/analyzer_test.go @@ -29,7 +29,6 @@ func TestAnalyzer_Analyze(t *testing.T) { wantExclude: []string{ filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json"), filepath.FromSlash("../../test/fixtures/analyzer_test/pnpm-lock.yaml"), - filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink"), filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml")}, typesFromFlag: []string{""}, excludeTypesFromFlag: []string{""}, @@ -149,7 +148,8 @@ func TestAnalyzer_Analyze(t *testing.T) { filepath.FromSlash("../../test/fixtures/gitignore"), }, wantTypes: []string{"kubernetes"}, - wantExclude: []string{filepath.FromSlash("../../test/fixtures/gitignore/positive.dockerfile"), + wantExclude: []string{ + filepath.FromSlash("../../test/fixtures/gitignore/positive.dockerfile"), filepath.FromSlash("../../test/fixtures/gitignore/secrets.tf"), filepath.FromSlash("../../test/fixtures/gitignore/gitignore"), }, @@ -229,7 +229,6 @@ func TestAnalyzer_Analyze(t *testing.T) { filepath.FromSlash("../../test/fixtures/analyzer_test/azureResourceManager.json"), filepath.FromSlash("../../test/fixtures/analyzer_test/cloudformation.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/crossplane.yaml"), - filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink"), filepath.FromSlash("../../test/fixtures/analyzer_test/docker-compose.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/gdm.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/helm/Chart.yaml"), @@ -262,7 +261,6 @@ func TestAnalyzer_Analyze(t *testing.T) { filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json"), filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/pnpm-lock.yaml"), - filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink"), }, typesFromFlag: []string{""}, excludeTypesFromFlag: []string{"ansible", "pulumi"}, @@ -279,7 +277,6 @@ func TestAnalyzer_Analyze(t *testing.T) { wantExclude: []string{ filepath.FromSlash("../../test/fixtures/analyzer_test/pnpm-lock.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json"), - filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink"), filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml"), }, typesFromFlag: []string{""}, @@ -297,7 +294,6 @@ func TestAnalyzer_Analyze(t *testing.T) { wantExclude: []string{ filepath.FromSlash("../../test/fixtures/analyzer_test/pnpm-lock.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json"), - filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink"), filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml"), }, typesFromFlag: []string{""}, diff --git a/pkg/utils/get_extension.go b/pkg/utils/get_extension.go index d108dd22183..257ad42eb78 100644 --- a/pkg/utils/get_extension.go +++ b/pkg/utils/get_extension.go @@ -34,6 +34,9 @@ func GetExtension(path string) string { func readPossibleDockerFile(path string) bool { path = filepath.Clean(path) + if strings.HasSuffix(path, "gitignore") { + return true + } file, err := os.Open(path) if err != nil { return false From 8d56e217933ac227158ad55d82bbd7d85ad0d5bd Mon Sep 17 00:00:00 2001 From: JoaoCxMartins Date: Wed, 3 Apr 2024 12:10:29 +0100 Subject: [PATCH 4/4] use the error to take decisions --- pkg/analyzer/analyzer.go | 102 +++++++++++++++--------------- pkg/engine/provider/filesystem.go | 4 +- pkg/parser/parser.go | 2 +- pkg/remediation/scan.go | 2 +- pkg/utils/get_extension.go | 28 +++++--- pkg/utils/get_extension_test.go | 2 +- 6 files changed, 75 insertions(+), 65 deletions(-) diff --git a/pkg/analyzer/analyzer.go b/pkg/analyzer/analyzer.go index 0d4a0db48bb..3455593450b 100644 --- a/pkg/analyzer/analyzer.go +++ b/pkg/analyzer/analyzer.go @@ -303,20 +303,20 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) { return err } - ext := utils.GetExtension(path) - - trimmedPath := strings.ReplaceAll(path, a.Paths[0], filepath.Base(a.Paths[0])) - ignoreFiles = a.checkIgnore(info.Size(), hasGitIgnoreFile, gitIgnore, path, trimmedPath, ignoreFiles) - - if isConfigFile(path, defaultConfigFiles) { - projectConfigFiles = append(projectConfigFiles, path) - a.Exc = append(a.Exc, path) - } + ext, errExt := utils.GetExtension(path) + if errExt == nil { + trimmedPath := strings.ReplaceAll(path, a.Paths[0], filepath.Base(a.Paths[0])) + ignoreFiles = a.checkIgnore(info.Size(), hasGitIgnoreFile, gitIgnore, path, trimmedPath, ignoreFiles) + + if isConfigFile(path, defaultConfigFiles) { + projectConfigFiles = append(projectConfigFiles, path) + a.Exc = append(a.Exc, path) + } - if _, ok := possibleFileTypes[ext]; ok && !isExcludedFile(path, a.Exc) { - files = append(files, path) + if _, ok := possibleFileTypes[ext]; ok && !isExcludedFile(path, a.Exc) { + files = append(files, path) + } } - return nil }); err != nil { log.Error().Msgf("failed to analyze path %s: %s", path, err) @@ -369,46 +369,48 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) { func (a *analyzerInfo) worker(results, unwanted chan<- string, locCount chan<- int, wg *sync.WaitGroup) { defer wg.Done() - ext := utils.GetExtension(a.filePath) - linesCount, _ := utils.LineCounter(a.filePath) + ext, errExt := utils.GetExtension(a.filePath) + if errExt == nil { + linesCount, _ := utils.LineCounter(a.filePath) - switch ext { - // Dockerfile (direct identification) - case ".dockerfile", "Dockerfile": - if a.isAvailableType(dockerfile) { - results <- dockerfile - locCount <- linesCount - } - // Dockerfile (indirect identification) - case "possibleDockerfile", ".ubi8", ".debian": - if a.isAvailableType(dockerfile) && isDockerfile(a.filePath) { - results <- dockerfile - locCount <- linesCount - } else { - unwanted <- a.filePath - } - // Terraform - case ".tf", "tfvars": - if a.isAvailableType(terraform) { - results <- terraform - locCount <- linesCount - } - // GRPC - case ".proto": - if a.isAvailableType(grpc) { - results <- grpc - locCount <- linesCount - } - // It could be Ansible Config or Ansible Inventory - case ".cfg", ".conf", ".ini": - if a.isAvailableType(ansible) { - results <- ansible - locCount <- linesCount + switch ext { + // Dockerfile (direct identification) + case ".dockerfile", "Dockerfile": + if a.isAvailableType(dockerfile) { + results <- dockerfile + locCount <- linesCount + } + // Dockerfile (indirect identification) + case "possibleDockerfile", ".ubi8", ".debian": + if a.isAvailableType(dockerfile) && isDockerfile(a.filePath) { + results <- dockerfile + locCount <- linesCount + } else { + unwanted <- a.filePath + } + // Terraform + case ".tf", "tfvars": + if a.isAvailableType(terraform) { + results <- terraform + locCount <- linesCount + } + // GRPC + case ".proto": + if a.isAvailableType(grpc) { + results <- grpc + locCount <- linesCount + } + // It could be Ansible Config or Ansible Inventory + case ".cfg", ".conf", ".ini": + if a.isAvailableType(ansible) { + results <- ansible + locCount <- linesCount + } + /* It could be Ansible, Buildah, CICD, CloudFormation, Crossplane, OpenAPI, Azure Resource Manager + Docker Compose, Knative, Kubernetes, Pulumi, ServerlessFW or Google Deployment Manager*/ + case yaml, yml, json, sh: + a.checkContent(results, unwanted, locCount, linesCount, ext) } - /* It could be Ansible, Buildah, CICD, CloudFormation, Crossplane, OpenAPI, Azure Resource Manager - Docker Compose, Knative, Kubernetes, Pulumi, ServerlessFW or Google Deployment Manager*/ - case yaml, yml, json, sh: - a.checkContent(results, unwanted, locCount, linesCount, ext) } } diff --git a/pkg/engine/provider/filesystem.go b/pkg/engine/provider/filesystem.go index 5b56cc04082..38d88edd03e 100644 --- a/pkg/engine/provider/filesystem.go +++ b/pkg/engine/provider/filesystem.go @@ -206,7 +206,7 @@ func (s *FileSystemSourceProvider) walkDir(ctx context.Context, scanPath string, } func openScanFile(scanPath string, extensions model.Extensions) (*os.File, error) { - ext := utils.GetExtension(scanPath) + ext, _ := utils.GetExtension(scanPath) if !extensions.Include(ext) { return nil, ErrNotSupportedFile @@ -261,7 +261,7 @@ func (s *FileSystemSourceProvider) checkConditions(info os.FileInfo, extensions log.Trace().Msgf("File ignored: %s", path) return true, nil } - ext := utils.GetExtension(path) + ext, _ := utils.GetExtension(path) if !extensions.Include(ext) { log.Trace().Msgf("File ignored: %s", path) return true, nil diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index bd209ff300a..d07d2b4e5d7 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -180,7 +180,7 @@ func contains(types []string, supportedTypes map[string]bool) bool { } func (c *Parser) isValidExtension(filePath string) bool { - ext := utils.GetExtension(filePath) + ext, _ := utils.GetExtension(filePath) _, ok := c.extensions[ext] return ok } diff --git a/pkg/remediation/scan.go b/pkg/remediation/scan.go index 220344ad79f..1a12e9eab13 100644 --- a/pkg/remediation/scan.go +++ b/pkg/remediation/scan.go @@ -83,7 +83,7 @@ func scanTmpFile(tmpFile, queryID string, remediated []byte, openAPIResolveRefer // getPayload gets the payload of a file func getPayload(filePath string, content []byte, openAPIResolveReferences bool) (model.FileMetadatas, error) { - ext := utils.GetExtension(filePath) + ext, _ := utils.GetExtension(filePath) var p []*parser.Parser var err error diff --git a/pkg/utils/get_extension.go b/pkg/utils/get_extension.go index 257ad42eb78..5ca6ed1ca06 100644 --- a/pkg/utils/get_extension.go +++ b/pkg/utils/get_extension.go @@ -12,7 +12,7 @@ import ( ) // GetExtension gets the extension of a file path -func GetExtension(path string) string { +func GetExtension(path string) (string, error) { targets := []string{"Dockerfile", "tfvars"} ext := filepath.Ext(path) @@ -22,14 +22,22 @@ func GetExtension(path string) string { if Contains(base, targets) { ext = base - } else if isTextFile(path) { - if readPossibleDockerFile(path) { - ext = "possibleDockerfile" + } else { + isText, err := isTextFile(path) + + if err != nil { + return "", err + } + + if isText { + if readPossibleDockerFile(path) { + ext = "possibleDockerfile" + } } } } - return ext + return ext, nil } func readPossibleDockerFile(path string) bool { @@ -57,26 +65,26 @@ func readPossibleDockerFile(path string) bool { return false } -func isTextFile(path string) bool { +func isTextFile(path string) (bool, error) { info, err := os.Stat(path) if err != nil { log.Error().Msgf("failed to get file info: %s", err) - return false + return false, err } if info.IsDir() { - return false + return false, nil } content, err := os.ReadFile(filepath.Clean(path)) if err != nil { log.Error().Msgf("failed to analyze file: %s", err) - return false + return false, err } content = bytes.Replace(content, []byte("\r"), []byte(""), -1) isText := util.IsText(content) - return isText + return isText, nil } diff --git a/pkg/utils/get_extension_test.go b/pkg/utils/get_extension_test.go index aaf51a6410d..3585559c4c4 100644 --- a/pkg/utils/get_extension_test.go +++ b/pkg/utils/get_extension_test.go @@ -36,7 +36,7 @@ func TestGetExtension(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - got := GetExtension(test.filePath) + got, _ := GetExtension(test.filePath) require.Equal(t, test.want, got) }) }