From 9c46c81a4df7187d6c539516ae817ad716fe6efc Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 3 Mar 2024 10:13:22 +0200 Subject: [PATCH 01/59] Running jas scanners in parallel --- .../jas/applicability/applicabilitymanager.go | 27 +++--- .../applicabilitymanager_test.go | 87 +++++++++++-------- commands/audit/jas/common.go | 21 +++-- commands/audit/jas/common_test.go | 22 +++++ commands/audit/jas/iac/iacscanner.go | 26 ++++-- commands/audit/jas/iac/iacscanner_test.go | 31 ++++--- commands/audit/jas/sast/sastscanner.go | 29 ++++--- commands/audit/jas/sast/sastscanner_test.go | 18 ++-- commands/audit/jas/secrets/secretsscanner.go | 25 ++++-- .../audit/jas/secrets/secretsscanner_test.go | 38 ++++---- commands/audit/jasrunner.go | 74 ++++++++++------ go.sum | 2 + .../npm/npm-project/.jfrog/jfrog-cli.conf.v6 | 4 +- 13 files changed, 255 insertions(+), 149 deletions(-) diff --git a/commands/audit/jas/applicability/applicabilitymanager.go b/commands/audit/jas/applicability/applicabilitymanager.go index a727f1fd..1c65bdac 100644 --- a/commands/audit/jas/applicability/applicabilitymanager.go +++ b/commands/audit/jas/applicability/applicabilitymanager.go @@ -1,10 +1,9 @@ package applicability import ( - "path/filepath" - jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" + "path/filepath" "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -29,6 +28,8 @@ type ApplicabilityScanManager struct { xrayResults []services.ScanResponse scanner *jas.JasScanner thirdPartyScan bool + configFileName string + resultsFileName string } // The getApplicabilityScanResults function runs the applicability scan flow, which includes the following steps: @@ -41,8 +42,12 @@ type ApplicabilityScanManager struct { // bool: true if the user is entitled to the applicability scan, false otherwise. // error: An error object (if any). func RunApplicabilityScan(xrayResults []services.ScanResponse, directDependencies []string, - scannedTechnologies []coreutils.Technology, scanner *jas.JasScanner, thirdPartyContextualAnalysis bool) (results []*sarif.Run, err error) { - applicabilityScanManager := newApplicabilityScanManager(xrayResults, directDependencies, scanner, thirdPartyContextualAnalysis) + scannedTechnologies []coreutils.Technology, scanner *jas.JasScanner, thirdPartyContextualAnalysis bool, ExtendedScanResults *utils.ExtendedScanResults) (err error) { + var scannerTempDir string + if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)); err != nil { + return + } + applicabilityScanManager := newApplicabilityScanManager(xrayResults, directDependencies, scanner, thirdPartyContextualAnalysis, scannerTempDir) if !applicabilityScanManager.shouldRunApplicabilityScan(scannedTechnologies) { log.Debug("The technologies that have been scanned are currently not supported for contextual analysis scanning, or we couldn't find any vulnerable dependencies. Skipping....") return @@ -51,11 +56,11 @@ func RunApplicabilityScan(xrayResults []services.ScanResponse, directDependencie err = utils.ParseAnalyzerManagerError(utils.Applicability, err) return } - results = applicabilityScanManager.applicabilityScanResults + ExtendedScanResults.ApplicabilityScanResults = applicabilityScanManager.applicabilityScanResults return } -func newApplicabilityScanManager(xrayScanResults []services.ScanResponse, directDependencies []string, scanner *jas.JasScanner, thirdPartyScan bool) (manager *ApplicabilityScanManager) { +func newApplicabilityScanManager(xrayScanResults []services.ScanResponse, directDependencies []string, scanner *jas.JasScanner, thirdPartyScan bool, scannerTempDir string) (manager *ApplicabilityScanManager) { directDependenciesCves, indirectDependenciesCves := extractDependenciesCvesFromScan(xrayScanResults, directDependencies) return &ApplicabilityScanManager{ applicabilityScanResults: []*sarif.Run{}, @@ -64,6 +69,8 @@ func newApplicabilityScanManager(xrayScanResults []services.ScanResponse, direct xrayResults: xrayScanResults, scanner: scanner, thirdPartyScan: thirdPartyScan, + configFileName: filepath.Join(scannerTempDir, "config.yaml"), + resultsFileName: filepath.Join(scannerTempDir, "results.sarif"), } } @@ -124,7 +131,7 @@ func (asm *ApplicabilityScanManager) Run(module jfrogappsconfig.Module) (err err if err = asm.runAnalyzerManager(); err != nil { return } - workingDirResults, err := jas.ReadJasScanRunsFromFile(asm.scanner.ResultsFileName, module.SourceRoot, applicabilityDocsUrlSuffix) + workingDirResults, err := jas.ReadJasScanRunsFromFile(asm.resultsFileName, module.SourceRoot, applicabilityDocsUrlSuffix) if err != nil { return } @@ -168,7 +175,7 @@ func (asm *ApplicabilityScanManager) createConfigFile(module jfrogappsconfig.Mod Scans: []scanConfiguration{ { Roots: roots, - Output: asm.scanner.ResultsFileName, + Output: asm.resultsFileName, Type: applicabilityScanType, GrepDisable: false, CveWhitelist: asm.directDependenciesCves, @@ -177,13 +184,13 @@ func (asm *ApplicabilityScanManager) createConfigFile(module jfrogappsconfig.Mod }, }, } - return jas.CreateScannersConfigFile(asm.scanner.ConfigFileName, configFileContent, utils.Applicability) + return jas.CreateScannersConfigFile(asm.configFileName, configFileContent, utils.Applicability) } // Runs the analyzerManager app and returns a boolean to indicate whether the user is entitled for // advance security feature func (asm *ApplicabilityScanManager) runAnalyzerManager() error { - return asm.scanner.AnalyzerManager.Exec(asm.scanner.ConfigFileName, applicabilityScanCommand, filepath.Dir(asm.scanner.AnalyzerManager.AnalyzerManagerFullPath), asm.scanner.ServerDetails) + return asm.scanner.AnalyzerManager.Exec(asm.configFileName, applicabilityScanCommand, filepath.Dir(asm.scanner.AnalyzerManager.AnalyzerManagerFullPath), asm.scanner.ServerDetails) } func removeElementFromSlice(skipDirs []string, element string) []string { diff --git a/commands/audit/jas/applicability/applicabilitymanager_test.go b/commands/audit/jas/applicability/applicabilitymanager_test.go index 672f8623..cbf84716 100644 --- a/commands/audit/jas/applicability/applicabilitymanager_test.go +++ b/commands/audit/jas/applicability/applicabilitymanager_test.go @@ -1,6 +1,8 @@ package applicability import ( + "github.com/jfrog/jfrog-cli-security/utils" + "github.com/stretchr/testify/require" "os" "path/filepath" "testing" @@ -19,12 +21,13 @@ func TestNewApplicabilityScanManager_InputIsValid(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() // Act - applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false) + + applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false, "temoDirPath") // Assert if assert.NotNil(t, applicabilityManager) { - assert.NotEmpty(t, applicabilityManager.scanner.ConfigFileName) - assert.NotEmpty(t, applicabilityManager.scanner.ResultsFileName) + assert.NotEmpty(t, applicabilityManager.configFileName) + assert.NotEmpty(t, applicabilityManager.resultsFileName) assert.Len(t, applicabilityManager.directDependenciesCves, 5) } } @@ -33,14 +36,14 @@ func TestNewApplicabilityScanManager_DependencyTreeDoesntExist(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() // Act - applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, nil, scanner, false) + applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, nil, scanner, false, "temoDirPath") // Assert if assert.NotNil(t, applicabilityManager) { assert.NotNil(t, applicabilityManager.scanner.ScannerDirCleanupFunc) assert.Len(t, applicabilityManager.scanner.JFrogAppsConfig.Modules, 1) - assert.NotEmpty(t, applicabilityManager.scanner.ConfigFileName) - assert.NotEmpty(t, applicabilityManager.scanner.ResultsFileName) + assert.NotEmpty(t, applicabilityManager.configFileName) + assert.NotEmpty(t, applicabilityManager.resultsFileName) assert.Empty(t, applicabilityManager.directDependenciesCves) } } @@ -70,17 +73,17 @@ func TestNewApplicabilityScanManager_NoDirectDependenciesInScan(t *testing.T) { // Act scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - applicabilityManager := newApplicabilityScanManager(noDirectDependenciesResults, mockDirectDependencies, scanner, false) + applicabilityManager := newApplicabilityScanManager(noDirectDependenciesResults, mockDirectDependencies, scanner, false, "temoDirPath") assertApplicabilityScanner(t, applicabilityManager) // ThirdPartyContextual shouldn't change anything here as this is not npm. - applicabilityManager = newApplicabilityScanManager(noDirectDependenciesResults, mockDirectDependencies, scanner, true) + applicabilityManager = newApplicabilityScanManager(noDirectDependenciesResults, mockDirectDependencies, scanner, true, "temoDirPath") assertApplicabilityScanner(t, applicabilityManager) } func assertApplicabilityScanner(t *testing.T, applicabilityManager *ApplicabilityScanManager) { if assert.NotNil(t, applicabilityManager) { - assert.NotEmpty(t, applicabilityManager.scanner.ConfigFileName) - assert.NotEmpty(t, applicabilityManager.scanner.ResultsFileName) + assert.NotEmpty(t, applicabilityManager.configFileName) + assert.NotEmpty(t, applicabilityManager.resultsFileName) // Non-direct dependencies should not be added assert.Empty(t, applicabilityManager.directDependenciesCves) } @@ -91,12 +94,12 @@ func TestNewApplicabilityScanManager_MultipleDependencyTrees(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() // Act - applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockMultiRootDirectDependencies, scanner, false) + applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockMultiRootDirectDependencies, scanner, false, "temoDirPath") // Assert if assert.NotNil(t, applicabilityManager) { - assert.NotEmpty(t, applicabilityManager.scanner.ConfigFileName) - assert.NotEmpty(t, applicabilityManager.scanner.ResultsFileName) + assert.NotEmpty(t, applicabilityManager.configFileName) + assert.NotEmpty(t, applicabilityManager.resultsFileName) assert.Len(t, applicabilityManager.directDependenciesCves, 5) } } @@ -117,12 +120,12 @@ func TestNewApplicabilityScanManager_ViolationsDontExistInResults(t *testing.T) defer cleanUp() // Act - applicabilityManager := newApplicabilityScanManager(noViolationScanResponse, mockDirectDependencies, scanner, false) + applicabilityManager := newApplicabilityScanManager(noViolationScanResponse, mockDirectDependencies, scanner, false, "temoDirPath") // Assert if assert.NotNil(t, applicabilityManager) { - assert.NotEmpty(t, applicabilityManager.scanner.ConfigFileName) - assert.NotEmpty(t, applicabilityManager.scanner.ResultsFileName) + assert.NotEmpty(t, applicabilityManager.configFileName) + assert.NotEmpty(t, applicabilityManager.resultsFileName) assert.Len(t, applicabilityManager.directDependenciesCves, 3) } } @@ -143,12 +146,12 @@ func TestNewApplicabilityScanManager_VulnerabilitiesDontExist(t *testing.T) { defer cleanUp() // Act - applicabilityManager := newApplicabilityScanManager(noVulnerabilitiesScanResponse, mockDirectDependencies, scanner, false) + applicabilityManager := newApplicabilityScanManager(noVulnerabilitiesScanResponse, mockDirectDependencies, scanner, false, "temoDirPath") // Assert if assert.NotNil(t, applicabilityManager) { - assert.NotEmpty(t, applicabilityManager.scanner.ConfigFileName) - assert.NotEmpty(t, applicabilityManager.scanner.ResultsFileName) + assert.NotEmpty(t, applicabilityManager.configFileName) + assert.NotEmpty(t, applicabilityManager.resultsFileName) assert.Len(t, applicabilityManager.directDependenciesCves, 2) } } @@ -157,10 +160,11 @@ func TestApplicabilityScanManager_ShouldRun_TechnologiesNotEligibleForScan(t *te scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - results, err := RunApplicabilityScan(jas.FakeBasicXrayResults, mockDirectDependencies, []coreutils.Technology{coreutils.Nuget, coreutils.Go}, scanner, false) + var results utils.ExtendedScanResults + err := RunApplicabilityScan(jas.FakeBasicXrayResults, mockDirectDependencies, []coreutils.Technology{coreutils.Nuget, coreutils.Go}, scanner, false, &results) // Assert - assert.Nil(t, results) + assert.Nil(t, results.ApplicabilityScanResults) assert.NoError(t, err) } @@ -169,7 +173,7 @@ func TestApplicabilityScanManager_ShouldRun_ScanResultsAreEmpty(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - applicabilityManager := newApplicabilityScanManager(nil, mockDirectDependencies, scanner, false) + applicabilityManager := newApplicabilityScanManager(nil, mockDirectDependencies, scanner, false, "temoDirPath") // Assert eligible := applicabilityManager.shouldRunApplicabilityScan([]coreutils.Technology{coreutils.Nuget}) @@ -264,7 +268,9 @@ func TestCreateConfigFile_VerifyFileWasCreated(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, scanner, false) + scannerTempDir, err := jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)) + require.NoError(t, err) + applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, scanner, false, scannerTempDir) currWd, err := coreutils.GetWorkingDirectory() assert.NoError(t, err) @@ -272,13 +278,13 @@ func TestCreateConfigFile_VerifyFileWasCreated(t *testing.T) { assert.NoError(t, err) defer func() { - err = os.Remove(applicabilityManager.scanner.ConfigFileName) + err = os.Remove(applicabilityManager.configFileName) assert.NoError(t, err) }() - _, fileNotExistError := os.Stat(applicabilityManager.scanner.ConfigFileName) + _, fileNotExistError := os.Stat(applicabilityManager.configFileName) assert.NoError(t, fileNotExistError) - fileContent, err := os.ReadFile(applicabilityManager.scanner.ConfigFileName) + fileContent, err := os.ReadFile(applicabilityManager.configFileName) assert.NoError(t, err) assert.True(t, len(fileContent) > 0) } @@ -288,12 +294,13 @@ func TestParseResults_EmptyResults_AllCvesShouldGetUnknown(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false) - applicabilityManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "applicability-scan", "empty-results.sarif") + scannerTempDir, err := jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)) + require.NoError(t, err) + applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false, scannerTempDir) + applicabilityManager.resultsFileName = filepath.Join(jas.GetTestDataPath(), "applicability-scan", "empty-results.sarif") // Act - var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.resultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) if assert.NoError(t, err) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) @@ -305,12 +312,14 @@ func TestParseResults_ApplicableCveExist(t *testing.T) { // Arrange scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false) - applicabilityManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "applicability-scan", "applicable-cve-results.sarif") + + scannerTempDir, err := jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)) + require.NoError(t, err) + applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false, scannerTempDir) + applicabilityManager.resultsFileName = filepath.Join(jas.GetTestDataPath(), "applicability-scan", "applicable-cve-results.sarif") // Act - var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.resultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) @@ -322,12 +331,14 @@ func TestParseResults_AllCvesNotApplicable(t *testing.T) { // Arrange scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false) - applicabilityManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "applicability-scan", "no-applicable-cves-results.sarif") + + scannerTempDir, err := jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)) + require.NoError(t, err) + applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false, scannerTempDir) + applicabilityManager.resultsFileName = filepath.Join(jas.GetTestDataPath(), "applicability-scan", "no-applicable-cves-results.sarif") // Act - var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.resultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) diff --git a/commands/audit/jas/common.go b/commands/audit/jas/common.go index 55e2fdc4..efc8d1ca 100644 --- a/commands/audit/jas/common.go +++ b/commands/audit/jas/common.go @@ -41,8 +41,7 @@ var ( ) type JasScanner struct { - ConfigFileName string - ResultsFileName string + TempDir string AnalyzerManager utils.AnalyzerManager ServerDetails *config.ServerDetails JFrogAppsConfig *jfrogappsconfig.JFrogAppsConfig @@ -58,12 +57,11 @@ func NewJasScanner(workingDirs []string, serverDetails *config.ServerDetails) (s if tempDir, err = fileutils.CreateTempDir(); err != nil { return } + scanner.TempDir = tempDir scanner.ScannerDirCleanupFunc = func() error { return fileutils.RemoveTempDir(tempDir) } scanner.ServerDetails = serverDetails - scanner.ConfigFileName = filepath.Join(tempDir, "config.yaml") - scanner.ResultsFileName = filepath.Join(tempDir, "results.sarif") scanner.JFrogAppsConfig, err = createJFrogAppsConfig(workingDirs) return } @@ -95,9 +93,6 @@ type ScannerCmd interface { func (a *JasScanner) Run(scannerCmd ScannerCmd) (err error) { for _, module := range a.JFrogAppsConfig.Modules { func() { - defer func() { - err = errors.Join(err, deleteJasProcessFiles(a.ConfigFileName, a.ResultsFileName)) - }() if err = scannerCmd.Run(module); err != nil { return } @@ -274,3 +269,15 @@ func GetExcludePatterns(module jfrogappsconfig.Module, scanner *jfrogappsconfig. } return excludePatterns } + +func CreateScannerTempDirectory(scanner *JasScanner, scanType string) (string, error) { + if scanner.TempDir == "" { + return "", errors.New("scanner temp dir cannot be created in an empty base dir") + } + scannerTempDir := scanner.TempDir + "/" + scanType + err := os.MkdirAll(scannerTempDir, 0777) + if err != nil { + return "", err + } + return scannerTempDir, nil +} diff --git a/commands/audit/jas/common_test.go b/commands/audit/jas/common_test.go index 98129bcf..6dd473ed 100644 --- a/commands/audit/jas/common_test.go +++ b/commands/audit/jas/common_test.go @@ -6,6 +6,7 @@ import ( "github.com/jfrog/jfrog-cli-security/utils" "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" + "os" ) func TestExcludeSuppressResults(t *testing.T) { @@ -88,3 +89,24 @@ func TestAddScoreToRunRules(t *testing.T) { assert.Equal(t, test.expectedOutput, test.sarifRun.Tool.Driver.Rules) } } + +func TestCreateScannerTempDirectory(t *testing.T) { + scanner, cleanUp := InitJasTest(t) + defer cleanUp() + tempDir, err := CreateScannerTempDirectory(scanner, string(utils.Applicability)) + assert.NoError(t, err) + assert.NotEmpty(t, tempDir) + + // Check directory exists. + _, err = os.Stat(tempDir) + assert.NoError(t, err) +} + +func TestCreateScannerTempDirectory_baseDirIsEmpty(t *testing.T) { + scanner, cleanUp := InitJasTest(t) + defer cleanUp() + + scanner.TempDir = "" + _, err := CreateScannerTempDirectory(scanner, string(utils.Applicability)) + assert.Error(t, err) +} diff --git a/commands/audit/jas/iac/iacscanner.go b/commands/audit/jas/iac/iacscanner.go index 312dcf4f..d8eda92b 100644 --- a/commands/audit/jas/iac/iacscanner.go +++ b/commands/audit/jas/iac/iacscanner.go @@ -20,6 +20,8 @@ const ( type IacScanManager struct { iacScannerResults []*sarif.Run scanner *jas.JasScanner + configFileName string + resultsFileName string } // The getIacScanResults function runs the iac scan flow, which includes the following steps: @@ -30,8 +32,12 @@ type IacScanManager struct { // []utils.SourceCodeScanResult: a list of the iac violations that were found. // bool: true if the user is entitled to iac scan, false otherwise. // error: An error object (if any). -func RunIacScan(scanner *jas.JasScanner) (results []*sarif.Run, err error) { - iacScanManager := newIacScanManager(scanner) +func RunIacScan(scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults) (err error) { + var scannerTempDir string + if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.IaC)); err != nil { + return + } + iacScanManager := newIacScanManager(scanner, scannerTempDir) log.Info("Running IaC scanning...") if err = iacScanManager.scanner.Run(iacScanManager); err != nil { err = utils.ParseAnalyzerManagerError(utils.IaC, err) @@ -40,15 +46,17 @@ func RunIacScan(scanner *jas.JasScanner) (results []*sarif.Run, err error) { if len(iacScanManager.iacScannerResults) > 0 { log.Info("Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities") } - results = iacScanManager.iacScannerResults + results := iacScanManager.iacScannerResults + ExtendedScanResults.IacScanResults = results return } -func newIacScanManager(scanner *jas.JasScanner) (manager *IacScanManager) { +func newIacScanManager(scanner *jas.JasScanner, scannerTempDir string) (manager *IacScanManager) { return &IacScanManager{ iacScannerResults: []*sarif.Run{}, scanner: scanner, - } + configFileName: filepath.Join(scannerTempDir, "config.yaml"), + resultsFileName: filepath.Join(scannerTempDir, "results.sarif")} } func (iac *IacScanManager) Run(module jfrogappsconfig.Module) (err error) { @@ -61,7 +69,7 @@ func (iac *IacScanManager) Run(module jfrogappsconfig.Module) (err error) { if err = iac.runAnalyzerManager(); err != nil { return } - workingDirResults, err := jas.ReadJasScanRunsFromFile(iac.scanner.ResultsFileName, module.SourceRoot, iacDocsUrlSuffix) + workingDirResults, err := jas.ReadJasScanRunsFromFile(iac.resultsFileName, module.SourceRoot, iacDocsUrlSuffix) if err != nil { return } @@ -89,15 +97,15 @@ func (iac *IacScanManager) createConfigFile(module jfrogappsconfig.Module) error Scans: []iacScanConfiguration{ { Roots: roots, - Output: iac.scanner.ResultsFileName, + Output: iac.resultsFileName, Type: iacScannerType, SkippedDirs: jas.GetExcludePatterns(module, module.Scanners.Iac), }, }, } - return jas.CreateScannersConfigFile(iac.scanner.ConfigFileName, configFileContent, utils.IaC) + return jas.CreateScannersConfigFile(iac.configFileName, configFileContent, utils.IaC) } func (iac *IacScanManager) runAnalyzerManager() error { - return iac.scanner.AnalyzerManager.Exec(iac.scanner.ConfigFileName, iacScanCommand, filepath.Dir(iac.scanner.AnalyzerManager.AnalyzerManagerFullPath), iac.scanner.ServerDetails) + return iac.scanner.AnalyzerManager.Exec(iac.configFileName, iacScanCommand, filepath.Dir(iac.scanner.AnalyzerManager.AnalyzerManagerFullPath), iac.scanner.ServerDetails) } diff --git a/commands/audit/jas/iac/iacscanner_test.go b/commands/audit/jas/iac/iacscanner_test.go index 1a5403f4..28c748f1 100644 --- a/commands/audit/jas/iac/iacscanner_test.go +++ b/commands/audit/jas/iac/iacscanner_test.go @@ -1,6 +1,8 @@ package iac import ( + "github.com/jfrog/jfrog-cli-security/utils" + "github.com/stretchr/testify/require" "os" "path/filepath" "testing" @@ -16,12 +18,13 @@ func TestNewIacScanManager(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t, "currentDir") defer cleanUp() // Act - iacScanManager := newIacScanManager(scanner) + + iacScanManager := newIacScanManager(scanner, "temoDirPath") // Assert if assert.NotNil(t, iacScanManager) { - assert.NotEmpty(t, iacScanManager.scanner.ConfigFileName) - assert.NotEmpty(t, iacScanManager.scanner.ResultsFileName) + assert.NotEmpty(t, iacScanManager.configFileName) + assert.NotEmpty(t, iacScanManager.resultsFileName) assert.NotEmpty(t, iacScanManager.scanner.JFrogAppsConfig.Modules[0].SourceRoot) assert.Equal(t, &jas.FakeServerDetails, iacScanManager.scanner.ServerDetails) } @@ -31,20 +34,22 @@ func TestIacScan_CreateConfigFile_VerifyFileWasCreated(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t, "currentDir") defer cleanUp() - iacScanManager := newIacScanManager(scanner) + scannerTempDir, err := jas.CreateScannerTempDirectory(scanner, string(utils.IaC)) + require.NoError(t, err) + iacScanManager := newIacScanManager(scanner, scannerTempDir) currWd, err := coreutils.GetWorkingDirectory() assert.NoError(t, err) err = iacScanManager.createConfigFile(jfrogappsconfig.Module{SourceRoot: currWd}) defer func() { - err = os.Remove(iacScanManager.scanner.ConfigFileName) + err = os.Remove(iacScanManager.configFileName) assert.NoError(t, err) }() - _, fileNotExistError := os.Stat(iacScanManager.scanner.ConfigFileName) + _, fileNotExistError := os.Stat(iacScanManager.configFileName) assert.NoError(t, fileNotExistError) - fileContent, err := os.ReadFile(iacScanManager.scanner.ConfigFileName) + fileContent, err := os.ReadFile(iacScanManager.configFileName) assert.NoError(t, err) assert.True(t, len(fileContent) > 0) } @@ -54,12 +59,12 @@ func TestIacParseResults_EmptyResults(t *testing.T) { defer cleanUp() // Arrange - iacScanManager := newIacScanManager(scanner) - iacScanManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "iac-scan", "no-violations.sarif") + iacScanManager := newIacScanManager(scanner, "temoDirPath") + iacScanManager.resultsFileName = filepath.Join(jas.GetTestDataPath(), "iac-scan", "no-violations.sarif") // Act var err error - iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, iacDocsUrlSuffix) + iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.resultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, iacDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, iacScanManager.iacScannerResults) { assert.Len(t, iacScanManager.iacScannerResults, 1) assert.Empty(t, iacScanManager.iacScannerResults[0].Results) @@ -70,12 +75,12 @@ func TestIacParseResults_ResultsContainIacViolations(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() // Arrange - iacScanManager := newIacScanManager(scanner) - iacScanManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "iac-scan", "contains-iac-violations.sarif") + iacScanManager := newIacScanManager(scanner, "temoDirPath") + iacScanManager.resultsFileName = filepath.Join(jas.GetTestDataPath(), "iac-scan", "contains-iac-violations.sarif") // Act var err error - iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, iacDocsUrlSuffix) + iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.resultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, iacDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, iacScanManager.iacScannerResults) { assert.Len(t, iacScanManager.iacScannerResults, 1) assert.Len(t, iacScanManager.iacScannerResults[0].Results, 4) diff --git a/commands/audit/jas/sast/sastscanner.go b/commands/audit/jas/sast/sastscanner.go index f81d869a..ce47ad16 100644 --- a/commands/audit/jas/sast/sastscanner.go +++ b/commands/audit/jas/sast/sastscanner.go @@ -2,15 +2,13 @@ package sast import ( "fmt" - - "path/filepath" - jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/owenrumney/go-sarif/v2/sarif" "golang.org/x/exp/maps" + "path/filepath" ) const ( @@ -22,10 +20,16 @@ const ( type SastScanManager struct { sastScannerResults []*sarif.Run scanner *jas.JasScanner + configFileName string + resultsFileName string } -func RunSastScan(scanner *jas.JasScanner) (results []*sarif.Run, err error) { - sastScanManager := newSastScanManager(scanner) +func RunSastScan(scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults) (err error) { + var scannerTempDir string + if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Sast)); err != nil { + return + } + sastScanManager := newSastScanManager(scanner, scannerTempDir) log.Info("Running SAST scanning...") if err = sastScanManager.scanner.Run(sastScanManager); err != nil { err = utils.ParseAnalyzerManagerError(utils.Sast, err) @@ -34,15 +38,17 @@ func RunSastScan(scanner *jas.JasScanner) (results []*sarif.Run, err error) { if len(sastScanManager.sastScannerResults) > 0 { log.Info("Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities") } - results = sastScanManager.sastScannerResults + results := sastScanManager.sastScannerResults + ExtendedScanResults.SastScanResults = results return } -func newSastScanManager(scanner *jas.JasScanner) (manager *SastScanManager) { +func newSastScanManager(scanner *jas.JasScanner, scannerTempDir string) (manager *SastScanManager) { return &SastScanManager{ sastScannerResults: []*sarif.Run{}, scanner: scanner, - } + configFileName: filepath.Join(scannerTempDir, "config.yaml"), + resultsFileName: filepath.Join(scannerTempDir, "results.sarif")} } func (ssm *SastScanManager) Run(module jfrogappsconfig.Module) (err error) { @@ -52,11 +58,10 @@ func (ssm *SastScanManager) Run(module jfrogappsconfig.Module) (err error) { if err = ssm.createConfigFile(module); err != nil { return } - scanner := ssm.scanner if err = ssm.runAnalyzerManager(filepath.Dir(ssm.scanner.AnalyzerManager.AnalyzerManagerFullPath)); err != nil { return } - workingDirRuns, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, module.SourceRoot, sastDocsUrlSuffix) + workingDirRuns, err := jas.ReadJasScanRunsFromFile(ssm.resultsFileName, module.SourceRoot, sastDocsUrlSuffix) if err != nil { return } @@ -97,11 +102,11 @@ func (ssm *SastScanManager) createConfigFile(module jfrogappsconfig.Module) erro }, }, } - return jas.CreateScannersConfigFile(ssm.scanner.ConfigFileName, configFileContent, utils.Sast) + return jas.CreateScannersConfigFile(ssm.configFileName, configFileContent, utils.Sast) } func (ssm *SastScanManager) runAnalyzerManager(wd string) error { - return ssm.scanner.AnalyzerManager.ExecWithOutputFile(ssm.scanner.ConfigFileName, sastScanCommand, wd, ssm.scanner.ResultsFileName, ssm.scanner.ServerDetails) + return ssm.scanner.AnalyzerManager.ExecWithOutputFile(ssm.configFileName, sastScanCommand, wd, ssm.resultsFileName, ssm.scanner.ServerDetails) } // In the Sast scanner, there can be multiple results with the same location. diff --git a/commands/audit/jas/sast/sastscanner_test.go b/commands/audit/jas/sast/sastscanner_test.go index 5dcd2110..8eab327c 100644 --- a/commands/audit/jas/sast/sastscanner_test.go +++ b/commands/audit/jas/sast/sastscanner_test.go @@ -15,12 +15,12 @@ func TestNewSastScanManager(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t, "currentDir") defer cleanUp() // Act - sastScanManager := newSastScanManager(scanner) + sastScanManager := newSastScanManager(scanner, "temoDirPath") // Assert if assert.NotNil(t, sastScanManager) { - assert.NotEmpty(t, sastScanManager.scanner.ConfigFileName) - assert.NotEmpty(t, sastScanManager.scanner.ResultsFileName) + assert.NotEmpty(t, sastScanManager.configFileName) + assert.NotEmpty(t, sastScanManager.resultsFileName) assert.NotEmpty(t, sastScanManager.scanner.JFrogAppsConfig.Modules[0].SourceRoot) assert.Equal(t, &jas.FakeServerDetails, sastScanManager.scanner.ServerDetails) } @@ -31,12 +31,12 @@ func TestSastParseResults_EmptyResults(t *testing.T) { defer cleanUp() // Arrange - sastScanManager := newSastScanManager(scanner) - sastScanManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "sast-scan", "no-violations.sarif") + sastScanManager := newSastScanManager(scanner, "temoDirPath") + sastScanManager.resultsFileName = filepath.Join(jas.GetTestDataPath(), "sast-scan", "no-violations.sarif") // Act var err error - sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, sastDocsUrlSuffix) + sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.resultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, sastDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, sastScanManager.sastScannerResults) { @@ -52,12 +52,12 @@ func TestSastParseResults_ResultsContainIacViolations(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() // Arrange - sastScanManager := newSastScanManager(scanner) - sastScanManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "sast-scan", "contains-sast-violations.sarif") + sastScanManager := newSastScanManager(scanner, "temoDirPath") + sastScanManager.resultsFileName = filepath.Join(jas.GetTestDataPath(), "sast-scan", "contains-sast-violations.sarif") // Act var err error - sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, sastDocsUrlSuffix) + sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.resultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, sastDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, sastScanManager.sastScannerResults) { diff --git a/commands/audit/jas/secrets/secretsscanner.go b/commands/audit/jas/secrets/secretsscanner.go index ca9d2ce7..96d622e5 100644 --- a/commands/audit/jas/secrets/secretsscanner.go +++ b/commands/audit/jas/secrets/secretsscanner.go @@ -20,6 +20,8 @@ const ( type SecretScanManager struct { secretsScannerResults []*sarif.Run scanner *jas.JasScanner + configFileName string + resultsFileName string } // The getSecretsScanResults function runs the secrets scan flow, which includes the following steps: @@ -29,24 +31,31 @@ type SecretScanManager struct { // Return values: // []utils.IacOrSecretResult: a list of the secrets that were found. // error: An error object (if any). -func RunSecretsScan(scanner *jas.JasScanner) (results []*sarif.Run, err error) { - secretScanManager := newSecretsScanManager(scanner) +func RunSecretsScan(scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults) (err error) { + var scannerTempDir string + if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Secrets)); err != nil { + return + } + secretScanManager := newSecretsScanManager(scanner, scannerTempDir) log.Info("Running secrets scanning...") if err = secretScanManager.scanner.Run(secretScanManager); err != nil { err = utils.ParseAnalyzerManagerError(utils.Secrets, err) return } - results = secretScanManager.secretsScannerResults + results := secretScanManager.secretsScannerResults if len(results) > 0 { log.Info("Found", utils.GetResultsLocationCount(results...), "secrets") } + ExtendedScanResults.SecretsScanResults = results return } -func newSecretsScanManager(scanner *jas.JasScanner) (manager *SecretScanManager) { +func newSecretsScanManager(scanner *jas.JasScanner, scannerTempDir string) (manager *SecretScanManager) { return &SecretScanManager{ secretsScannerResults: []*sarif.Run{}, scanner: scanner, + configFileName: filepath.Join(scannerTempDir, "config.yaml"), + resultsFileName: filepath.Join(scannerTempDir, "results.sarif"), } } @@ -60,7 +69,7 @@ func (ssm *SecretScanManager) Run(module jfrogappsconfig.Module) (err error) { if err = ssm.runAnalyzerManager(); err != nil { return } - workingDirRuns, err := jas.ReadJasScanRunsFromFile(ssm.scanner.ResultsFileName, module.SourceRoot, secretsDocsUrlSuffix) + workingDirRuns, err := jas.ReadJasScanRunsFromFile(ssm.resultsFileName, module.SourceRoot, secretsDocsUrlSuffix) if err != nil { return } @@ -88,17 +97,17 @@ func (s *SecretScanManager) createConfigFile(module jfrogappsconfig.Module) erro Scans: []secretsScanConfiguration{ { Roots: roots, - Output: s.scanner.ResultsFileName, + Output: s.resultsFileName, Type: secretsScannerType, SkippedDirs: jas.GetExcludePatterns(module, module.Scanners.Secrets), }, }, } - return jas.CreateScannersConfigFile(s.scanner.ConfigFileName, configFileContent, utils.Secrets) + return jas.CreateScannersConfigFile(s.configFileName, configFileContent, utils.Secrets) } func (s *SecretScanManager) runAnalyzerManager() error { - return s.scanner.AnalyzerManager.Exec(s.scanner.ConfigFileName, secretsScanCommand, filepath.Dir(s.scanner.AnalyzerManager.AnalyzerManagerFullPath), s.scanner.ServerDetails) + return s.scanner.AnalyzerManager.Exec(s.configFileName, secretsScanCommand, filepath.Dir(s.scanner.AnalyzerManager.AnalyzerManagerFullPath), s.scanner.ServerDetails) } func maskSecret(secret string) string { diff --git a/commands/audit/jas/secrets/secretsscanner_test.go b/commands/audit/jas/secrets/secretsscanner_test.go index 82f10c99..9b1ffa8d 100644 --- a/commands/audit/jas/secrets/secretsscanner_test.go +++ b/commands/audit/jas/secrets/secretsscanner_test.go @@ -1,6 +1,8 @@ package secrets import ( + "github.com/jfrog/jfrog-cli-security/utils" + "github.com/stretchr/testify/require" "os" "path/filepath" "testing" @@ -15,18 +17,21 @@ import ( func TestNewSecretsScanManager(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - secretScanManager := newSecretsScanManager(scanner) + secretScanManager := newSecretsScanManager(scanner, "temoDirPath") assert.NotEmpty(t, secretScanManager) - assert.NotEmpty(t, secretScanManager.scanner.ConfigFileName) - assert.NotEmpty(t, secretScanManager.scanner.ResultsFileName) + assert.NotEmpty(t, secretScanManager.configFileName) + assert.NotEmpty(t, secretScanManager.resultsFileName) assert.Equal(t, &jas.FakeServerDetails, secretScanManager.scanner.ServerDetails) } func TestSecretsScan_CreateConfigFile_VerifyFileWasCreated(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - secretScanManager := newSecretsScanManager(scanner) + + scannerTempDir, err := jas.CreateScannerTempDirectory(scanner, string(utils.Secrets)) + require.NoError(t, err) + secretScanManager := newSecretsScanManager(scanner, scannerTempDir) currWd, err := coreutils.GetWorkingDirectory() assert.NoError(t, err) @@ -34,13 +39,13 @@ func TestSecretsScan_CreateConfigFile_VerifyFileWasCreated(t *testing.T) { assert.NoError(t, err) defer func() { - err = os.Remove(secretScanManager.scanner.ConfigFileName) + err = os.Remove(secretScanManager.configFileName) assert.NoError(t, err) }() - _, fileNotExistError := os.Stat(secretScanManager.scanner.ConfigFileName) + _, fileNotExistError := os.Stat(secretScanManager.configFileName) assert.NoError(t, fileNotExistError) - fileContent, err := os.ReadFile(secretScanManager.scanner.ConfigFileName) + fileContent, err := os.ReadFile(secretScanManager.configFileName) assert.NoError(t, err) assert.True(t, len(fileContent) > 0) } @@ -53,7 +58,7 @@ func TestRunAnalyzerManager_ReturnsGeneralError(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - secretScanManager := newSecretsScanManager(scanner) + secretScanManager := newSecretsScanManager(scanner, "temoDirPath") assert.Error(t, secretScanManager.runAnalyzerManager()) } @@ -61,12 +66,12 @@ func TestParseResults_EmptyResults(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() // Arrange - secretScanManager := newSecretsScanManager(scanner) - secretScanManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "secrets-scan", "no-secrets.sarif") + secretScanManager := newSecretsScanManager(scanner, "temoDirPath") + secretScanManager.resultsFileName = filepath.Join(jas.GetTestDataPath(), "secrets-scan", "no-secrets.sarif") // Act var err error - secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, secretsDocsUrlSuffix) + secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.resultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, secretsDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, secretScanManager.secretsScannerResults) { @@ -84,12 +89,12 @@ func TestParseResults_ResultsContainSecrets(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - secretScanManager := newSecretsScanManager(scanner) - secretScanManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "secrets-scan", "contain-secrets.sarif") + secretScanManager := newSecretsScanManager(scanner, "temoDirPath") + secretScanManager.resultsFileName = filepath.Join(jas.GetTestDataPath(), "secrets-scan", "contain-secrets.sarif") // Act var err error - secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, secretsDocsUrlSuffix) + secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.resultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, secretsDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, secretScanManager.secretsScannerResults) { @@ -107,11 +112,12 @@ func TestGetSecretsScanResults_AnalyzerManagerReturnsError(t *testing.T) { scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() - secretsResults, err := RunSecretsScan(scanner) + var scanResults utils.ExtendedScanResults + err := RunSecretsScan(scanner, &scanResults) assert.Error(t, err) assert.ErrorContains(t, err, "failed to run Secrets scan") - assert.Nil(t, secretsResults) + assert.Nil(t, scanResults.SecretsScanResults) } func TestHideSecret(t *testing.T) { diff --git a/commands/audit/jasrunner.go b/commands/audit/jasrunner.go index 434d9044..2934dbd1 100644 --- a/commands/audit/jasrunner.go +++ b/commands/audit/jasrunner.go @@ -11,6 +11,7 @@ import ( "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-client-go/utils/io" "github.com/jfrog/jfrog-client-go/utils/log" + "golang.org/x/sync/errgroup" ) func runJasScannersAndSetResults(scanResults *utils.Results, directDependencies []string, @@ -27,34 +28,57 @@ func runJasScannersAndSetResults(scanResults *utils.Results, directDependencies cleanup := scanner.ScannerDirCleanupFunc err = errors.Join(err, cleanup()) }() - if progress != nil { - progress.SetHeadlineMsg("Running applicability scanning") - } - scanResults.ExtendedScanResults.ApplicabilityScanResults, err = applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan) - if err != nil { - return - } + + errGroup := new(errgroup.Group) + errGroup.Go(func() error { + if progress != nil { + progress.SetHeadlineMsg("Running applicability scanning") + } + err = applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan, scanResults.ExtendedScanResults) + if err != nil { + return err + } + return nil + }) // Don't execute other scanners when scanning third party dependencies. if thirdPartyApplicabilityScan { + if err = errGroup.Wait(); err != nil { + return err + } return } - if progress != nil { - progress.SetHeadlineMsg("Running secrets scanning") - } - scanResults.ExtendedScanResults.SecretsScanResults, err = secrets.RunSecretsScan(scanner) - if err != nil { - return - } - if progress != nil { - progress.SetHeadlineMsg("Running IaC scanning") - } - scanResults.ExtendedScanResults.IacScanResults, err = iac.RunIacScan(scanner) - if err != nil { - return - } - if progress != nil { - progress.SetHeadlineMsg("Running SAST scanning") + errGroup.Go(func() error { + if progress != nil { + progress.SetHeadlineMsg("Running secrets scanning") + } + err = secrets.RunSecretsScan(scanner, scanResults.ExtendedScanResults) + if err != nil { + return err + } + return nil + }) + errGroup.Go(func() error { + if progress != nil { + progress.SetHeadlineMsg("Running IaC scanning") + } + err = iac.RunIacScan(scanner, scanResults.ExtendedScanResults) + if err != nil { + return err + } + return nil + }) + errGroup.Go(func() error { + if progress != nil { + progress.SetHeadlineMsg("Running SAST scanning") + } + err = sast.RunSastScan(scanner, scanResults.ExtendedScanResults) + if err != nil { + return err + } + return nil + }) + if err = errGroup.Wait(); err != nil { + return err } - scanResults.ExtendedScanResults.SastScanResults, err = sast.RunSastScan(scanner) - return + return err } diff --git a/go.sum b/go.sum index d01e3f1a..c1550f43 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240218125754-97305051f203 h1:zPKW github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240218125754-97305051f203/go.mod h1:+azTP7jL4ebGF//oVz+kCsLOIFxBp/19McVB0KHwT8U= github.com/jfrog/jfrog-client-go v1.28.1-0.20240218093454-1c352a93c23d h1:59C2AOjdp5Wp/WoeQZkOGbAtKnhY1a9YhrnAdeVbVYs= github.com/jfrog/jfrog-client-go v1.28.1-0.20240218093454-1c352a93c23d/go.mod h1:fV5wrs86ihQkFKfMKpGxMbNf3mbVT4LUf320C1T9C2M= +github.com/jfrog/jfrog-client-go v1.36.1 h1:22Ucy5XdEP1yHEjbN8zOt2dZys5rbwcwhC3l3pcOdf4= +github.com/jfrog/jfrog-client-go v1.36.1/go.mod h1:y1WF6eiZ7V2DortiwjpMEicEH6NIJH+hOXI5QI2W3NU= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= diff --git a/tests/testdata/projects/package-managers/npm/npm-project/.jfrog/jfrog-cli.conf.v6 b/tests/testdata/projects/package-managers/npm/npm-project/.jfrog/jfrog-cli.conf.v6 index 9d0a2fb5..d680e118 100644 --- a/tests/testdata/projects/package-managers/npm/npm-project/.jfrog/jfrog-cli.conf.v6 +++ b/tests/testdata/projects/package-managers/npm/npm-project/.jfrog/jfrog-cli.conf.v6 @@ -1,8 +1,8 @@ { "servers": [ { - "url": "http://127.0.0.1:62857/", - "artifactoryUrl": "http://127.0.0.1:62857/", + "url": "http://127.0.0.1:52017/", + "artifactoryUrl": "http://127.0.0.1:52017/", "user": "admin", "password": "password", "serverId": "test" From 4f8d68dfff3160e552d176f31cbe4581cc35ded1 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 21 Mar 2024 11:58:50 +0200 Subject: [PATCH 02/59] running scanners in parallel - second commit --- commands/audit/audit.go | 65 +++++---- .../jas/applicability/applicabilitymanager.go | 19 ++- .../applicabilitymanager_test.go | 3 +- commands/audit/jas/common.go | 28 ++-- commands/audit/jas/commons_test.go | 4 +- commands/audit/jas/iac/iacscanner.go | 11 +- commands/audit/jas/sast/sastscanner.go | 11 +- commands/audit/jas/secrets/secretsscanner.go | 11 +- .../audit/jas/secrets/secretsscanner_test.go | 3 +- commands/audit/jasrunner.go | 123 ++++++++++++------ commands/audit/jasrunner_test.go | 18 ++- commands/audit/scarunner.go | 69 ++++++---- commands/scan/buildscan.go | 2 +- commands/scan/scan.go | 2 +- utils/parallel_runner.go | 27 ++++ utils/results.go | 2 +- utils/resultwriter.go | 2 +- utils/resultwriter_test.go | 3 +- 18 files changed, 260 insertions(+), 143 deletions(-) create mode 100644 utils/parallel_runner.go diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 4b9d5f4b..917e32a3 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -2,18 +2,17 @@ package audit import ( "errors" - "github.com/jfrog/jfrog-cli-security/scangraph" - "os" - + "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-cli-security/scangraph" "github.com/jfrog/jfrog-cli-security/utils" + xrayutils "github.com/jfrog/jfrog-cli-security/utils" clientutils "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/io" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray" "github.com/jfrog/jfrog-client-go/xray/services" - "golang.org/x/sync/errgroup" - - xrayutils "github.com/jfrog/jfrog-cli-security/utils" + "os" ) type AuditCommand struct { @@ -148,7 +147,6 @@ func (auditCmd *AuditCommand) CommandName() string { func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) { // Initialize Results struct results = xrayutils.NewAuditResults() - serverDetails, err := auditParams.ServerDetails() if err != nil { return @@ -166,12 +164,6 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) return } - errGroup := new(errgroup.Group) - if results.ExtendedScanResults.EntitledForJas { - // Download (if needed) the analyzer manager in a background routine. - errGroup.Go(utils.DownloadAnalyzerManagerIfNeeded) - } - if auditParams.xrayGraphScanParams.XscGitInfoContext != nil { if err = xrayutils.SendXscGitInfoRequestIfEnabled(auditParams.xrayGraphScanParams, xrayManager); err != nil { return nil, err @@ -179,18 +171,33 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.xrayGraphScanParams.MultiScanId } - // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. - results.ScaError = runScaScan(auditParams, results) - - // Wait for the Download of the AnalyzerManager to complete. - if err = errGroup.Wait(); err != nil { - err = errors.New("failed while trying to get Analyzer Manager: " + err.Error()) + auditParallelRunner := utils.CreateAuditParallelRunner() + if results.ExtendedScanResults.EntitledForJas { + // Download (if needed) the analyzer manager and run scanners. + auditParallelRunner.JasWg.Add(1) + log.Debug("added 1 am task") + _, err = auditParallelRunner.Runner.AddTaskWithError(func(_ int) error { + return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, auditParams.DirectDependencies(), serverDetails, + auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan, auditParams) + }, auditParallelRunner.ErrorsQueue.AddError) } - // Run scanners only if the user is entitled for Advanced Security - if results.ExtendedScanResults.EntitledForJas { - results.JasError = runJasScannersAndSetResults(results, auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan) + // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. + err = runScaScan(auditParallelRunner, auditParams, results) + if err != nil { + return } + go func() { + auditParallelRunner.JasWg.Wait() + log.Debug("finishing am and scanners") + auditParallelRunner.ScaScansWg.Wait() + log.Debug("finishing sca") + auditParallelRunner.Runner.Done() + log.Debug("done total run") + }() + auditParallelRunner.Runner.Run() + log.Debug("finish running") + //auditParallelRunner.ErrorsQueue.GetError() return } @@ -202,3 +209,17 @@ func isEntitledForJas(xrayManager *xray.XrayServicesManager, xrayVersion string) entitled, err = xrayManager.IsEntitled(xrayutils.ApplicabilityFeatureId) return } + +func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.AuditParallelRunner, scanResults *utils.Results, directDependencies []string, + serverDetails *config.ServerDetails, workingDirs []string, progress io.ProgressMgr, thirdPartyApplicabilityScan bool, auditParams *AuditParams) (err error) { + defer func() { + log.Debug("remove 1 am task") + auditParallelRunner.JasWg.Done() + }() + err = utils.DownloadAnalyzerManagerIfNeeded() + if err != nil { + return + } + err = RunJasScannersAndSetResults(auditParallelRunner, scanResults, directDependencies, serverDetails, workingDirs, progress, thirdPartyApplicabilityScan, auditParams) + return +} diff --git a/commands/audit/jas/applicability/applicabilitymanager.go b/commands/audit/jas/applicability/applicabilitymanager.go index 1c65bdac..2ca4ec41 100644 --- a/commands/audit/jas/applicability/applicabilitymanager.go +++ b/commands/audit/jas/applicability/applicabilitymanager.go @@ -41,8 +41,8 @@ type ApplicabilityScanManager struct { // map[string]string: A map containing the applicability result of each XRAY CVE. // bool: true if the user is entitled to the applicability scan, false otherwise. // error: An error object (if any). -func RunApplicabilityScan(xrayResults []services.ScanResponse, directDependencies []string, - scannedTechnologies []coreutils.Technology, scanner *jas.JasScanner, thirdPartyContextualAnalysis bool, ExtendedScanResults *utils.ExtendedScanResults) (err error) { +func RunApplicabilityScan(auditParallelRunner *utils.AuditParallelRunner, xrayResults []services.ScanResponse, directDependencies []string, scannedTechnologies []coreutils.Technology, + scanner *jas.JasScanner, thirdPartyContextualAnalysis bool, ExtendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)); err != nil { return @@ -52,11 +52,13 @@ func RunApplicabilityScan(xrayResults []services.ScanResponse, directDependencie log.Debug("The technologies that have been scanned are currently not supported for contextual analysis scanning, or we couldn't find any vulnerable dependencies. Skipping....") return } - if err = applicabilityScanManager.scanner.Run(applicabilityScanManager); err != nil { + if err = applicabilityScanManager.scanner.Run(applicabilityScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Applicability, err) return } + auditParallelRunner.Mu.Lock() ExtendedScanResults.ApplicabilityScanResults = applicabilityScanManager.applicabilityScanResults + auditParallelRunner.Mu.Unlock() return } @@ -117,9 +119,6 @@ func isDirectComponents(components []string, directDependencies []string) bool { } func (asm *ApplicabilityScanManager) Run(module jfrogappsconfig.Module) (err error) { - if jas.ShouldSkipScanner(module, utils.Applicability) { - return - } if len(asm.scanner.JFrogAppsConfig.Modules) > 1 { log.Info("Running applicability scanning in the", module.SourceRoot, "directory...") } else { @@ -200,3 +199,11 @@ func removeElementFromSlice(skipDirs []string, element string) []string { } return slices.Delete(skipDirs, deleteIndex, deleteIndex+1) } + +func convertDirectDependecies(directDependencies []*string) []string { + var stringsBack []string + for _, d := range directDependencies { + stringsBack = append(stringsBack, *d) + } + return stringsBack +} diff --git a/commands/audit/jas/applicability/applicabilitymanager_test.go b/commands/audit/jas/applicability/applicabilitymanager_test.go index cbf84716..246867cd 100644 --- a/commands/audit/jas/applicability/applicabilitymanager_test.go +++ b/commands/audit/jas/applicability/applicabilitymanager_test.go @@ -157,11 +157,12 @@ func TestNewApplicabilityScanManager_VulnerabilitiesDontExist(t *testing.T) { } func TestApplicabilityScanManager_ShouldRun_TechnologiesNotEligibleForScan(t *testing.T) { + auditParallelRunnerForTest := utils.NewAuditParallelRunner() scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() var results utils.ExtendedScanResults - err := RunApplicabilityScan(jas.FakeBasicXrayResults, mockDirectDependencies, []coreutils.Technology{coreutils.Nuget, coreutils.Go}, scanner, false, &results) + err := RunApplicabilityScan(&auditParallelRunnerForTest, jas.FakeBasicXrayResults, mockDirectDependencies, []coreutils.Technology{coreutils.Nuget, coreutils.Go}, scanner, false, &results, scanner.JFrogAppsConfig.Modules[0]) // Assert assert.Nil(t, results.ApplicabilityScanResults) diff --git a/commands/audit/jas/common.go b/commands/audit/jas/common.go index efc8d1ca..3b47ebf4 100644 --- a/commands/audit/jas/common.go +++ b/commands/audit/jas/common.go @@ -3,10 +3,12 @@ package jas import ( "errors" "fmt" + "golang.org/x/exp/rand" "os" "path/filepath" "strings" "testing" + "time" "unicode" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" @@ -62,11 +64,11 @@ func NewJasScanner(workingDirs []string, serverDetails *config.ServerDetails) (s return fileutils.RemoveTempDir(tempDir) } scanner.ServerDetails = serverDetails - scanner.JFrogAppsConfig, err = createJFrogAppsConfig(workingDirs) + scanner.JFrogAppsConfig, err = CreateJFrogAppsConfig(workingDirs) return } -func createJFrogAppsConfig(workingDirs []string) (*jfrogappsconfig.JFrogAppsConfig, error) { +func CreateJFrogAppsConfig(workingDirs []string) (*jfrogappsconfig.JFrogAppsConfig, error) { if jfrogAppsConfig, err := jfrogappsconfig.LoadConfigIfExist(); err != nil { return nil, errorutils.CheckError(err) } else if jfrogAppsConfig != nil { @@ -90,14 +92,12 @@ type ScannerCmd interface { Run(module jfrogappsconfig.Module) (err error) } -func (a *JasScanner) Run(scannerCmd ScannerCmd) (err error) { - for _, module := range a.JFrogAppsConfig.Modules { - func() { - if err = scannerCmd.Run(module); err != nil { - return - } - }() - } +func (a *JasScanner) Run(scannerCmd ScannerCmd, module jfrogappsconfig.Module) (err error) { + func() { + if err = scannerCmd.Run(module); err != nil { + return + } + }() return } @@ -274,7 +274,13 @@ func CreateScannerTempDirectory(scanner *JasScanner, scanType string) (string, e if scanner.TempDir == "" { return "", errors.New("scanner temp dir cannot be created in an empty base dir") } - scannerTempDir := scanner.TempDir + "/" + scanType + rand.Seed(uint64(time.Now().UnixNano())) + randomString := "" + for i := 0; i < 4; i++ { + randomDigit := rand.Intn(10) + randomString += fmt.Sprintf("%d", randomDigit) + } + scannerTempDir := scanner.TempDir + "/" + scanType + "_" + randomString err := os.MkdirAll(scannerTempDir, 0777) if err != nil { return "", err diff --git a/commands/audit/jas/commons_test.go b/commands/audit/jas/commons_test.go index fb7834e5..64fca31a 100644 --- a/commands/audit/jas/commons_test.go +++ b/commands/audit/jas/commons_test.go @@ -27,7 +27,7 @@ func TestCreateJFrogAppsConfig(t *testing.T) { for _, testCase := range createJFrogAppsConfigCases { t.Run(fmt.Sprintf("%v", testCase.workingDirs), func(t *testing.T) { - jfrogAppsConfig, err := createJFrogAppsConfig(testCase.workingDirs) + jfrogAppsConfig, err := CreateJFrogAppsConfig(testCase.workingDirs) assert.NoError(t, err) assert.NotNil(t, jfrogAppsConfig) if len(testCase.workingDirs) == 0 { @@ -49,7 +49,7 @@ func TestCreateJFrogAppsConfigWithConfig(t *testing.T) { chdirCallback := clientTestUtils.ChangeDirWithCallback(t, wd, "testdata") defer chdirCallback() - jfrogAppsConfig, err := createJFrogAppsConfig([]string{}) + jfrogAppsConfig, err := CreateJFrogAppsConfig([]string{}) assert.NoError(t, err) assert.NotNil(t, jfrogAppsConfig) assert.Equal(t, "1.0", jfrogAppsConfig.Version) diff --git a/commands/audit/jas/iac/iacscanner.go b/commands/audit/jas/iac/iacscanner.go index d8eda92b..5f133f1a 100644 --- a/commands/audit/jas/iac/iacscanner.go +++ b/commands/audit/jas/iac/iacscanner.go @@ -32,14 +32,14 @@ type IacScanManager struct { // []utils.SourceCodeScanResult: a list of the iac violations that were found. // bool: true if the user is entitled to iac scan, false otherwise. // error: An error object (if any). -func RunIacScan(scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults) (err error) { +func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.IaC)); err != nil { return } iacScanManager := newIacScanManager(scanner, scannerTempDir) log.Info("Running IaC scanning...") - if err = iacScanManager.scanner.Run(iacScanManager); err != nil { + if err = iacScanManager.scanner.Run(iacScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.IaC, err) return } @@ -47,7 +47,9 @@ func RunIacScan(scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScan log.Info("Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities") } results := iacScanManager.iacScannerResults - ExtendedScanResults.IacScanResults = results + auditParallelRunner.Mu.Lock() + ExtendedScanResults.IacScanResults = append(ExtendedScanResults.IacScanResults, results...) + auditParallelRunner.Mu.Unlock() return } @@ -60,9 +62,6 @@ func newIacScanManager(scanner *jas.JasScanner, scannerTempDir string) (manager } func (iac *IacScanManager) Run(module jfrogappsconfig.Module) (err error) { - if jas.ShouldSkipScanner(module, utils.IaC) { - return - } if err = iac.createConfigFile(module); err != nil { return } diff --git a/commands/audit/jas/sast/sastscanner.go b/commands/audit/jas/sast/sastscanner.go index ce47ad16..06f2739a 100644 --- a/commands/audit/jas/sast/sastscanner.go +++ b/commands/audit/jas/sast/sastscanner.go @@ -24,14 +24,14 @@ type SastScanManager struct { resultsFileName string } -func RunSastScan(scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults) (err error) { +func RunSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Sast)); err != nil { return } sastScanManager := newSastScanManager(scanner, scannerTempDir) log.Info("Running SAST scanning...") - if err = sastScanManager.scanner.Run(sastScanManager); err != nil { + if err = sastScanManager.scanner.Run(sastScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Sast, err) return } @@ -39,7 +39,9 @@ func RunSastScan(scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedSca log.Info("Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities") } results := sastScanManager.sastScannerResults - ExtendedScanResults.SastScanResults = results + auditParallelRunner.Mu.Lock() + ExtendedScanResults.SastScanResults = append(ExtendedScanResults.SastScanResults, results...) + auditParallelRunner.Mu.Unlock() return } @@ -52,9 +54,6 @@ func newSastScanManager(scanner *jas.JasScanner, scannerTempDir string) (manager } func (ssm *SastScanManager) Run(module jfrogappsconfig.Module) (err error) { - if jas.ShouldSkipScanner(module, utils.Sast) { - return - } if err = ssm.createConfigFile(module); err != nil { return } diff --git a/commands/audit/jas/secrets/secretsscanner.go b/commands/audit/jas/secrets/secretsscanner.go index 96d622e5..0a5865d3 100644 --- a/commands/audit/jas/secrets/secretsscanner.go +++ b/commands/audit/jas/secrets/secretsscanner.go @@ -31,14 +31,14 @@ type SecretScanManager struct { // Return values: // []utils.IacOrSecretResult: a list of the secrets that were found. // error: An error object (if any). -func RunSecretsScan(scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults) (err error) { +func RunSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Secrets)); err != nil { return } secretScanManager := newSecretsScanManager(scanner, scannerTempDir) log.Info("Running secrets scanning...") - if err = secretScanManager.scanner.Run(secretScanManager); err != nil { + if err = secretScanManager.scanner.Run(secretScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Secrets, err) return } @@ -46,7 +46,9 @@ func RunSecretsScan(scanner *jas.JasScanner, ExtendedScanResults *utils.Extended if len(results) > 0 { log.Info("Found", utils.GetResultsLocationCount(results...), "secrets") } - ExtendedScanResults.SecretsScanResults = results + auditParallelRunner.Mu.Lock() + ExtendedScanResults.SecretsScanResults = append(ExtendedScanResults.SecretsScanResults, results...) + auditParallelRunner.Mu.Unlock() return } @@ -60,9 +62,6 @@ func newSecretsScanManager(scanner *jas.JasScanner, scannerTempDir string) (mana } func (ssm *SecretScanManager) Run(module jfrogappsconfig.Module) (err error) { - if jas.ShouldSkipScanner(module, utils.Secrets) { - return - } if err = ssm.createConfigFile(module); err != nil { return } diff --git a/commands/audit/jas/secrets/secretsscanner_test.go b/commands/audit/jas/secrets/secretsscanner_test.go index 9b1ffa8d..2be98a48 100644 --- a/commands/audit/jas/secrets/secretsscanner_test.go +++ b/commands/audit/jas/secrets/secretsscanner_test.go @@ -109,11 +109,12 @@ func TestParseResults_ResultsContainSecrets(t *testing.T) { } func TestGetSecretsScanResults_AnalyzerManagerReturnsError(t *testing.T) { + auditParallelRunnerForTest := utils.NewAuditParallelRunner() scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() var scanResults utils.ExtendedScanResults - err := RunSecretsScan(scanner, &scanResults) + err := RunSecretsScan(&auditParallelRunnerForTest, scanner, &scanResults, scanner.JFrogAppsConfig.Modules[0]) assert.Error(t, err) assert.ErrorContains(t, err, "failed to run Secrets scan") diff --git a/commands/audit/jasrunner.go b/commands/audit/jasrunner.go index 2934dbd1..5275f211 100644 --- a/commands/audit/jasrunner.go +++ b/commands/audit/jasrunner.go @@ -2,6 +2,8 @@ package audit import ( "errors" + "github.com/jfrog/gofrog/parallel" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" "github.com/jfrog/jfrog-cli-security/commands/audit/jas/applicability" @@ -11,11 +13,10 @@ import ( "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-client-go/utils/io" "github.com/jfrog/jfrog-client-go/utils/log" - "golang.org/x/sync/errgroup" ) -func runJasScannersAndSetResults(scanResults *utils.Results, directDependencies []string, - serverDetails *config.ServerDetails, workingDirs []string, progress io.ProgressMgr, thirdPartyApplicabilityScan bool) (err error) { +func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, scanResults *utils.Results, directDependencies []string, + serverDetails *config.ServerDetails, workingDirs []string, progress io.ProgressMgr, thirdPartyApplicabilityScan bool, auditParams *AuditParams) (err error) { if serverDetails == nil || len(serverDetails.Url) == 0 { log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.") return @@ -25,60 +26,98 @@ func runJasScannersAndSetResults(scanResults *utils.Results, directDependencies return } defer func() { + auditParallelRunner.ScannersWg.Wait() + log.Debug("doing temp dir cleanup") cleanup := scanner.ScannerDirCleanupFunc err = errors.Join(err, cleanup()) }() - errGroup := new(errgroup.Group) - errGroup.Go(func() error { - if progress != nil { - progress.SetHeadlineMsg("Running applicability scanning") - } - err = applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan, scanResults.ExtendedScanResults) - if err != nil { - return err - } - return nil - }) // Don't execute other scanners when scanning third party dependencies. - if thirdPartyApplicabilityScan { - if err = errGroup.Wait(); err != nil { - return err + if !thirdPartyApplicabilityScan { + for _, module := range scanner.JFrogAppsConfig.Modules { + if !jas.ShouldSkipScanner(module, utils.Secrets) { + log.Debug("added secrets scanner task") + auditParallelRunner.ScannersWg.Add(1) + _, err = auditParallelRunner.Runner.AddTaskWithError(runSecretsScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.ErrorsQueue.AddError) + } + if !jas.ShouldSkipScanner(module, utils.IaC) { + log.Debug("added iac scanner task") + auditParallelRunner.ScannersWg.Add(1) + _, err = auditParallelRunner.Runner.AddTaskWithError(runIacScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.ErrorsQueue.AddError) + } + if !jas.ShouldSkipScanner(module, utils.Sast) { + log.Debug("added sast scanner task") + auditParallelRunner.ScannersWg.Add(1) + _, err = auditParallelRunner.Runner.AddTaskWithError(runSastScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.ErrorsQueue.AddError) + } + } + } + + // Wait for sca scan to complete + auditParallelRunner.ScaScansWg.Wait() + for _, module := range scanner.JFrogAppsConfig.Modules { + if !jas.ShouldSkipScanner(module, utils.Applicability) { + log.Debug("added contextual scanner task") + auditParallelRunner.ScannersWg.Add(1) + _, err = auditParallelRunner.Runner.AddTaskWithError(runContextualScan(auditParallelRunner, scanner, thirdPartyApplicabilityScan, progress, scanResults, directDependencies, module, auditParams), auditParallelRunner.ErrorsQueue.AddError) } - return } - errGroup.Go(func() error { + return err +} + +func runSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { + return func(threadId int) (err error) { + defer func() { + log.Debug("remove secrets scanner task") + auditParallelRunner.ScannersWg.Done() + }() if progress != nil { progress.SetHeadlineMsg("Running secrets scanning") } - err = secrets.RunSecretsScan(scanner, scanResults.ExtendedScanResults) - if err != nil { - return err - } - return nil - }) - errGroup.Go(func() error { + err = secrets.RunSecretsScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module) + return + } +} + +func runIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { + return func(threadId int) (err error) { + defer func() { + log.Debug("remove iac scanner task") + auditParallelRunner.ScannersWg.Done() + }() if progress != nil { progress.SetHeadlineMsg("Running IaC scanning") } - err = iac.RunIacScan(scanner, scanResults.ExtendedScanResults) - if err != nil { - return err - } - return nil - }) - errGroup.Go(func() error { + err = iac.RunIacScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module) + return + } +} + +func runSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { + return func(threadId int) (err error) { + defer func() { + log.Debug("remove sast scanner task") + auditParallelRunner.ScannersWg.Done() + }() if progress != nil { - progress.SetHeadlineMsg("Running SAST scanning") + progress.SetHeadlineMsg("Running Sast scanning") } - err = sast.RunSastScan(scanner, scanResults.ExtendedScanResults) - if err != nil { - return err + err = sast.RunSastScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module) + return + } +} + +func runContextualScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, thirdPartyApplicabilityScan bool, progress io.ProgressMgr, + scanResults *utils.Results, directDependencies []string, module jfrogappsconfig.Module, auditParams *AuditParams) parallel.TaskFunc { + return func(threadId int) (err error) { + defer func() { + log.Debug("remove contextual scanner task") + auditParallelRunner.ScannersWg.Done() + }() + if progress != nil { + progress.SetHeadlineMsg("Running applicability scanning") } - return nil - }) - if err = errGroup.Wait(); err != nil { - return err + err = applicability.RunApplicabilityScan(auditParallelRunner, scanResults.GetScaScansXrayResults(), auditParams.DirectDependencies(), scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan, scanResults.ExtendedScanResults, module) + return } - return err } diff --git a/commands/audit/jasrunner_test.go b/commands/audit/jasrunner_test.go index 20b28c8a..67c61808 100644 --- a/commands/audit/jasrunner_test.go +++ b/commands/audit/jasrunner_test.go @@ -1,17 +1,18 @@ package audit import ( + "github.com/jfrog/jfrog-cli-security/commands/audit/jas" "os" "testing" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - "github.com/jfrog/jfrog-cli-security/commands/audit/jas" "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/stretchr/testify/assert" ) func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { + aduit := utils.NewAuditParallelRunner() tmpDir, err := fileutils.CreateTempDir() defer func() { assert.NoError(t, fileutils.RemoveTempDir(tmpDir)) @@ -21,23 +22,26 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { defer func() { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() - scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err = runJasScannersAndSetResults(scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, nil, false) + scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} + err = RunJasScannersAndSetResults(&aduit, scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, nil, false, NewAuditParams()) // Expect error: assert.Error(t, err) } func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { - scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := runJasScannersAndSetResults(scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, nil, false) + auditParallelRunnerForTest := utils.NewAuditParallelRunner() + scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: coreutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} + err := RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, nil, false, NewAuditParams()) assert.NoError(t, err) } func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { + auditParallelRunnerForTest := utils.NewAuditParallelRunner() + assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded()) - scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := runJasScannersAndSetResults(scanResults, []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, nil, false) + scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} + err := RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, nil, false, NewAuditParams()) // Expect error: assert.ErrorContains(t, err, "failed to run Applicability scan") diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 7e2952f0..77f0d57e 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -4,11 +4,9 @@ import ( "encoding/json" "errors" "fmt" - "os" - "time" - "github.com/jfrog/build-info-go/utils/pythonutils" "github.com/jfrog/gofrog/datastructures" + "github.com/jfrog/gofrog/parallel" "github.com/jfrog/jfrog-cli-core/v2/common/project" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -21,6 +19,7 @@ import ( "github.com/jfrog/jfrog-cli-security/commands/audit/sca/python" "github.com/jfrog/jfrog-cli-security/commands/audit/sca/yarn" "github.com/jfrog/jfrog-cli-security/scangraph" + "github.com/jfrog/jfrog-cli-security/utils" xrayutils "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-client-go/artifactory/services/fspatterns" clientutils "github.com/jfrog/jfrog-client-go/utils" @@ -28,11 +27,13 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray/services" xrayCmdUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" + "os" + "time" ) var DefaultExcludePatterns = []string{"*.git*", "*node_modules*", "*target*", "*venv*", "*test*"} -func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { +func runScaScan(auditParallelRunner *utils.AuditParallelRunner, params *AuditParams, results *xrayutils.Results) (err error) { // Prepare currentWorkingDir, err := os.Getwd() if errorutils.CheckError(err) != nil { @@ -61,12 +62,16 @@ func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { for _, scan := range scans { // Run the scan log.Info("Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") - if wdScanErr := executeScaScan(serverDetails, params, scan); wdScanErr != nil { - err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, wdScanErr.Error())) - continue + log.Debug("added 1 sca scan task") + auditParallelRunner.ScaScansWg.Add(1) + _, wdScanErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, params, scan), auditParallelRunner.ErrorsQueue.AddError) + if wdScanErr != nil { + err = fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, wdScanErr.Error()) } // Add the scan to the results - results.ScaResults = append(results.ScaResults, *scan) + auditParallelRunner.Mu.Lock() + results.ScaResults = append(results.ScaResults, scan) + auditParallelRunner.Mu.Unlock() } return } @@ -118,27 +123,35 @@ func getExcludePattern(params *AuditParams, recursive bool) string { // Preform the SCA scan for the given scan information. // This method will change the working directory to the scan's working directory. -func executeScaScan(serverDetails *config.ServerDetails, params *AuditParams, scan *xrayutils.ScaScanResult) (err error) { - // Get the dependency tree for the technology in the working directory. - if err = os.Chdir(scan.WorkingDirectory); err != nil { - return errorutils.CheckError(err) - } - flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology) - if techErr != nil { - return fmt.Errorf("failed while building '%s' dependency tree:\n%s", scan.Technology, techErr.Error()) - } - if flattenTree == nil || len(flattenTree.Nodes) == 0 { - return errorutils.CheckErrorf("no dependencies were found. Please try to build your project and re-run the audit command") - } - // Scan the dependency tree. - scanResults, xrayErr := runScaWithTech(scan.Technology, params, serverDetails, flattenTree, fullDependencyTrees) - if xrayErr != nil { - return fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s", scan.Technology, xrayErr.Error()) +func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetails *config.ServerDetails, params *AuditParams, scan *xrayutils.ScaScanResult) parallel.TaskFunc { + return func(threadId int) (err error) { + defer func() { + log.Debug("remove 1 sca scan task") + auditParallelRunner.ScaScansWg.Done() + }() + // Get the dependency tree for the technology in the working directory. + if err = os.Chdir(scan.WorkingDirectory); err != nil { + return errorutils.CheckError(err) + } + flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology) + if techErr != nil { + return fmt.Errorf("failed while building '%s' dependency tree:\n%s", scan.Technology, techErr.Error()) + } + if flattenTree == nil || len(flattenTree.Nodes) == 0 { + return errorutils.CheckErrorf("no dependencies were found. Please try to build your project and re-run the audit command") + } + // Scan the dependency tree. + scanResults, xrayErr := runScaWithTech(scan.Technology, params, serverDetails, flattenTree, fullDependencyTrees) + if xrayErr != nil { + return fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s", scan.Technology, xrayErr.Error()) + } + scan.IsMultipleRootProject = clientutils.Pointer(len(fullDependencyTrees) > 1) + addThirdPartyDependenciesToParams(params, scan.Technology, flattenTree, fullDependencyTrees) + auditParallelRunner.Mu.Lock() + scan.XrayResults = append(scan.XrayResults, scanResults...) + auditParallelRunner.Mu.Unlock() + return } - scan.IsMultipleRootProject = clientutils.Pointer(len(fullDependencyTrees) > 1) - addThirdPartyDependenciesToParams(params, scan.Technology, flattenTree, fullDependencyTrees) - scan.XrayResults = append(scan.XrayResults, scanResults...) - return } func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []services.ScanResponse, err error) { diff --git a/commands/scan/buildscan.go b/commands/scan/buildscan.go index bbd18d0b..842bd6a7 100644 --- a/commands/scan/buildscan.go +++ b/commands/scan/buildscan.go @@ -130,7 +130,7 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS scanResults := xrutils.NewAuditResults() scanResults.XrayVersion = xrayVersion - scanResults.ScaResults = []xrutils.ScaScanResult{{XrayResults: scanResponse}} + scanResults.ScaResults = []*xrutils.ScaScanResult{{XrayResults: scanResponse}} resultsPrinter := xrutils.NewResultsWriter(scanResults). SetOutputFormat(bsc.outputFormat). diff --git a/commands/scan/scan.go b/commands/scan/scan.go index b21e2ea5..ef44920b 100644 --- a/commands/scan/scan.go +++ b/commands/scan/scan.go @@ -248,7 +248,7 @@ func (scanCmd *ScanCommand) Run() (err error) { scanResults := xrutils.NewAuditResults() scanResults.XrayVersion = xrayVersion - scanResults.ScaResults = []xrutils.ScaScanResult{{XrayResults: flatResults}} + scanResults.ScaResults = []*xrutils.ScaScanResult{{XrayResults: flatResults}} if err = xrutils.NewResultsWriter(scanResults). SetOutputFormat(scanCmd.outputFormat). diff --git a/utils/parallel_runner.go b/utils/parallel_runner.go new file mode 100644 index 00000000..e0ae3148 --- /dev/null +++ b/utils/parallel_runner.go @@ -0,0 +1,27 @@ +package utils + +import ( + "github.com/jfrog/gofrog/parallel" + clientutils "github.com/jfrog/jfrog-client-go/utils" + "sync" +) + +type AuditParallelRunner struct { + Runner parallel.Runner + ErrorsQueue clientutils.ErrorsQueue + Mu sync.Mutex + ScaScansWg sync.WaitGroup // verify that the sca scan routines are done before running contextual scan + ScannersWg sync.WaitGroup // verify that all scanners routines are done before cleaning temp dir + JasWg sync.WaitGroup // verify that downloading analyzer manager and running all scanners are done +} + +func NewAuditParallelRunner() AuditParallelRunner { + return AuditParallelRunner{ + Runner: parallel.NewRunner(3, 20000, false), + } +} + +func CreateAuditParallelRunner() *AuditParallelRunner { + auditParallelRunner := NewAuditParallelRunner() + return &auditParallelRunner +} diff --git a/utils/results.go b/utils/results.go index 9ab671ea..14e69656 100644 --- a/utils/results.go +++ b/utils/results.go @@ -8,7 +8,7 @@ import ( ) type Results struct { - ScaResults []ScaScanResult + ScaResults []*ScaScanResult XrayVersion string ScaError error diff --git a/utils/resultwriter.go b/utils/resultwriter.go index b7dafea4..3589c5d8 100644 --- a/utils/resultwriter.go +++ b/utils/resultwriter.go @@ -482,7 +482,7 @@ func findMaxCVEScore(cves []formats.CveRow) (string, error) { } // Splits scan responses into aggregated lists of violations, vulnerabilities and licenses. -func SplitScanResults(results []ScaScanResult) ([]services.Violation, []services.Vulnerability, []services.License) { +func SplitScanResults(results []*ScaScanResult) ([]services.Violation, []services.Vulnerability, []services.License) { var violations []services.Violation var vulnerabilities []services.Vulnerability var licenses []services.License diff --git a/utils/resultwriter_test.go b/utils/resultwriter_test.go index 87279153..fcf0012a 100644 --- a/utils/resultwriter_test.go +++ b/utils/resultwriter_test.go @@ -365,7 +365,8 @@ func TestConvertXrayScanToSimpleJson(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { results := NewAuditResults() - results.ScaResults = append(results.ScaResults, ScaScanResult{XrayResults: []services.ScanResponse{tc.result}}) + scaScanResult := ScaScanResult{XrayResults: []services.ScanResponse{tc.result}} + results.ScaResults = append(results.ScaResults, &scaScanResult) output, err := ConvertXrayScanToSimpleJson(results, false, tc.includeLicenses, true, tc.allowedLicenses) if assert.NoError(t, err) { assert.ElementsMatch(t, tc.expectedOutput.Vulnerabilities, output.Vulnerabilities) From 0b3a8364d24fd24eeacb112c9e706322da954e55 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 21 Mar 2024 13:11:00 +0200 Subject: [PATCH 03/59] go sec fixes --- commands/audit/audit.go | 2 +- commands/audit/jas/applicability/applicabilitymanager.go | 4 ++-- commands/audit/jas/sast/sastscanner.go | 4 ++-- commands/audit/jas/secrets/secretsscanner.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 1369c2ea..481201db 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -196,7 +196,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) }() auditParallelRunner.Runner.Run() log.Debug("finish running") - //auditParallelRunner.ErrorsQueue.GetError() + // auditParallelRunner.ErrorsQueue.GetError() return } diff --git a/commands/audit/jas/applicability/applicabilitymanager.go b/commands/audit/jas/applicability/applicabilitymanager.go index 2ca4ec41..2f361af5 100644 --- a/commands/audit/jas/applicability/applicabilitymanager.go +++ b/commands/audit/jas/applicability/applicabilitymanager.go @@ -42,7 +42,7 @@ type ApplicabilityScanManager struct { // bool: true if the user is entitled to the applicability scan, false otherwise. // error: An error object (if any). func RunApplicabilityScan(auditParallelRunner *utils.AuditParallelRunner, xrayResults []services.ScanResponse, directDependencies []string, scannedTechnologies []coreutils.Technology, - scanner *jas.JasScanner, thirdPartyContextualAnalysis bool, ExtendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { + scanner *jas.JasScanner, thirdPartyContextualAnalysis bool, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)); err != nil { return @@ -57,7 +57,7 @@ func RunApplicabilityScan(auditParallelRunner *utils.AuditParallelRunner, xrayRe return } auditParallelRunner.Mu.Lock() - ExtendedScanResults.ApplicabilityScanResults = applicabilityScanManager.applicabilityScanResults + extendedScanResults.ApplicabilityScanResults = applicabilityScanManager.applicabilityScanResults auditParallelRunner.Mu.Unlock() return } diff --git a/commands/audit/jas/sast/sastscanner.go b/commands/audit/jas/sast/sastscanner.go index 06f2739a..2359728f 100644 --- a/commands/audit/jas/sast/sastscanner.go +++ b/commands/audit/jas/sast/sastscanner.go @@ -24,7 +24,7 @@ type SastScanManager struct { resultsFileName string } -func RunSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { +func RunSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Sast)); err != nil { return @@ -40,7 +40,7 @@ func RunSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Ja } results := sastScanManager.sastScannerResults auditParallelRunner.Mu.Lock() - ExtendedScanResults.SastScanResults = append(ExtendedScanResults.SastScanResults, results...) + extendedScanResults.SastScanResults = append(extendedScanResults.SastScanResults, results...) auditParallelRunner.Mu.Unlock() return } diff --git a/commands/audit/jas/secrets/secretsscanner.go b/commands/audit/jas/secrets/secretsscanner.go index 0a5865d3..f4f114c0 100644 --- a/commands/audit/jas/secrets/secretsscanner.go +++ b/commands/audit/jas/secrets/secretsscanner.go @@ -31,7 +31,7 @@ type SecretScanManager struct { // Return values: // []utils.IacOrSecretResult: a list of the secrets that were found. // error: An error object (if any). -func RunSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { +func RunSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Secrets)); err != nil { return @@ -47,7 +47,7 @@ func RunSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas log.Info("Found", utils.GetResultsLocationCount(results...), "secrets") } auditParallelRunner.Mu.Lock() - ExtendedScanResults.SecretsScanResults = append(ExtendedScanResults.SecretsScanResults, results...) + extendedScanResults.SecretsScanResults = append(extendedScanResults.SecretsScanResults, results...) auditParallelRunner.Mu.Unlock() return } From ef6ccd7fdcff718000468b007b0e71666241c778 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 31 Mar 2024 13:51:53 +0300 Subject: [PATCH 04/59] third commit --- artifactory_test.go | 4 +-- commands/audit/audit.go | 26 +++++++++--------- .../jas/applicability/applicabilitymanager.go | 11 +++----- commands/audit/jas/common.go | 2 +- commands/audit/jas/iac/iacscanner.go | 7 ++--- commands/audit/jas/sast/sastscanner.go | 7 ++--- commands/audit/jas/secrets/secretsscanner.go | 8 +++--- commands/audit/jasrunner.go | 27 +++++++------------ commands/audit/jasrunner_test.go | 2 +- commands/audit/scarunner.go | 7 +++-- utils/analyzermanager.go | 5 ++-- utils/parallel_runner.go | 10 ++++--- utils/results.go | 2 +- 13 files changed, 56 insertions(+), 62 deletions(-) diff --git a/artifactory_test.go b/artifactory_test.go index 26df2789..3f3ec63d 100644 --- a/artifactory_test.go +++ b/artifactory_test.go @@ -215,7 +215,7 @@ func TestDownloadAnalyzerManagerIfNeeded(t *testing.T) { defer setEnvCallBack() // Download - err := utils.DownloadAnalyzerManagerIfNeeded() + err := utils.DownloadAnalyzerManagerIfNeeded(0) assert.NoError(t, err) // Validate Analyzer manager app & checksum.sh2 file exist @@ -236,7 +236,7 @@ func TestDownloadAnalyzerManagerIfNeeded(t *testing.T) { // Validate no second download occurred firstFileStat, err := os.Stat(amPath) assert.NoError(t, err) - err = utils.DownloadAnalyzerManagerIfNeeded() + err = utils.DownloadAnalyzerManagerIfNeeded(0) assert.NoError(t, err) secondFileStat, err := os.Stat(amPath) assert.NoError(t, err) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 481201db..4bb87617 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -111,7 +111,7 @@ func (auditCmd *AuditCommand) Run() (err error) { messages = []string{coreutils.PrintTitle("The ‘jf audit’ command also supports JFrog Advanced Security features, such as 'Contextual Analysis', 'Secret Detection', 'IaC Scan' and ‘SAST’.\nThis feature isn't enabled on your system. Read more - ") + coreutils.PrintLink("https://jfrog.com/xray/")} } // Print Scan results on all cases except if errors accrued on SCA scan and no security/license issues found. - printScanResults := !(auditResults.ScaError != nil && !auditResults.IsScaIssuesFound()) + printScanResults := !(auditResults.ScansErr != nil && !auditResults.IsScaIssuesFound()) if printScanResults { if err = xrayutils.NewResultsWriter(auditResults). SetIsMultipleRootProject(auditResults.IsMultipleProject()). @@ -125,7 +125,7 @@ func (auditCmd *AuditCommand) Run() (err error) { return } } - if err = errors.Join(auditResults.ScaError, auditResults.JasError); err != nil { + if auditResults.ScansErr != nil { return } @@ -174,11 +174,10 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) if results.ExtendedScanResults.EntitledForJas { // Download (if needed) the analyzer manager and run scanners. auditParallelRunner.JasWg.Add(1) - log.Debug("added 1 am task") - _, err = auditParallelRunner.Runner.AddTaskWithError(func(_ int) error { + _, err = auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, auditParams.DirectDependencies(), serverDetails, - auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan, auditParams) - }, auditParallelRunner.ErrorsQueue.AddError) + auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan, auditParams, threadId) + }, auditParallelRunner.AddErrorToChan) } // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. @@ -188,15 +187,15 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } go func() { auditParallelRunner.JasWg.Wait() - log.Debug("finishing am and scanners") auditParallelRunner.ScaScansWg.Wait() - log.Debug("finishing sca") auditParallelRunner.Runner.Done() - log.Debug("done total run") + }() + go func() { + for e := range auditParallelRunner.ErrorsQueue { + results.ScansErr = errors.Join(err, e) + } }() auditParallelRunner.Runner.Run() - log.Debug("finish running") - // auditParallelRunner.ErrorsQueue.GetError() return } @@ -210,12 +209,11 @@ func isEntitledForJas(xrayManager *xray.XrayServicesManager, xrayVersion string) } func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.AuditParallelRunner, scanResults *utils.Results, directDependencies []string, - serverDetails *config.ServerDetails, workingDirs []string, progress io.ProgressMgr, thirdPartyApplicabilityScan bool, auditParams *AuditParams) (err error) { + serverDetails *config.ServerDetails, workingDirs []string, progress io.ProgressMgr, thirdPartyApplicabilityScan bool, auditParams *AuditParams, threadId int) (err error) { defer func() { - log.Debug("remove 1 am task") auditParallelRunner.JasWg.Done() }() - err = utils.DownloadAnalyzerManagerIfNeeded() + err = utils.DownloadAnalyzerManagerIfNeeded(threadId) if err != nil { return } diff --git a/commands/audit/jas/applicability/applicabilitymanager.go b/commands/audit/jas/applicability/applicabilitymanager.go index 2f361af5..07b8394b 100644 --- a/commands/audit/jas/applicability/applicabilitymanager.go +++ b/commands/audit/jas/applicability/applicabilitymanager.go @@ -4,6 +4,7 @@ import ( jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" "path/filepath" + "strconv" "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -42,16 +43,17 @@ type ApplicabilityScanManager struct { // bool: true if the user is entitled to the applicability scan, false otherwise. // error: An error object (if any). func RunApplicabilityScan(auditParallelRunner *utils.AuditParallelRunner, xrayResults []services.ScanResponse, directDependencies []string, scannedTechnologies []coreutils.Technology, - scanner *jas.JasScanner, thirdPartyContextualAnalysis bool, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { + scanner *jas.JasScanner, thirdPartyContextualAnalysis bool, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module, threadId int) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)); err != nil { return } applicabilityScanManager := newApplicabilityScanManager(xrayResults, directDependencies, scanner, thirdPartyContextualAnalysis, scannerTempDir) if !applicabilityScanManager.shouldRunApplicabilityScan(scannedTechnologies) { - log.Debug("The technologies that have been scanned are currently not supported for contextual analysis scanning, or we couldn't find any vulnerable dependencies. Skipping....") + log.Debug("[thread_id: "+strconv.Itoa(threadId)+"] The technologies that have been scanned are currently not supported for contextual analysis scanning, or we couldn't find any vulnerable dependencies. Skipping....", threadId) return } + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running applicability scanning...", threadId) if err = applicabilityScanManager.scanner.Run(applicabilityScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Applicability, err) return @@ -119,11 +121,6 @@ func isDirectComponents(components []string, directDependencies []string) bool { } func (asm *ApplicabilityScanManager) Run(module jfrogappsconfig.Module) (err error) { - if len(asm.scanner.JFrogAppsConfig.Modules) > 1 { - log.Info("Running applicability scanning in the", module.SourceRoot, "directory...") - } else { - log.Info("Running applicability scanning...") - } if err = asm.createConfigFile(module); err != nil { return } diff --git a/commands/audit/jas/common.go b/commands/audit/jas/common.go index 3b47ebf4..f3515e42 100644 --- a/commands/audit/jas/common.go +++ b/commands/audit/jas/common.go @@ -223,7 +223,7 @@ var FakeBasicXrayResults = []services.ScanResponse{ } func InitJasTest(t *testing.T, workingDirs ...string) (*JasScanner, func()) { - assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded()) + assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) scanner, err := NewJasScanner(workingDirs, &FakeServerDetails) assert.NoError(t, err) return scanner, func() { diff --git a/commands/audit/jas/iac/iacscanner.go b/commands/audit/jas/iac/iacscanner.go index 5f133f1a..3f1931b4 100644 --- a/commands/audit/jas/iac/iacscanner.go +++ b/commands/audit/jas/iac/iacscanner.go @@ -2,6 +2,7 @@ package iac import ( "path/filepath" + "strconv" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" @@ -32,19 +33,19 @@ type IacScanManager struct { // []utils.SourceCodeScanResult: a list of the iac violations that were found. // bool: true if the user is entitled to iac scan, false otherwise. // error: An error object (if any). -func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { +func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module, threadId int) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.IaC)); err != nil { return } iacScanManager := newIacScanManager(scanner, scannerTempDir) - log.Info("Running IaC scanning...") + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running IaC scanning...", threadId) if err = iacScanManager.scanner.Run(iacScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.IaC, err) return } if len(iacScanManager.iacScannerResults) > 0 { - log.Info("Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities") + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities", threadId) } results := iacScanManager.iacScannerResults auditParallelRunner.Mu.Lock() diff --git a/commands/audit/jas/sast/sastscanner.go b/commands/audit/jas/sast/sastscanner.go index 2359728f..a99066f6 100644 --- a/commands/audit/jas/sast/sastscanner.go +++ b/commands/audit/jas/sast/sastscanner.go @@ -9,6 +9,7 @@ import ( "github.com/owenrumney/go-sarif/v2/sarif" "golang.org/x/exp/maps" "path/filepath" + "strconv" ) const ( @@ -24,19 +25,19 @@ type SastScanManager struct { resultsFileName string } -func RunSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { +func RunSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module, threadId int) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Sast)); err != nil { return } sastScanManager := newSastScanManager(scanner, scannerTempDir) - log.Info("Running SAST scanning...") + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running SAST scanning...", threadId) if err = sastScanManager.scanner.Run(sastScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Sast, err) return } if len(sastScanManager.sastScannerResults) > 0 { - log.Info("Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities") + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities", threadId) } results := sastScanManager.sastScannerResults auditParallelRunner.Mu.Lock() diff --git a/commands/audit/jas/secrets/secretsscanner.go b/commands/audit/jas/secrets/secretsscanner.go index f4f114c0..11c7fa6e 100644 --- a/commands/audit/jas/secrets/secretsscanner.go +++ b/commands/audit/jas/secrets/secretsscanner.go @@ -2,6 +2,7 @@ package secrets import ( "path/filepath" + "strconv" "strings" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" @@ -31,20 +32,21 @@ type SecretScanManager struct { // Return values: // []utils.IacOrSecretResult: a list of the secrets that were found. // error: An error object (if any). -func RunSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) (err error) { +func RunSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, + module jfrogappsconfig.Module, threadId int) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Secrets)); err != nil { return } secretScanManager := newSecretsScanManager(scanner, scannerTempDir) - log.Info("Running secrets scanning...") + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running secrets scanning...", threadId) if err = secretScanManager.scanner.Run(secretScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Secrets, err) return } results := secretScanManager.secretsScannerResults if len(results) > 0 { - log.Info("Found", utils.GetResultsLocationCount(results...), "secrets") + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(results...), "secrets", threadId) } auditParallelRunner.Mu.Lock() extendedScanResults.SecretsScanResults = append(extendedScanResults.SecretsScanResults, results...) diff --git a/commands/audit/jasrunner.go b/commands/audit/jasrunner.go index 5275f211..c5a7a228 100644 --- a/commands/audit/jasrunner.go +++ b/commands/audit/jasrunner.go @@ -27,7 +27,6 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, } defer func() { auditParallelRunner.ScannersWg.Wait() - log.Debug("doing temp dir cleanup") cleanup := scanner.ScannerDirCleanupFunc err = errors.Join(err, cleanup()) }() @@ -36,19 +35,16 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { if !jas.ShouldSkipScanner(module, utils.Secrets) { - log.Debug("added secrets scanner task") auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runSecretsScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.ErrorsQueue.AddError) + _, err = auditParallelRunner.Runner.AddTaskWithError(runSecretsScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.AddErrorToChan) } if !jas.ShouldSkipScanner(module, utils.IaC) { - log.Debug("added iac scanner task") auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runIacScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.ErrorsQueue.AddError) + _, err = auditParallelRunner.Runner.AddTaskWithError(runIacScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.AddErrorToChan) } if !jas.ShouldSkipScanner(module, utils.Sast) { - log.Debug("added sast scanner task") auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runSastScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.ErrorsQueue.AddError) + _, err = auditParallelRunner.Runner.AddTaskWithError(runSastScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.AddErrorToChan) } } } @@ -57,9 +53,8 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, auditParallelRunner.ScaScansWg.Wait() for _, module := range scanner.JFrogAppsConfig.Modules { if !jas.ShouldSkipScanner(module, utils.Applicability) { - log.Debug("added contextual scanner task") auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runContextualScan(auditParallelRunner, scanner, thirdPartyApplicabilityScan, progress, scanResults, directDependencies, module, auditParams), auditParallelRunner.ErrorsQueue.AddError) + _, err = auditParallelRunner.Runner.AddTaskWithError(runContextualScan(auditParallelRunner, scanner, thirdPartyApplicabilityScan, progress, scanResults, module, auditParams), auditParallelRunner.AddErrorToChan) } } return err @@ -68,13 +63,12 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, func runSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { - log.Debug("remove secrets scanner task") auditParallelRunner.ScannersWg.Done() }() if progress != nil { progress.SetHeadlineMsg("Running secrets scanning") } - err = secrets.RunSecretsScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module) + err = secrets.RunSecretsScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module, threadId) return } } @@ -82,13 +76,12 @@ func runSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas func runIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { - log.Debug("remove iac scanner task") auditParallelRunner.ScannersWg.Done() }() if progress != nil { progress.SetHeadlineMsg("Running IaC scanning") } - err = iac.RunIacScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module) + err = iac.RunIacScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module, threadId) return } } @@ -96,28 +89,26 @@ func runIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Jas func runSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { - log.Debug("remove sast scanner task") auditParallelRunner.ScannersWg.Done() }() if progress != nil { progress.SetHeadlineMsg("Running Sast scanning") } - err = sast.RunSastScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module) + err = sast.RunSastScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module, threadId) return } } func runContextualScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, thirdPartyApplicabilityScan bool, progress io.ProgressMgr, - scanResults *utils.Results, directDependencies []string, module jfrogappsconfig.Module, auditParams *AuditParams) parallel.TaskFunc { + scanResults *utils.Results, module jfrogappsconfig.Module, auditParams *AuditParams) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { - log.Debug("remove contextual scanner task") auditParallelRunner.ScannersWg.Done() }() if progress != nil { progress.SetHeadlineMsg("Running applicability scanning") } - err = applicability.RunApplicabilityScan(auditParallelRunner, scanResults.GetScaScansXrayResults(), auditParams.DirectDependencies(), scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan, scanResults.ExtendedScanResults, module) + err = applicability.RunApplicabilityScan(auditParallelRunner, scanResults.GetScaScansXrayResults(), auditParams.DirectDependencies(), scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan, scanResults.ExtendedScanResults, module, threadId) return } } diff --git a/commands/audit/jasrunner_test.go b/commands/audit/jasrunner_test.go index 67c61808..4803d078 100644 --- a/commands/audit/jasrunner_test.go +++ b/commands/audit/jasrunner_test.go @@ -38,7 +38,7 @@ func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { auditParallelRunnerForTest := utils.NewAuditParallelRunner() - assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded()) + assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} err := RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, nil, false, NewAuditParams()) diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index fc03bd9c..f626045a 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + "strconv" "github.com/jfrog/build-info-go/utils/pythonutils" "github.com/jfrog/gofrog/datastructures" @@ -60,10 +61,8 @@ func runScaScan(auditParallelRunner *utils.AuditParallelRunner, params *AuditPar }() for _, scan := range scans { // Run the scan - log.Info("Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") - log.Debug("added 1 sca scan task") auditParallelRunner.ScaScansWg.Add(1) - _, wdScanErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, params, scan), auditParallelRunner.ErrorsQueue.AddError) + _, wdScanErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, params, scan), auditParallelRunner.AddErrorToChan) if wdScanErr != nil { err = fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, wdScanErr.Error()) } @@ -116,8 +115,8 @@ func getRequestedDescriptors(params *AuditParams) map[coreutils.Technology][]str // This method will change the working directory to the scan's working directory. func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetails *config.ServerDetails, params *AuditParams, scan *xrayutils.ScaScanResult) parallel.TaskFunc { return func(threadId int) (err error) { + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") defer func() { - log.Debug("remove 1 sca scan task") auditParallelRunner.ScaScansWg.Done() }() // Get the dependency tree for the technology in the working directory. diff --git a/utils/analyzermanager.go b/utils/analyzermanager.go index 5a20be0d..0cb6815e 100644 --- a/utils/analyzermanager.go +++ b/utils/analyzermanager.go @@ -7,6 +7,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "github.com/jfrog/jfrog-cli-core/v2/utils/config" @@ -211,7 +212,7 @@ func ParseAnalyzerManagerError(scanner JasScanType, err error) error { // Download the latest AnalyzerManager executable if not cached locally. // By default, the zip is downloaded directly from jfrog releases. -func DownloadAnalyzerManagerIfNeeded() error { +func DownloadAnalyzerManagerIfNeeded(threadId int) error { downloadPath, err := GetAnalyzerManagerDownloadPath() if err != nil { return err @@ -253,7 +254,7 @@ func DownloadAnalyzerManagerIfNeeded() error { } } // Download & unzip the analyzer manager files - log.Debug("The 'Analyzer Manager' app is not cached locally. Downloading it now...") + log.Debug("[thread_id: " + strconv.Itoa(threadId) + "] The 'Analyzer Manager' app is not cached locally. Downloading it now...") if err = dependencies.DownloadDependency(artDetails, remotePath, filepath.Join(analyzerManagerDir, AnalyzerManagerZipName), true); err != nil { return err } diff --git a/utils/parallel_runner.go b/utils/parallel_runner.go index e0ae3148..270bbc3c 100644 --- a/utils/parallel_runner.go +++ b/utils/parallel_runner.go @@ -2,13 +2,12 @@ package utils import ( "github.com/jfrog/gofrog/parallel" - clientutils "github.com/jfrog/jfrog-client-go/utils" "sync" ) type AuditParallelRunner struct { Runner parallel.Runner - ErrorsQueue clientutils.ErrorsQueue + ErrorsQueue chan error Mu sync.Mutex ScaScansWg sync.WaitGroup // verify that the sca scan routines are done before running contextual scan ScannersWg sync.WaitGroup // verify that all scanners routines are done before cleaning temp dir @@ -17,7 +16,8 @@ type AuditParallelRunner struct { func NewAuditParallelRunner() AuditParallelRunner { return AuditParallelRunner{ - Runner: parallel.NewRunner(3, 20000, false), + Runner: parallel.NewRunner(3, 20000, false), + ErrorsQueue: make(chan error, 100), } } @@ -25,3 +25,7 @@ func CreateAuditParallelRunner() *AuditParallelRunner { auditParallelRunner := NewAuditParallelRunner() return &auditParallelRunner } + +func (apr *AuditParallelRunner) AddErrorToChan(err error) { + apr.ErrorsQueue <- err +} diff --git a/utils/results.go b/utils/results.go index 14e69656..2ffce6f0 100644 --- a/utils/results.go +++ b/utils/results.go @@ -10,7 +10,7 @@ import ( type Results struct { ScaResults []*ScaScanResult XrayVersion string - ScaError error + ScansErr error ExtendedScanResults *ExtendedScanResults JasError error From adabfef2d273649b97aebdd90393040d57bae215 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Wed, 3 Apr 2024 16:48:12 +0300 Subject: [PATCH 05/59] 4th commit --- commands/audit/audit.go | 55 ++++++++++--------- .../jas/applicability/applicabilitymanager.go | 4 +- .../applicabilitymanager_test.go | 2 +- commands/audit/jas/common.go | 14 +++-- commands/audit/jas/iac/iacscanner.go | 4 +- commands/audit/jas/sast/sastscanner.go | 4 +- commands/audit/jas/secrets/secretsscanner.go | 4 +- .../audit/jas/secrets/secretsscanner_test.go | 2 +- commands/audit/jasrunner.go | 33 +++++++---- commands/audit/jasrunner_test.go | 18 ++++-- commands/audit/sca/common.go | 4 +- commands/audit/scarunner.go | 52 +++++++++++------- 12 files changed, 120 insertions(+), 76 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 4bb87617..f7b47fe3 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -2,13 +2,15 @@ package audit import ( "errors" + "fmt" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-cli-security/commands/audit/jas" "github.com/jfrog/jfrog-cli-security/scangraph" "github.com/jfrog/jfrog-cli-security/utils" xrayutils "github.com/jfrog/jfrog-cli-security/utils" clientutils "github.com/jfrog/jfrog-client-go/utils" - "github.com/jfrog/jfrog-client-go/utils/io" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray" "github.com/jfrog/jfrog-client-go/xray/services" @@ -110,23 +112,20 @@ func (auditCmd *AuditCommand) Run() (err error) { if !auditResults.ExtendedScanResults.EntitledForJas { messages = []string{coreutils.PrintTitle("The ‘jf audit’ command also supports JFrog Advanced Security features, such as 'Contextual Analysis', 'Secret Detection', 'IaC Scan' and ‘SAST’.\nThis feature isn't enabled on your system. Read more - ") + coreutils.PrintLink("https://jfrog.com/xray/")} } - // Print Scan results on all cases except if errors accrued on SCA scan and no security/license issues found. - printScanResults := !(auditResults.ScansErr != nil && !auditResults.IsScaIssuesFound()) - if printScanResults { - if err = xrayutils.NewResultsWriter(auditResults). - SetIsMultipleRootProject(auditResults.IsMultipleProject()). - SetIncludeVulnerabilities(auditCmd.IncludeVulnerabilities). - SetIncludeLicenses(auditCmd.IncludeLicenses). - SetOutputFormat(auditCmd.OutputFormat()). - SetPrintExtendedTable(auditCmd.PrintExtendedTable). - SetExtraMessages(messages). - SetScanType(services.Dependency). - PrintScanResults(); err != nil { - return - } + if err = xrayutils.NewResultsWriter(auditResults). + SetIsMultipleRootProject(auditResults.IsMultipleProject()). + SetIncludeVulnerabilities(auditCmd.IncludeVulnerabilities). + SetIncludeLicenses(auditCmd.IncludeLicenses). + SetOutputFormat(auditCmd.OutputFormat()). + SetPrintExtendedTable(auditCmd.PrintExtendedTable). + SetExtraMessages(messages). + SetScanType(services.Dependency). + PrintScanResults(); err != nil { + return } + if auditResults.ScansErr != nil { - return + return auditResults.ScansErr } // Only in case Xray's context was given (!auditCmd.IncludeVulnerabilities), and the user asked to fail the build accordingly, do so. @@ -171,19 +170,25 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } auditParallelRunner := utils.CreateAuditParallelRunner() + JFrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) + if err != nil { + return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) + } if results.ExtendedScanResults.EntitledForJas { // Download (if needed) the analyzer manager and run scanners. auditParallelRunner.JasWg.Add(1) _, err = auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { - return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, auditParams.DirectDependencies(), serverDetails, - auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan, auditParams, threadId) + return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, serverDetails, auditParams, JFrogAppsConfig, threadId) }, auditParallelRunner.AddErrorToChan) + if err != nil { + return results, fmt.Errorf("failed to creat AM and jas scanners task: %s", err.Error()) + } } // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. - err = runScaScan(auditParallelRunner, auditParams, results) - if err != nil { - return + scaScanErr := runScaScan(auditParallelRunner, auditParams, results) + if scaScanErr != nil { + auditParallelRunner.AddErrorToChan(scaScanErr) } go func() { auditParallelRunner.JasWg.Wait() @@ -192,7 +197,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) }() go func() { for e := range auditParallelRunner.ErrorsQueue { - results.ScansErr = errors.Join(err, e) + results.ScansErr = errors.Join(results.ScansErr, e) } }() auditParallelRunner.Runner.Run() @@ -208,8 +213,8 @@ func isEntitledForJas(xrayManager *xray.XrayServicesManager, xrayVersion string) return } -func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.AuditParallelRunner, scanResults *utils.Results, directDependencies []string, - serverDetails *config.ServerDetails, workingDirs []string, progress io.ProgressMgr, thirdPartyApplicabilityScan bool, auditParams *AuditParams, threadId int) (err error) { +func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.AuditParallelRunner, scanResults *utils.Results, + serverDetails *config.ServerDetails, auditParams *AuditParams, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, threadId int) (err error) { defer func() { auditParallelRunner.JasWg.Done() }() @@ -217,6 +222,6 @@ func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.AuditParal if err != nil { return } - err = RunJasScannersAndSetResults(auditParallelRunner, scanResults, directDependencies, serverDetails, workingDirs, progress, thirdPartyApplicabilityScan, auditParams) + err = RunJasScannersAndSetResults(auditParallelRunner, scanResults, serverDetails, auditParams, jfrogAppsConfig) return } diff --git a/commands/audit/jas/applicability/applicabilitymanager.go b/commands/audit/jas/applicability/applicabilitymanager.go index 07b8394b..2c5c9ea2 100644 --- a/commands/audit/jas/applicability/applicabilitymanager.go +++ b/commands/audit/jas/applicability/applicabilitymanager.go @@ -50,10 +50,10 @@ func RunApplicabilityScan(auditParallelRunner *utils.AuditParallelRunner, xrayRe } applicabilityScanManager := newApplicabilityScanManager(xrayResults, directDependencies, scanner, thirdPartyContextualAnalysis, scannerTempDir) if !applicabilityScanManager.shouldRunApplicabilityScan(scannedTechnologies) { - log.Debug("[thread_id: "+strconv.Itoa(threadId)+"] The technologies that have been scanned are currently not supported for contextual analysis scanning, or we couldn't find any vulnerable dependencies. Skipping....", threadId) + log.Debug("[thread_id: " + strconv.Itoa(threadId) + "] The technologies that have been scanned are currently not supported for contextual analysis scanning, or we couldn't find any vulnerable dependencies. Skipping....") return } - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running applicability scanning...", threadId) + log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running applicability scanning...") if err = applicabilityScanManager.scanner.Run(applicabilityScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Applicability, err) return diff --git a/commands/audit/jas/applicability/applicabilitymanager_test.go b/commands/audit/jas/applicability/applicabilitymanager_test.go index 246867cd..45dc1fcc 100644 --- a/commands/audit/jas/applicability/applicabilitymanager_test.go +++ b/commands/audit/jas/applicability/applicabilitymanager_test.go @@ -162,7 +162,7 @@ func TestApplicabilityScanManager_ShouldRun_TechnologiesNotEligibleForScan(t *te defer cleanUp() var results utils.ExtendedScanResults - err := RunApplicabilityScan(&auditParallelRunnerForTest, jas.FakeBasicXrayResults, mockDirectDependencies, []coreutils.Technology{coreutils.Nuget, coreutils.Go}, scanner, false, &results, scanner.JFrogAppsConfig.Modules[0]) + err := RunApplicabilityScan(&auditParallelRunnerForTest, jas.FakeBasicXrayResults, mockDirectDependencies, []coreutils.Technology{coreutils.Nuget, coreutils.Go}, scanner, false, &results, scanner.JFrogAppsConfig.Modules[0], 0) // Assert assert.Nil(t, results.ApplicabilityScanResults) diff --git a/commands/audit/jas/common.go b/commands/audit/jas/common.go index f3515e42..78c7323e 100644 --- a/commands/audit/jas/common.go +++ b/commands/audit/jas/common.go @@ -50,7 +50,7 @@ type JasScanner struct { ScannerDirCleanupFunc func() error } -func NewJasScanner(workingDirs []string, serverDetails *config.ServerDetails) (scanner *JasScanner, err error) { +func NewJasScanner(serverDetails *config.ServerDetails, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig) (scanner *JasScanner, err error) { scanner = &JasScanner{} if scanner.AnalyzerManager.AnalyzerManagerFullPath, err = utils.GetAnalyzerManagerExecutable(); err != nil { return @@ -64,7 +64,7 @@ func NewJasScanner(workingDirs []string, serverDetails *config.ServerDetails) (s return fileutils.RemoveTempDir(tempDir) } scanner.ServerDetails = serverDetails - scanner.JFrogAppsConfig, err = CreateJFrogAppsConfig(workingDirs) + scanner.JFrogAppsConfig = jfrogAppsConfig return } @@ -73,6 +73,12 @@ func CreateJFrogAppsConfig(workingDirs []string) (*jfrogappsconfig.JFrogAppsConf return nil, errorutils.CheckError(err) } else if jfrogAppsConfig != nil { // jfrog-apps-config.yml exist in the workspace + for _, module := range jfrogAppsConfig.Modules { + module.SourceRoot, err = filepath.Abs(module.SourceRoot) + if err != nil { + return nil, errorutils.CheckError(err) + } + } return jfrogAppsConfig, nil } @@ -222,9 +228,9 @@ var FakeBasicXrayResults = []services.ScanResponse{ }, } -func InitJasTest(t *testing.T, workingDirs ...string) (*JasScanner, func()) { +func InitJasTest(t *testing.T) (*JasScanner, func()) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) - scanner, err := NewJasScanner(workingDirs, &FakeServerDetails) + scanner, err := NewJasScanner(&FakeServerDetails, nil) assert.NoError(t, err) return scanner, func() { assert.NoError(t, scanner.ScannerDirCleanupFunc()) diff --git a/commands/audit/jas/iac/iacscanner.go b/commands/audit/jas/iac/iacscanner.go index 3f1931b4..73617008 100644 --- a/commands/audit/jas/iac/iacscanner.go +++ b/commands/audit/jas/iac/iacscanner.go @@ -39,13 +39,13 @@ func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Jas return } iacScanManager := newIacScanManager(scanner, scannerTempDir) - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running IaC scanning...", threadId) + log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running IaC scanning...") if err = iacScanManager.scanner.Run(iacScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.IaC, err) return } if len(iacScanManager.iacScannerResults) > 0 { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities", threadId) + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities") } results := iacScanManager.iacScannerResults auditParallelRunner.Mu.Lock() diff --git a/commands/audit/jas/sast/sastscanner.go b/commands/audit/jas/sast/sastscanner.go index a99066f6..8d8f930d 100644 --- a/commands/audit/jas/sast/sastscanner.go +++ b/commands/audit/jas/sast/sastscanner.go @@ -31,13 +31,13 @@ func RunSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Ja return } sastScanManager := newSastScanManager(scanner, scannerTempDir) - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running SAST scanning...", threadId) + log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running SAST scanning...") if err = sastScanManager.scanner.Run(sastScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Sast, err) return } if len(sastScanManager.sastScannerResults) > 0 { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities", threadId) + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities") } results := sastScanManager.sastScannerResults auditParallelRunner.Mu.Lock() diff --git a/commands/audit/jas/secrets/secretsscanner.go b/commands/audit/jas/secrets/secretsscanner.go index 11c7fa6e..1d2e71bd 100644 --- a/commands/audit/jas/secrets/secretsscanner.go +++ b/commands/audit/jas/secrets/secretsscanner.go @@ -39,14 +39,14 @@ func RunSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas return } secretScanManager := newSecretsScanManager(scanner, scannerTempDir) - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running secrets scanning...", threadId) + log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running secrets scanning...") if err = secretScanManager.scanner.Run(secretScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Secrets, err) return } results := secretScanManager.secretsScannerResults if len(results) > 0 { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(results...), "secrets", threadId) + log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(results...), "secrets") } auditParallelRunner.Mu.Lock() extendedScanResults.SecretsScanResults = append(extendedScanResults.SecretsScanResults, results...) diff --git a/commands/audit/jas/secrets/secretsscanner_test.go b/commands/audit/jas/secrets/secretsscanner_test.go index 2be98a48..6dc193e8 100644 --- a/commands/audit/jas/secrets/secretsscanner_test.go +++ b/commands/audit/jas/secrets/secretsscanner_test.go @@ -114,7 +114,7 @@ func TestGetSecretsScanResults_AnalyzerManagerReturnsError(t *testing.T) { defer cleanUp() var scanResults utils.ExtendedScanResults - err := RunSecretsScan(&auditParallelRunnerForTest, scanner, &scanResults, scanner.JFrogAppsConfig.Modules[0]) + err := RunSecretsScan(&auditParallelRunnerForTest, scanner, &scanResults, scanner.JFrogAppsConfig.Modules[0], 0) assert.Error(t, err) assert.ErrorContains(t, err, "failed to run Secrets scan") diff --git a/commands/audit/jasrunner.go b/commands/audit/jasrunner.go index c5a7a228..170d5b76 100644 --- a/commands/audit/jasrunner.go +++ b/commands/audit/jasrunner.go @@ -2,6 +2,7 @@ package audit import ( "errors" + "fmt" "github.com/jfrog/gofrog/parallel" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/utils/config" @@ -15,13 +16,13 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" ) -func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, scanResults *utils.Results, directDependencies []string, - serverDetails *config.ServerDetails, workingDirs []string, progress io.ProgressMgr, thirdPartyApplicabilityScan bool, auditParams *AuditParams) (err error) { +func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, scanResults *utils.Results, serverDetails *config.ServerDetails, + auditParams *AuditParams, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig) (err error) { if serverDetails == nil || len(serverDetails.Url) == 0 { log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.") return } - scanner, err := jas.NewJasScanner(workingDirs, serverDetails) + scanner, err := jas.NewJasScanner(serverDetails, jfrogAppsConfig) if err != nil { return } @@ -32,19 +33,28 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, }() // Don't execute other scanners when scanning third party dependencies. - if !thirdPartyApplicabilityScan { + if !auditParams.thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { if !jas.ShouldSkipScanner(module, utils.Secrets) { auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runSecretsScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.AddErrorToChan) + _, err = auditParallelRunner.Runner.AddTaskWithError(runSecretsScan(auditParallelRunner, scanner, auditParams.Progress(), scanResults, module), auditParallelRunner.AddErrorToChan) + if err != nil { + return fmt.Errorf("failed to create secrets scan task: %s", err.Error()) + } } if !jas.ShouldSkipScanner(module, utils.IaC) { auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runIacScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.AddErrorToChan) + _, err = auditParallelRunner.Runner.AddTaskWithError(runIacScan(auditParallelRunner, scanner, auditParams.Progress(), scanResults, module), auditParallelRunner.AddErrorToChan) + if err != nil { + return fmt.Errorf("failed to create iac scan task: %s", err.Error()) + } } if !jas.ShouldSkipScanner(module, utils.Sast) { auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runSastScan(auditParallelRunner, scanner, progress, scanResults, module), auditParallelRunner.AddErrorToChan) + _, err = auditParallelRunner.Runner.AddTaskWithError(runSastScan(auditParallelRunner, scanner, auditParams.Progress(), scanResults, module), auditParallelRunner.AddErrorToChan) + if err != nil { + return fmt.Errorf("failed to create sast scan task: %s", err.Error()) + } } } } @@ -54,7 +64,10 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, for _, module := range scanner.JFrogAppsConfig.Modules { if !jas.ShouldSkipScanner(module, utils.Applicability) { auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runContextualScan(auditParallelRunner, scanner, thirdPartyApplicabilityScan, progress, scanResults, module, auditParams), auditParallelRunner.AddErrorToChan) + _, err = auditParallelRunner.Runner.AddTaskWithError(runContextualScan(auditParallelRunner, scanner, auditParams.Progress(), scanResults, module, auditParams), auditParallelRunner.AddErrorToChan) + if err != nil { + return fmt.Errorf("failed to create contextual scan task: %s", err.Error()) + } } } return err @@ -99,7 +112,7 @@ func runSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Ja } } -func runContextualScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, thirdPartyApplicabilityScan bool, progress io.ProgressMgr, +func runContextualScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, scanResults *utils.Results, module jfrogappsconfig.Module, auditParams *AuditParams) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { @@ -108,7 +121,7 @@ func runContextualScan(auditParallelRunner *utils.AuditParallelRunner, scanner * if progress != nil { progress.SetHeadlineMsg("Running applicability scanning") } - err = applicability.RunApplicabilityScan(auditParallelRunner, scanResults.GetScaScansXrayResults(), auditParams.DirectDependencies(), scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan, scanResults.ExtendedScanResults, module, threadId) + err = applicability.RunApplicabilityScan(auditParallelRunner, scanResults.GetScaScansXrayResults(), auditParams.DirectDependencies(), scanResults.GetScaScannedTechnologies(), scanner, auditParams.thirdPartyApplicabilityScan, scanResults.ExtendedScanResults, module, threadId) return } } diff --git a/commands/audit/jasrunner_test.go b/commands/audit/jasrunner_test.go index 4803d078..a3324e3f 100644 --- a/commands/audit/jasrunner_test.go +++ b/commands/audit/jasrunner_test.go @@ -2,6 +2,7 @@ package audit import ( "github.com/jfrog/jfrog-cli-security/commands/audit/jas" + "github.com/jfrog/jfrog-cli-security/commands/audit/jas/applicability" "os" "testing" @@ -12,7 +13,7 @@ import ( ) func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { - aduit := utils.NewAuditParallelRunner() + auditParallelRunnerForTest := utils.NewAuditParallelRunner() tmpDir, err := fileutils.CreateTempDir() defer func() { assert.NoError(t, fileutils.RemoveTempDir(tmpDir)) @@ -23,7 +24,11 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err = RunJasScannersAndSetResults(&aduit, scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, nil, false, NewAuditParams()) + + auditParamsForTest := NewAuditParams().SetThirdPartyApplicabilityScan(false) + auditParamsForTest.AuditBasicParams.AppendDependenciesForApplicabilityScan([]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}) + + err = RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, &jas.FakeServerDetails, auditParamsForTest) // Expect error: assert.Error(t, err) } @@ -31,7 +36,9 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { auditParallelRunnerForTest := utils.NewAuditParallelRunner() scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: coreutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, nil, false, NewAuditParams()) + auditParamsForTest := NewAuditParams().SetThirdPartyApplicabilityScan(false) + auditParamsForTest.AuditBasicParams.AppendDependenciesForApplicabilityScan([]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}) + err := RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, nil, auditParamsForTest) assert.NoError(t, err) } @@ -40,8 +47,9 @@ func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) - scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, nil, false, NewAuditParams()) + scanner, _ := jas.NewJasScanner(nil, &jas.FakeServerDetails) + err := applicability.RunApplicabilityScan(&auditParallelRunnerForTest, jas.FakeBasicXrayResults, []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, + []coreutils.Technology{coreutils.Yarn}, scanner, false, &utils.ExtendedScanResults{}, scanner.JFrogAppsConfig.Modules[0], 0) // Expect error: assert.ErrorContains(t, err, "failed to run Applicability scan") diff --git a/commands/audit/sca/common.go b/commands/audit/sca/common.go index ff93b280..613ef37b 100644 --- a/commands/audit/sca/common.go +++ b/commands/audit/sca/common.go @@ -30,8 +30,8 @@ func GetExcludePattern(params utils.AuditParams) string { return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, params.IsRecursiveScan()) } -func RunXrayDependenciesTreeScanGraph(dependencyTree *xrayUtils.GraphNode, progress ioUtils.ProgressMgr, technology coreutils.Technology, scanGraphParams *scangraph.ScanGraphParams) (results []services.ScanResponse, err error) { - scanGraphParams.XrayGraphScanParams().DependenciesGraph = dependencyTree +func RunXrayDependenciesTreeScanGraph(dependencyTree xrayUtils.GraphNode, progress ioUtils.ProgressMgr, technology coreutils.Technology, scanGraphParams *scangraph.ScanGraphParams) (results []services.ScanResponse, err error) { + scanGraphParams.XrayGraphScanParams().DependenciesGraph = &dependencyTree xscGitInfoContext := scanGraphParams.XrayGraphScanParams().XscGitInfoContext if xscGitInfoContext != nil { xscGitInfoContext.Technologies = []string{technology.String()} diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 70a12300..967ec3a5 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -56,15 +56,23 @@ func runScaScan(auditParallelRunner *utils.AuditParallelRunner, params *AuditPar log.Info(fmt.Sprintf("Preforming %d SCA scans:\n%s", len(scans), scanInfo)) defer func() { - // Make sure to return to the original working directory, executeScaScan may change it + // Make sure to return to the original working directory, building the dependency tree may change it err = errors.Join(err, os.Chdir(currentWorkingDir)) }() for _, scan := range scans { - // Run the scan + // Get the dependency tree for the technology in the working directory. + flattenTree, fullDependencyTrees, bdtErr := buildDependencyTree(scan, params) + if bdtErr != nil { + err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, bdtErr.Error())) + continue + } + // Create sca scan task auditParallelRunner.ScaScansWg.Add(1) - _, wdScanErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, params, scan), auditParallelRunner.AddErrorToChan) - if wdScanErr != nil { - err = fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, wdScanErr.Error()) + _, taskErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, params, scan, *flattenTree, fullDependencyTrees), func(err error) { + auditParallelRunner.AddErrorToChan(fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, err.Error())) + }) + if taskErr != nil { + return fmt.Errorf("failed to creat sca scan task: %s", taskErr.Error()) } // Add the scan to the results auditParallelRunner.Mu.Lock() @@ -112,31 +120,20 @@ func getRequestedDescriptors(params *AuditParams) map[coreutils.Technology][]str } // Preform the SCA scan for the given scan information. -// This method will change the working directory to the scan's working directory. -func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetails *config.ServerDetails, params *AuditParams, scan *xrayutils.ScaScanResult) parallel.TaskFunc { +func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetails *config.ServerDetails, params *AuditParams, + scan *xrayutils.ScaScanResult, flattenTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) parallel.TaskFunc { return func(threadId int) (err error) { log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") defer func() { auditParallelRunner.ScaScansWg.Done() }() - // Get the dependency tree for the technology in the working directory. - if err = os.Chdir(scan.WorkingDirectory); err != nil { - return errorutils.CheckError(err) - } - flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology) - if techErr != nil { - return fmt.Errorf("failed while building '%s' dependency tree:\n%s", scan.Technology, techErr.Error()) - } - if flattenTree == nil || len(flattenTree.Nodes) == 0 { - return errorutils.CheckErrorf("no dependencies were found. Please try to build your project and re-run the audit command") - } // Scan the dependency tree. scanResults, xrayErr := runScaWithTech(scan.Technology, params, serverDetails, flattenTree, fullDependencyTrees) if xrayErr != nil { return fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s", scan.Technology, xrayErr.Error()) } scan.IsMultipleRootProject = clientutils.Pointer(len(fullDependencyTrees) > 1) - addThirdPartyDependenciesToParams(params, scan.Technology, flattenTree, fullDependencyTrees) + addThirdPartyDependenciesToParams(params, scan.Technology, &flattenTree, fullDependencyTrees) auditParallelRunner.Mu.Lock() scan.XrayResults = append(scan.XrayResults, scanResults...) auditParallelRunner.Mu.Unlock() @@ -144,7 +141,7 @@ func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetail } } -func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []services.ScanResponse, err error) { +func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []services.ScanResponse, err error) { scanGraphParams := scangraph.NewScanGraphParams(). SetServerDetails(serverDetails). SetXrayGraphScanParams(params.xrayGraphScanParams). @@ -385,3 +382,18 @@ func logDeps(uniqueDeps any) (err error) { return } + +// This method will change the working directory to the scan's working directory. +func buildDependencyTree(scan *utils.ScaScanResult, params *AuditParams) (*xrayCmdUtils.GraphNode, []*xrayCmdUtils.GraphNode, error) { + if err := os.Chdir(scan.WorkingDirectory); err != nil { + return nil, nil, errorutils.CheckError(err) + } + flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology) + if techErr != nil { + return nil, nil, fmt.Errorf("failed while building '%s' dependency tree:\n%s", scan.Technology, techErr.Error()) + } + if flattenTree == nil || len(flattenTree.Nodes) == 0 { + return nil, nil, errorutils.CheckErrorf("no dependencies were found. Please try to build your project and re-run the audit command") + } + return flattenTree, fullDependencyTrees, nil +} From 834fdaa56cb786940ff1558608d5bae31f5c9433 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Wed, 3 Apr 2024 17:27:47 +0300 Subject: [PATCH 06/59] fix unit test --- commands/audit/jas/common.go | 5 +++-- commands/audit/jasrunner_test.go | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/commands/audit/jas/common.go b/commands/audit/jas/common.go index 78c7323e..d5c33765 100644 --- a/commands/audit/jas/common.go +++ b/commands/audit/jas/common.go @@ -228,9 +228,10 @@ var FakeBasicXrayResults = []services.ScanResponse{ }, } -func InitJasTest(t *testing.T) (*JasScanner, func()) { +func InitJasTest(t *testing.T, workingDirs ...string) (*JasScanner, func()) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) - scanner, err := NewJasScanner(&FakeServerDetails, nil) + jfrogAppsConfigForTest, _ := CreateJFrogAppsConfig(workingDirs) + scanner, err := NewJasScanner(&FakeServerDetails, jfrogAppsConfigForTest) assert.NoError(t, err) return scanner, func() { assert.NoError(t, scanner.ScannerDirCleanupFunc()) diff --git a/commands/audit/jasrunner_test.go b/commands/audit/jasrunner_test.go index a3324e3f..27636af1 100644 --- a/commands/audit/jasrunner_test.go +++ b/commands/audit/jasrunner_test.go @@ -28,7 +28,7 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { auditParamsForTest := NewAuditParams().SetThirdPartyApplicabilityScan(false) auditParamsForTest.AuditBasicParams.AppendDependenciesForApplicabilityScan([]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}) - err = RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, &jas.FakeServerDetails, auditParamsForTest) + err = RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, &jas.FakeServerDetails, auditParamsForTest, nil) // Expect error: assert.Error(t, err) } @@ -38,7 +38,7 @@ func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: coreutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} auditParamsForTest := NewAuditParams().SetThirdPartyApplicabilityScan(false) auditParamsForTest.AuditBasicParams.AppendDependenciesForApplicabilityScan([]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}) - err := RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, nil, auditParamsForTest) + err := RunJasScannersAndSetResults(&auditParallelRunnerForTest, scanResults, nil, auditParamsForTest, nil) assert.NoError(t, err) } @@ -47,9 +47,10 @@ func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) - scanner, _ := jas.NewJasScanner(nil, &jas.FakeServerDetails) + jfrogAppsConfigForTest, _ := jas.CreateJFrogAppsConfig(nil) + scanner, _ := jas.NewJasScanner(&jas.FakeServerDetails, nil) err := applicability.RunApplicabilityScan(&auditParallelRunnerForTest, jas.FakeBasicXrayResults, []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, - []coreutils.Technology{coreutils.Yarn}, scanner, false, &utils.ExtendedScanResults{}, scanner.JFrogAppsConfig.Modules[0], 0) + []coreutils.Technology{coreutils.Yarn}, scanner, false, &utils.ExtendedScanResults{}, jfrogAppsConfigForTest.Modules[0], 0) // Expect error: assert.ErrorContains(t, err, "failed to run Applicability scan") From 2f86fb41d9c06448bc552a138a8baaa8f9a964b6 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 4 Apr 2024 15:41:41 +0300 Subject: [PATCH 07/59] unit tests fix, progress bar changes and adding a threads flag --- cli/docs/flags.go | 7 ++++--- cli/scancommands.go | 12 ++++++++++- commands/audit/audit.go | 20 ++++++++++++++----- commands/audit/auditparams.go | 6 ++++++ .../jas/applicability/applicabilitymanager.go | 8 -------- .../applicabilitymanager_test.go | 3 ++- commands/audit/jas/common.go | 20 ------------------- commands/audit/jas/iac/iacscanner.go | 4 ++-- .../audit/jas/secrets/secretsscanner_test.go | 3 ++- commands/audit/jasrunner.go | 12 ----------- commands/audit/jasrunner_test.go | 7 ++++--- commands/audit/sca/common.go | 6 +----- commands/audit/scarunner.go | 2 +- commands/common.go | 14 +++++++++++++ commands/curation/curationaudit.go | 12 ++--------- utils/parallel_runner.go | 8 ++++---- 16 files changed, 68 insertions(+), 76 deletions(-) create mode 100644 commands/common.go diff --git a/cli/docs/flags.go b/cli/docs/flags.go index 8c285c48..90fee20a 100644 --- a/cli/docs/flags.go +++ b/cli/docs/flags.go @@ -2,13 +2,13 @@ package docs import ( "fmt" + "github.com/jfrog/jfrog-cli-security/commands" "strings" "github.com/jfrog/jfrog-cli-core/v2/common/cliutils" pluginsCommon "github.com/jfrog/jfrog-cli-core/v2/plugins/common" "github.com/jfrog/jfrog-cli-core/v2/plugins/components" "github.com/jfrog/jfrog-cli-security/commands/audit/sca" - "github.com/jfrog/jfrog-cli-security/commands/curation" "github.com/jfrog/jfrog-cli-security/commands/xray/offlineupdate" ) @@ -125,7 +125,8 @@ var commandFlags = map[string][]string{ }, Audit: { url, user, password, accessToken, ServerId, InsecureTls, Project, Watches, RepoPath, Licenses, OutputFormat, ExcludeTestDeps, - useWrapperAudit, DepType, RequirementsFile, Fail, ExtendedTable, WorkingDirs, ExclusionsAudit, Mvn, Gradle, Npm, Pnpm, Yarn, Go, Nuget, Pip, Pipenv, Poetry, MinSeverity, FixableOnly, ThirdPartyContextualAnalysis, + useWrapperAudit, DepType, RequirementsFile, Fail, ExtendedTable, WorkingDirs, ExclusionsAudit, Mvn, Gradle, Npm, + Pnpm, Yarn, Go, Nuget, Pip, Pipenv, Poetry, MinSeverity, FixableOnly, ThirdPartyContextualAnalysis, Threads, }, CurationAudit: { CurationOutput, WorkingDirs, CurationThreads, @@ -219,7 +220,7 @@ var flagsMap = map[string]components.Flag{ components.SetHiddenBoolFlag(), ), RequirementsFile: components.NewStringFlag(RequirementsFile, "[Pip] Defines pip requirements file name. For example: 'requirements.txt'."), - CurationThreads: components.NewStringFlag(Threads, "Number of working threads.", components.WithIntDefaultValue(curation.TotalConcurrentRequests)), + CurationThreads: components.NewStringFlag(Threads, "Number of working threads.", components.WithIntDefaultValue(commands.TotalConcurrentRequests)), CurationOutput: components.NewStringFlag(OutputFormat, "Defines the output format of the command. Acceptable values are: table, json.", components.WithStrDefaultValue("table")), } diff --git a/cli/scancommands.go b/cli/scancommands.go index ebc6aee6..8a1f79b7 100644 --- a/cli/scancommands.go +++ b/cli/scancommands.go @@ -1,6 +1,7 @@ package cli import ( + "github.com/jfrog/jfrog-cli-security/commands" "os" "strings" @@ -328,6 +329,15 @@ func AuditCmd(c *components.Context) error { } } auditCmd.SetTechnologies(technologies) + threadsFlag, err := c.GetIntFlagValue(flags.Threads) + if err != nil { + return err + } + threads, err := commands.DetectNumOfThreads(threadsFlag) + if err != nil { + return err + } + auditCmd.SetParallelScans(threads) return progressbar.ExecWithProgress(auditCmd) } @@ -404,7 +414,7 @@ func CurationCmd(c *components.Context) error { if err != nil { return err } - threads, err := curation.DetectNumOfThreads(threadsFlag) + threads, err := commands.DetectNumOfThreads(threadsFlag) if err != nil { return err } diff --git a/commands/audit/audit.go b/commands/audit/audit.go index f7b47fe3..f4d13753 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -25,6 +25,7 @@ type AuditCommand struct { IncludeLicenses bool Fail bool PrintExtendedTable bool + ParallelScans int AuditParams } @@ -67,6 +68,11 @@ func (auditCmd *AuditCommand) SetPrintExtendedTable(printExtendedTable bool) *Au return auditCmd } +func (auditCmd *AuditCommand) SetParallelScans(threads int) *AuditCommand { + auditCmd.ParallelScans = threads + return auditCmd +} + func (auditCmd *AuditCommand) CreateXrayGraphScanParams() *services.XrayGraphScanParams { params := &services.XrayGraphScanParams{ RepoPath: auditCmd.targetRepoPath, @@ -97,7 +103,8 @@ func (auditCmd *AuditCommand) Run() (err error) { SetMinSeverityFilter(auditCmd.minSeverityFilter). SetFixableOnly(auditCmd.fixableOnly). SetGraphBasicParams(auditCmd.AuditBasicParams). - SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan) + SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan). + SetParallelScans(auditCmd.ParallelScans) auditParams.SetIsRecursiveScan(isRecursiveScan).SetExclusions(auditCmd.Exclusions()) auditResults, err := RunAudit(auditParams) if err != nil { @@ -169,7 +176,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.xrayGraphScanParams.MultiScanId } - auditParallelRunner := utils.CreateAuditParallelRunner() + auditParallelRunner := utils.CreateAuditParallelRunner(auditParams.numOfParallelScans) JFrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) @@ -177,11 +184,11 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) if results.ExtendedScanResults.EntitledForJas { // Download (if needed) the analyzer manager and run scanners. auditParallelRunner.JasWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { + _, jasErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, serverDetails, auditParams, JFrogAppsConfig, threadId) }, auditParallelRunner.AddErrorToChan) - if err != nil { - return results, fmt.Errorf("failed to creat AM and jas scanners task: %s", err.Error()) + if jasErr != nil { + auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to creat AM and jas scanners task: %s", err.Error())) } } @@ -200,6 +207,9 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.ScansErr = errors.Join(results.ScansErr, e) } }() + if auditParams.Progress() != nil { + auditParams.Progress().SetHeadlineMsg("Scanning for issues") + } auditParallelRunner.Runner.Run() return } diff --git a/commands/audit/auditparams.go b/commands/audit/auditparams.go index aaa5a2b5..5712ef54 100644 --- a/commands/audit/auditparams.go +++ b/commands/audit/auditparams.go @@ -15,6 +15,7 @@ type AuditParams struct { xrayVersion string // Include third party dependencies source code in the applicability scan. thirdPartyApplicabilityScan bool + numOfParallelScans int } func NewAuditParams() *AuditParams { @@ -87,3 +88,8 @@ func (params *AuditParams) SetDepsRepo(depsRepo string) *AuditParams { params.AuditBasicParams.SetDepsRepo(depsRepo) return params } + +func (params *AuditParams) SetParallelScans(numOfParallelScans int) *AuditParams { + params.numOfParallelScans = numOfParallelScans + return params +} diff --git a/commands/audit/jas/applicability/applicabilitymanager.go b/commands/audit/jas/applicability/applicabilitymanager.go index 2c5c9ea2..619fdace 100644 --- a/commands/audit/jas/applicability/applicabilitymanager.go +++ b/commands/audit/jas/applicability/applicabilitymanager.go @@ -196,11 +196,3 @@ func removeElementFromSlice(skipDirs []string, element string) []string { } return slices.Delete(skipDirs, deleteIndex, deleteIndex+1) } - -func convertDirectDependecies(directDependencies []*string) []string { - var stringsBack []string - for _, d := range directDependencies { - stringsBack = append(stringsBack, *d) - } - return stringsBack -} diff --git a/commands/audit/jas/applicability/applicabilitymanager_test.go b/commands/audit/jas/applicability/applicabilitymanager_test.go index 45dc1fcc..1ec72a0e 100644 --- a/commands/audit/jas/applicability/applicabilitymanager_test.go +++ b/commands/audit/jas/applicability/applicabilitymanager_test.go @@ -1,6 +1,7 @@ package applicability import ( + "github.com/jfrog/jfrog-cli-core/v2/common/cliutils" "github.com/jfrog/jfrog-cli-security/utils" "github.com/stretchr/testify/require" "os" @@ -157,7 +158,7 @@ func TestNewApplicabilityScanManager_VulnerabilitiesDontExist(t *testing.T) { } func TestApplicabilityScanManager_ShouldRun_TechnologiesNotEligibleForScan(t *testing.T) { - auditParallelRunnerForTest := utils.NewAuditParallelRunner() + auditParallelRunnerForTest := utils.NewAuditParallelRunner(cliutils.Threads) scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() diff --git a/commands/audit/jas/common.go b/commands/audit/jas/common.go index d5c33765..d75b88ea 100644 --- a/commands/audit/jas/common.go +++ b/commands/audit/jas/common.go @@ -107,26 +107,6 @@ func (a *JasScanner) Run(scannerCmd ScannerCmd, module jfrogappsconfig.Module) ( return } -func deleteJasProcessFiles(configFile string, resultFile string) error { - exist, err := fileutils.IsFileExists(configFile, false) - if err != nil { - return err - } - if exist { - if err = os.Remove(configFile); err != nil { - return errorutils.CheckError(err) - } - } - exist, err = fileutils.IsFileExists(resultFile, false) - if err != nil { - return err - } - if exist { - err = os.Remove(resultFile) - } - return errorutils.CheckError(err) -} - func ReadJasScanRunsFromFile(fileName, wd, informationUrlSuffix string) (sarifRuns []*sarif.Run, err error) { if sarifRuns, err = utils.ReadScanRunsFromFile(fileName); err != nil { return diff --git a/commands/audit/jas/iac/iacscanner.go b/commands/audit/jas/iac/iacscanner.go index 73617008..0f524b9d 100644 --- a/commands/audit/jas/iac/iacscanner.go +++ b/commands/audit/jas/iac/iacscanner.go @@ -33,7 +33,7 @@ type IacScanManager struct { // []utils.SourceCodeScanResult: a list of the iac violations that were found. // bool: true if the user is entitled to iac scan, false otherwise. // error: An error object (if any). -func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, ExtendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module, threadId int) (err error) { +func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module, threadId int) (err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.IaC)); err != nil { return @@ -49,7 +49,7 @@ func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Jas } results := iacScanManager.iacScannerResults auditParallelRunner.Mu.Lock() - ExtendedScanResults.IacScanResults = append(ExtendedScanResults.IacScanResults, results...) + extendedScanResults.IacScanResults = append(extendedScanResults.IacScanResults, results...) auditParallelRunner.Mu.Unlock() return } diff --git a/commands/audit/jas/secrets/secretsscanner_test.go b/commands/audit/jas/secrets/secretsscanner_test.go index 6dc193e8..6eaf091d 100644 --- a/commands/audit/jas/secrets/secretsscanner_test.go +++ b/commands/audit/jas/secrets/secretsscanner_test.go @@ -1,6 +1,7 @@ package secrets import ( + "github.com/jfrog/jfrog-cli-core/v2/common/cliutils" "github.com/jfrog/jfrog-cli-security/utils" "github.com/stretchr/testify/require" "os" @@ -109,7 +110,7 @@ func TestParseResults_ResultsContainSecrets(t *testing.T) { } func TestGetSecretsScanResults_AnalyzerManagerReturnsError(t *testing.T) { - auditParallelRunnerForTest := utils.NewAuditParallelRunner() + auditParallelRunnerForTest := utils.NewAuditParallelRunner(cliutils.Threads) scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() diff --git a/commands/audit/jasrunner.go b/commands/audit/jasrunner.go index 170d5b76..f66cc637 100644 --- a/commands/audit/jasrunner.go +++ b/commands/audit/jasrunner.go @@ -78,9 +78,6 @@ func runSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas defer func() { auditParallelRunner.ScannersWg.Done() }() - if progress != nil { - progress.SetHeadlineMsg("Running secrets scanning") - } err = secrets.RunSecretsScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module, threadId) return } @@ -91,9 +88,6 @@ func runIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Jas defer func() { auditParallelRunner.ScannersWg.Done() }() - if progress != nil { - progress.SetHeadlineMsg("Running IaC scanning") - } err = iac.RunIacScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module, threadId) return } @@ -104,9 +98,6 @@ func runSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Ja defer func() { auditParallelRunner.ScannersWg.Done() }() - if progress != nil { - progress.SetHeadlineMsg("Running Sast scanning") - } err = sast.RunSastScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module, threadId) return } @@ -118,9 +109,6 @@ func runContextualScan(auditParallelRunner *utils.AuditParallelRunner, scanner * defer func() { auditParallelRunner.ScannersWg.Done() }() - if progress != nil { - progress.SetHeadlineMsg("Running applicability scanning") - } err = applicability.RunApplicabilityScan(auditParallelRunner, scanResults.GetScaScansXrayResults(), auditParams.DirectDependencies(), scanResults.GetScaScannedTechnologies(), scanner, auditParams.thirdPartyApplicabilityScan, scanResults.ExtendedScanResults, module, threadId) return } diff --git a/commands/audit/jasrunner_test.go b/commands/audit/jasrunner_test.go index 27636af1..844c951c 100644 --- a/commands/audit/jasrunner_test.go +++ b/commands/audit/jasrunner_test.go @@ -1,6 +1,7 @@ package audit import ( + "github.com/jfrog/jfrog-cli-core/v2/common/cliutils" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" "github.com/jfrog/jfrog-cli-security/commands/audit/jas/applicability" "os" @@ -13,7 +14,7 @@ import ( ) func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { - auditParallelRunnerForTest := utils.NewAuditParallelRunner() + auditParallelRunnerForTest := utils.NewAuditParallelRunner(cliutils.Threads) tmpDir, err := fileutils.CreateTempDir() defer func() { assert.NoError(t, fileutils.RemoveTempDir(tmpDir)) @@ -34,7 +35,7 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { } func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { - auditParallelRunnerForTest := utils.NewAuditParallelRunner() + auditParallelRunnerForTest := utils.NewAuditParallelRunner(cliutils.Threads) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: coreutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} auditParamsForTest := NewAuditParams().SetThirdPartyApplicabilityScan(false) auditParamsForTest.AuditBasicParams.AppendDependenciesForApplicabilityScan([]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}) @@ -43,7 +44,7 @@ func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { } func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { - auditParallelRunnerForTest := utils.NewAuditParallelRunner() + auditParallelRunnerForTest := utils.NewAuditParallelRunner(cliutils.Threads) assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) diff --git a/commands/audit/sca/common.go b/commands/audit/sca/common.go index 613ef37b..e5bc0b91 100644 --- a/commands/audit/sca/common.go +++ b/commands/audit/sca/common.go @@ -14,7 +14,6 @@ import ( "github.com/jfrog/jfrog-client-go/artifactory/services/fspatterns" clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" - ioUtils "github.com/jfrog/jfrog-client-go/utils/io" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray/services" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" @@ -30,16 +29,13 @@ func GetExcludePattern(params utils.AuditParams) string { return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, params.IsRecursiveScan()) } -func RunXrayDependenciesTreeScanGraph(dependencyTree xrayUtils.GraphNode, progress ioUtils.ProgressMgr, technology coreutils.Technology, scanGraphParams *scangraph.ScanGraphParams) (results []services.ScanResponse, err error) { +func RunXrayDependenciesTreeScanGraph(dependencyTree xrayUtils.GraphNode, technology coreutils.Technology, scanGraphParams *scangraph.ScanGraphParams) (results []services.ScanResponse, err error) { scanGraphParams.XrayGraphScanParams().DependenciesGraph = &dependencyTree xscGitInfoContext := scanGraphParams.XrayGraphScanParams().XscGitInfoContext if xscGitInfoContext != nil { xscGitInfoContext.Technologies = []string{technology.String()} } scanMessage := fmt.Sprintf("Scanning %d %s dependencies", len(dependencyTree.Nodes), technology) - if progress != nil { - progress.SetHeadlineMsg(scanMessage) - } log.Info(scanMessage + "...") var scanResults *services.ScanResponse xrayManager, err := utils.CreateXrayServiceManager(scanGraphParams.ServerDetails()) diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 967ec3a5..bd5c5509 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -148,7 +148,7 @@ func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetail SetXrayVersion(params.xrayVersion). SetFixableOnly(params.fixableOnly). SetSeverityLevel(params.minSeverityFilter) - techResults, err = sca.RunXrayDependenciesTreeScanGraph(flatTree, params.Progress(), tech, scanGraphParams) + techResults, err = sca.RunXrayDependenciesTreeScanGraph(flatTree, tech, scanGraphParams) if err != nil { return } diff --git a/commands/common.go b/commands/common.go new file mode 100644 index 00000000..1db74f38 --- /dev/null +++ b/commands/common.go @@ -0,0 +1,14 @@ +package commands + +import "github.com/jfrog/jfrog-client-go/utils/errorutils" + +const ( + TotalConcurrentRequests = 10 +) + +func DetectNumOfThreads(threadsCount int) (int, error) { + if threadsCount > TotalConcurrentRequests { + return 0, errorutils.CheckErrorf("number of threads crossed the maximum, the maximum threads allowed is %v", TotalConcurrentRequests) + } + return threadsCount, nil +} diff --git a/commands/curation/curationaudit.go b/commands/curation/curationaudit.go index af582ceb..bb2c50d4 100644 --- a/commands/curation/curationaudit.go +++ b/commands/curation/curationaudit.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" config "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-cli-security/commands" "net/http" "os" "path/filepath" @@ -49,8 +50,6 @@ const ( errorTemplateUnsupportedTech = "It looks like this project uses '%s' to download its dependencies. " + "This package manager however isn't supported by this command." - TotalConcurrentRequests = 10 - MinArtiMavenSupport = "7.82.0" MinArtiXraySupport = "3.92.0" ) @@ -306,7 +305,7 @@ func (ca *CurationAuditCommand) auditTree(tech coreutils.Technology, results map projectName = projectScope + "/" + projectName } if ca.parallelRequests == 0 { - ca.parallelRequests = TotalConcurrentRequests + ca.parallelRequests = commands.TotalConcurrentRequests } var packagesStatus []*PackageStatus analyzer := treeAnalyzer{ @@ -663,13 +662,6 @@ func buildNpmDownloadUrl(url, repo, name, scope, version string) []string { return []string{packageUrl} } -func DetectNumOfThreads(threadsCount int) (int, error) { - if threadsCount > TotalConcurrentRequests { - return 0, errorutils.CheckErrorf("number of threads crossed the maximum, the maximum threads allowed is %v", TotalConcurrentRequests) - } - return threadsCount, nil -} - func GetCurationOutputFormat(formatFlagVal string) (format outFormat.OutputFormat, err error) { // Default print format is table. format = outFormat.Table diff --git a/utils/parallel_runner.go b/utils/parallel_runner.go index 270bbc3c..22151fe2 100644 --- a/utils/parallel_runner.go +++ b/utils/parallel_runner.go @@ -14,15 +14,15 @@ type AuditParallelRunner struct { JasWg sync.WaitGroup // verify that downloading analyzer manager and running all scanners are done } -func NewAuditParallelRunner() AuditParallelRunner { +func NewAuditParallelRunner(numOfParallelScans int) AuditParallelRunner { return AuditParallelRunner{ - Runner: parallel.NewRunner(3, 20000, false), + Runner: parallel.NewRunner(numOfParallelScans, 20000, false), ErrorsQueue: make(chan error, 100), } } -func CreateAuditParallelRunner() *AuditParallelRunner { - auditParallelRunner := NewAuditParallelRunner() +func CreateAuditParallelRunner(numOfParallelScans int) *AuditParallelRunner { + auditParallelRunner := NewAuditParallelRunner(numOfParallelScans) return &auditParallelRunner } From c3ce94955a83c5687d668f8b23afc2f9bf1a8184 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 4 Apr 2024 15:45:55 +0300 Subject: [PATCH 08/59] fix static analysis checks --- commands/audit/jasrunner.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/commands/audit/jasrunner.go b/commands/audit/jasrunner.go index f66cc637..9a5ffa4d 100644 --- a/commands/audit/jasrunner.go +++ b/commands/audit/jasrunner.go @@ -12,7 +12,6 @@ import ( "github.com/jfrog/jfrog-cli-security/commands/audit/jas/sast" "github.com/jfrog/jfrog-cli-security/commands/audit/jas/secrets" "github.com/jfrog/jfrog-cli-security/utils" - "github.com/jfrog/jfrog-client-go/utils/io" "github.com/jfrog/jfrog-client-go/utils/log" ) @@ -37,21 +36,21 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, for _, module := range scanner.JFrogAppsConfig.Modules { if !jas.ShouldSkipScanner(module, utils.Secrets) { auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runSecretsScan(auditParallelRunner, scanner, auditParams.Progress(), scanResults, module), auditParallelRunner.AddErrorToChan) + _, err = auditParallelRunner.Runner.AddTaskWithError(runSecretsScan(auditParallelRunner, scanner, scanResults, module), auditParallelRunner.AddErrorToChan) if err != nil { return fmt.Errorf("failed to create secrets scan task: %s", err.Error()) } } if !jas.ShouldSkipScanner(module, utils.IaC) { auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runIacScan(auditParallelRunner, scanner, auditParams.Progress(), scanResults, module), auditParallelRunner.AddErrorToChan) + _, err = auditParallelRunner.Runner.AddTaskWithError(runIacScan(auditParallelRunner, scanner, scanResults, module), auditParallelRunner.AddErrorToChan) if err != nil { return fmt.Errorf("failed to create iac scan task: %s", err.Error()) } } if !jas.ShouldSkipScanner(module, utils.Sast) { auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runSastScan(auditParallelRunner, scanner, auditParams.Progress(), scanResults, module), auditParallelRunner.AddErrorToChan) + _, err = auditParallelRunner.Runner.AddTaskWithError(runSastScan(auditParallelRunner, scanner, scanResults, module), auditParallelRunner.AddErrorToChan) if err != nil { return fmt.Errorf("failed to create sast scan task: %s", err.Error()) } @@ -64,7 +63,7 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, for _, module := range scanner.JFrogAppsConfig.Modules { if !jas.ShouldSkipScanner(module, utils.Applicability) { auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runContextualScan(auditParallelRunner, scanner, auditParams.Progress(), scanResults, module, auditParams), auditParallelRunner.AddErrorToChan) + _, err = auditParallelRunner.Runner.AddTaskWithError(runContextualScan(auditParallelRunner, scanner, scanResults, module, auditParams), auditParallelRunner.AddErrorToChan) if err != nil { return fmt.Errorf("failed to create contextual scan task: %s", err.Error()) } @@ -73,7 +72,8 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, return err } -func runSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { +func runSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, + module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { auditParallelRunner.ScannersWg.Done() @@ -83,7 +83,8 @@ func runSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas } } -func runIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { +func runIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, + module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { auditParallelRunner.ScannersWg.Done() @@ -93,7 +94,8 @@ func runIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Jas } } -func runSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { +func runSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, + module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { auditParallelRunner.ScannersWg.Done() @@ -103,8 +105,8 @@ func runSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Ja } } -func runContextualScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, progress io.ProgressMgr, - scanResults *utils.Results, module jfrogappsconfig.Module, auditParams *AuditParams) parallel.TaskFunc { +func runContextualScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, + module jfrogappsconfig.Module, auditParams *AuditParams) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { auditParallelRunner.ScannersWg.Done() From 8dbe5d37ba1a648ee785200fc51f18b4d22bd8bd Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 14 Apr 2024 13:48:52 +0300 Subject: [PATCH 09/59] fix unit test --- commands/audit/audit.go | 32 +++++++++++++++------------- commands/audit/auditparams.go | 11 +++++----- commands/audit/scarunner.go | 40 ++++++++++++++++++++++++----------- 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index f4d13753..af1d6365 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -14,7 +14,6 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray" "github.com/jfrog/jfrog-client-go/xray/services" - "os" ) type AuditCommand struct { @@ -29,6 +28,14 @@ type AuditCommand struct { AuditParams } +type CommonCommandParams struct { + watches []string + projectKey string + targetRepoPath string + IncludeVulnerabilities bool + IncludeLicenses bool +} + func NewGenericAuditCommand() *AuditCommand { return &AuditCommand{AuditParams: *NewAuditParams()} } @@ -73,20 +80,15 @@ func (auditCmd *AuditCommand) SetParallelScans(threads int) *AuditCommand { return auditCmd } -func (auditCmd *AuditCommand) CreateXrayGraphScanParams() *services.XrayGraphScanParams { - params := &services.XrayGraphScanParams{ - RepoPath: auditCmd.targetRepoPath, - Watches: auditCmd.watches, - ScanType: services.Dependency, - } - if auditCmd.projectKey == "" { - params.ProjectKey = os.Getenv(coreutils.Project) - } else { - params.ProjectKey = auditCmd.projectKey +func (auditCmd *AuditCommand) CreateCommonCommandParams() *CommonCommandParams { + commonParams := &CommonCommandParams{ + watches: auditCmd.watches, + projectKey: auditCmd.projectKey, + targetRepoPath: auditCmd.targetRepoPath, + IncludeVulnerabilities: auditCmd.IncludeVulnerabilities, + IncludeLicenses: auditCmd.IncludeLicenses, } - params.IncludeVulnerabilities = auditCmd.IncludeVulnerabilities - params.IncludeLicenses = auditCmd.IncludeLicenses - return params + return commonParams } func (auditCmd *AuditCommand) Run() (err error) { @@ -98,11 +100,11 @@ func (auditCmd *AuditCommand) Run() (err error) { } auditParams := NewAuditParams(). - SetXrayGraphScanParams(auditCmd.CreateXrayGraphScanParams()). SetWorkingDirs(workingDirs). SetMinSeverityFilter(auditCmd.minSeverityFilter). SetFixableOnly(auditCmd.fixableOnly). SetGraphBasicParams(auditCmd.AuditBasicParams). + SetCommonCommandParams(auditCmd.CreateCommonCommandParams()). SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan). SetParallelScans(auditCmd.ParallelScans) auditParams.SetIsRecursiveScan(isRecursiveScan).SetExclusions(auditCmd.Exclusions()) diff --git a/commands/audit/auditparams.go b/commands/audit/auditparams.go index 5712ef54..17fba77c 100644 --- a/commands/audit/auditparams.go +++ b/commands/audit/auditparams.go @@ -7,6 +7,7 @@ import ( type AuditParams struct { xrayGraphScanParams *services.XrayGraphScanParams + commonCommandParams *CommonCommandParams workingDirs []string installFunc func(tech string) error fixableOnly bool @@ -41,11 +42,6 @@ func (params *AuditParams) XrayVersion() string { return params.xrayVersion } -func (params *AuditParams) SetXrayGraphScanParams(xrayGraphScanParams *services.XrayGraphScanParams) *AuditParams { - params.xrayGraphScanParams = xrayGraphScanParams - return params -} - func (params *AuditParams) SetGraphBasicParams(gbp *xrayutils.AuditBasicParams) *AuditParams { params.AuditBasicParams = gbp return params @@ -93,3 +89,8 @@ func (params *AuditParams) SetParallelScans(numOfParallelScans int) *AuditParams params.numOfParallelScans = numOfParallelScans return params } + +func (params *AuditParams) SetCommonCommandParams(commonCommandParams *CommonCommandParams) *AuditParams { + params.commonCommandParams = commonCommandParams + return params +} diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index bd5c5509..e183e3cf 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -33,18 +33,18 @@ import ( "time" ) -func runScaScan(auditParallelRunner *utils.AuditParallelRunner, params *AuditParams, results *xrayutils.Results) (err error) { +func runScaScan(auditParallelRunner *utils.AuditParallelRunner, auditParams *AuditParams, results *xrayutils.Results) (err error) { // Prepare currentWorkingDir, err := os.Getwd() if errorutils.CheckError(err) != nil { return } - serverDetails, err := params.ServerDetails() + serverDetails, err := auditParams.ServerDetails() if err != nil { return } - scans := getScaScansToPreform(params) + scans := getScaScansToPreform(auditParams) if len(scans) == 0 { log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") return @@ -61,23 +61,21 @@ func runScaScan(auditParallelRunner *utils.AuditParallelRunner, params *AuditPar }() for _, scan := range scans { // Get the dependency tree for the technology in the working directory. - flattenTree, fullDependencyTrees, bdtErr := buildDependencyTree(scan, params) + flattenTree, fullDependencyTrees, bdtErr := buildDependencyTree(scan, auditParams) if bdtErr != nil { err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, bdtErr.Error())) continue } // Create sca scan task auditParallelRunner.ScaScansWg.Add(1) - _, taskErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, params, scan, *flattenTree, fullDependencyTrees), func(err error) { + _, taskErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, auditParams, scan, *flattenTree, fullDependencyTrees), func(err error) { auditParallelRunner.AddErrorToChan(fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, err.Error())) }) if taskErr != nil { return fmt.Errorf("failed to creat sca scan task: %s", taskErr.Error()) } // Add the scan to the results - auditParallelRunner.Mu.Lock() results.ScaResults = append(results.ScaResults, scan) - auditParallelRunner.Mu.Unlock() } return } @@ -120,7 +118,7 @@ func getRequestedDescriptors(params *AuditParams) map[coreutils.Technology][]str } // Preform the SCA scan for the given scan information. -func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetails *config.ServerDetails, params *AuditParams, +func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetails *config.ServerDetails, auditParams *AuditParams, scan *xrayutils.ScaScanResult, flattenTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) parallel.TaskFunc { return func(threadId int) (err error) { log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") @@ -128,12 +126,13 @@ func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetail auditParallelRunner.ScaScansWg.Done() }() // Scan the dependency tree. - scanResults, xrayErr := runScaWithTech(scan.Technology, params, serverDetails, flattenTree, fullDependencyTrees) + xrayGraphScanParams := createXrayGraphScanParams(auditParams) + scanResults, xrayErr := runScaWithTech(scan.Technology, auditParams, xrayGraphScanParams, serverDetails, flattenTree, fullDependencyTrees) if xrayErr != nil { return fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s", scan.Technology, xrayErr.Error()) } scan.IsMultipleRootProject = clientutils.Pointer(len(fullDependencyTrees) > 1) - addThirdPartyDependenciesToParams(params, scan.Technology, &flattenTree, fullDependencyTrees) + addThirdPartyDependenciesToParams(auditParams, scan.Technology, &flattenTree, fullDependencyTrees) auditParallelRunner.Mu.Lock() scan.XrayResults = append(scan.XrayResults, scanResults...) auditParallelRunner.Mu.Unlock() @@ -141,10 +140,11 @@ func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetail } } -func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []services.ScanResponse, err error) { +func runScaWithTech(tech coreutils.Technology, params *AuditParams, xrayGraphScanParams *services.XrayGraphScanParams, serverDetails *config.ServerDetails, + flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []services.ScanResponse, err error) { scanGraphParams := scangraph.NewScanGraphParams(). SetServerDetails(serverDetails). - SetXrayGraphScanParams(params.xrayGraphScanParams). + SetXrayGraphScanParams(xrayGraphScanParams). SetXrayVersion(params.xrayVersion). SetFixableOnly(params.fixableOnly). SetSeverityLevel(params.minSeverityFilter) @@ -397,3 +397,19 @@ func buildDependencyTree(scan *utils.ScaScanResult, params *AuditParams) (*xrayC } return flattenTree, fullDependencyTrees, nil } + +func createXrayGraphScanParams(auditParams *AuditParams) *services.XrayGraphScanParams { + params := &services.XrayGraphScanParams{ + RepoPath: auditParams.commonCommandParams.targetRepoPath, + Watches: auditParams.commonCommandParams.watches, + ScanType: services.Dependency, + } + if auditParams.commonCommandParams.projectKey == "" { + params.ProjectKey = os.Getenv(coreutils.Project) + } else { + params.ProjectKey = auditParams.commonCommandParams.projectKey + } + params.IncludeVulnerabilities = auditParams.commonCommandParams.IncludeVulnerabilities + params.IncludeLicenses = auditParams.commonCommandParams.IncludeLicenses + return params +} From 93af9541d3421397741622719aee00551815111e Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Mon, 15 Apr 2024 10:23:21 +0300 Subject: [PATCH 10/59] merge fixes --- cli/scancommands.go | 7 +++---- commands/audit/jasrunner.go | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cli/scancommands.go b/cli/scancommands.go index 82003e69..f6b9735b 100644 --- a/cli/scancommands.go +++ b/cli/scancommands.go @@ -331,10 +331,6 @@ func AuditCmd(c *components.Context) error { } } auditCmd.SetTechnologies(technologies) - err = progressbar.ExecWithProgress(auditCmd) - // Reporting error if Xsc service is enabled - reportErrorIfExists(err, auditCmd) - threadsFlag, err := c.GetIntFlagValue(flags.Threads) if err != nil { return err @@ -344,6 +340,9 @@ func AuditCmd(c *components.Context) error { return err } auditCmd.SetParallelScans(threads) + err = progressbar.ExecWithProgress(auditCmd) + // Reporting error if Xsc service is enabled + reportErrorIfExists(err, auditCmd) return err } diff --git a/commands/audit/jasrunner.go b/commands/audit/jasrunner.go index dae8279d..00723772 100644 --- a/commands/audit/jasrunner.go +++ b/commands/audit/jasrunner.go @@ -25,16 +25,15 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, if err != nil { return } + // Set environments variables for analytics in analyzers manager. + callback := jas.SetAnalyticsMetricsDataForAnalyzerManager(msi, scanResults.GetScaScannedTechnologies()) defer func() { auditParallelRunner.ScannersWg.Wait() + callback() cleanup := scanner.ScannerDirCleanupFunc err = errors.Join(err, cleanup()) }() - // Set environments variables for analytics in analyzers manager. - callback := jas.SetAnalyticsMetricsDataForAnalyzerManager(msi, scanResults.GetScaScannedTechnologies()) - defer callback() - // Don't execute other scanners when scanning third party dependencies. if !auditParams.thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { From 70d47944cef4baf97c4e8d51403210f565ae9517 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 5 May 2024 11:58:20 +0300 Subject: [PATCH 11/59] Cr fixes --- cli/docs/flags.go | 7 +-- cli/scancommands.go | 15 +---- commands/audit/audit.go | 58 +++++++------------ commands/audit/auditparams.go | 34 ++++++----- .../jas/applicability/applicabilitymanager.go | 15 +++-- commands/audit/jas/common.go | 13 ++--- commands/audit/jas/common_test.go | 8 +-- commands/audit/jas/iac/iacscanner.go | 13 ++--- commands/audit/jas/sast/sastscanner.go | 10 ++-- commands/audit/jas/secrets/secretsscanner.go | 10 ++-- commands/audit/jasrunner.go | 43 ++++++-------- commands/audit/scarunner.go | 37 ++++-------- commands/curation/curationaudit.go | 6 +- scangraph/params.go | 11 ++++ utils/analyzermanager.go | 4 +- utils/parallel_runner.go | 2 +- 16 files changed, 123 insertions(+), 163 deletions(-) diff --git a/cli/docs/flags.go b/cli/docs/flags.go index 7132117d..cce3a3e2 100644 --- a/cli/docs/flags.go +++ b/cli/docs/flags.go @@ -2,7 +2,6 @@ package docs import ( "fmt" - "github.com/jfrog/jfrog-cli-security/commands" "strings" "github.com/jfrog/jfrog-cli-core/v2/common/cliutils" @@ -105,8 +104,7 @@ const ( WorkingDirs = "working-dirs" // Unique curation flags - CurationOutput = "curation-format" - CurationThreads = "curation-threads" + CurationOutput = "curation-format" ) // Mapping between security commands (key) and their flags (key). @@ -129,7 +127,7 @@ var commandFlags = map[string][]string{ Pnpm, Yarn, Go, Nuget, Pip, Pipenv, Poetry, MinSeverity, FixableOnly, ThirdPartyContextualAnalysis, Threads, }, CurationAudit: { - CurationOutput, WorkingDirs, CurationThreads, RequirementsFile, + CurationOutput, WorkingDirs, Threads, RequirementsFile, }, // TODO: Deprecated commands (remove at next CLI major version) AuditMvn: { @@ -220,7 +218,6 @@ var flagsMap = map[string]components.Flag{ components.SetHiddenBoolFlag(), ), RequirementsFile: components.NewStringFlag(RequirementsFile, "[Pip] Defines pip requirements file name. For example: 'requirements.txt'."), - CurationThreads: components.NewStringFlag(Threads, "Number of working threads.", components.WithIntDefaultValue(commands.TotalConcurrentRequests)), CurationOutput: components.NewStringFlag(OutputFormat, "Defines the output format of the command. Acceptable values are: table, json.", components.WithStrDefaultValue("table")), } diff --git a/cli/scancommands.go b/cli/scancommands.go index f6b9735b..3655827d 100644 --- a/cli/scancommands.go +++ b/cli/scancommands.go @@ -3,7 +3,6 @@ package cli import ( "fmt" "github.com/jfrog/jfrog-cli-core/v2/utils/usage" - "github.com/jfrog/jfrog-cli-security/commands" "os" "strings" @@ -331,15 +330,11 @@ func AuditCmd(c *components.Context) error { } } auditCmd.SetTechnologies(technologies) - threadsFlag, err := c.GetIntFlagValue(flags.Threads) - if err != nil { - return err - } - threads, err := commands.DetectNumOfThreads(threadsFlag) + threads, err := pluginsCommon.GetThreadsCount(c) if err != nil { return err } - auditCmd.SetParallelScans(threads) + auditCmd.SetThreads(threads) err = progressbar.ExecWithProgress(auditCmd) // Reporting error if Xsc service is enabled reportErrorIfExists(err, auditCmd) @@ -436,11 +431,7 @@ func AuditSpecificCmd(c *components.Context, technology coreutils.Technology) er } func CurationCmd(c *components.Context) error { - threadsFlag, err := c.GetIntFlagValue(flags.Threads) - if err != nil { - return err - } - threads, err := commands.DetectNumOfThreads(threadsFlag) + threads, err := pluginsCommon.GetThreadsCount(c) if err != nil { return err } diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 3bbb3f6c..af205e7f 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -27,21 +27,10 @@ type AuditCommand struct { Fail bool PrintExtendedTable bool analyticsMetricsService *xrayutils.AnalyticsMetricsService - ParallelScans int + Threads int AuditParams } -type CommonGraphScanParams struct { - repoPath string - projectKey string - watches []string - scanType services.ScanType - includeVulnerabilities bool - includeLicenses bool - xscVersion string - multiScanId string -} - func NewGenericAuditCommand() *AuditCommand { return &AuditCommand{AuditParams: *NewAuditParams()} } @@ -86,33 +75,33 @@ func (auditCmd *AuditCommand) SetAnalyticsMetricsService(analyticsMetricsService return auditCmd } -func (auditCmd *AuditCommand) SetParallelScans(threads int) *AuditCommand { - auditCmd.ParallelScans = threads +func (auditCmd *AuditCommand) SetThreads(threads int) *AuditCommand { + auditCmd.Threads = threads return auditCmd } -func (auditCmd *AuditCommand) CreateCommonGraphScanParams() *CommonGraphScanParams { - commonParams := &CommonGraphScanParams{ - repoPath: auditCmd.targetRepoPath, - watches: auditCmd.watches, - scanType: services.Dependency, +func (auditCmd *AuditCommand) CreateCommonGraphScanParams() *scangraph.CommonGraphScanParams { + commonParams := &scangraph.CommonGraphScanParams{ + RepoPath: auditCmd.targetRepoPath, + Watches: auditCmd.watches, + ScanType: services.Dependency, } if auditCmd.projectKey == "" { - commonParams.projectKey = os.Getenv(coreutils.Project) + commonParams.ProjectKey = os.Getenv(coreutils.Project) } else { - commonParams.projectKey = auditCmd.projectKey + commonParams.ProjectKey = auditCmd.projectKey } - commonParams.includeVulnerabilities = auditCmd.IncludeVulnerabilities - commonParams.includeLicenses = auditCmd.IncludeLicenses - commonParams.multiScanId = auditCmd.analyticsMetricsService.GetMsi() - if commonParams.multiScanId != "" { + commonParams.IncludeVulnerabilities = auditCmd.IncludeVulnerabilities + commonParams.IncludeLicenses = auditCmd.IncludeLicenses + commonParams.MultiScanId = auditCmd.analyticsMetricsService.GetMsi() + if commonParams.MultiScanId != "" { xscManager := auditCmd.analyticsMetricsService.XscManager() if xscManager != nil { version, err := xscManager.GetVersion() if err != nil { log.Debug(fmt.Sprintf("Can't get XSC version for xray graph scan params. Cause: %s", err.Error())) } - commonParams.xscVersion = version + commonParams.XscVersion = version } } return commonParams @@ -135,7 +124,7 @@ func (auditCmd *AuditCommand) Run() (err error) { SetGraphBasicParams(auditCmd.AuditBasicParams). SetCommonGraphScanParams(auditCmd.CreateCommonGraphScanParams()). SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan). - SetParallelScans(auditCmd.ParallelScans) + SetThreads(auditCmd.Threads) auditParams.SetIsRecursiveScan(isRecursiveScan).SetExclusions(auditCmd.Exclusions()) auditResults, err := RunAudit(auditParams) @@ -202,9 +191,9 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) return } - results.MultiScanId = auditParams.commonGraphScanParams.multiScanId + results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId - auditParallelRunner := utils.CreateAuditParallelRunner(auditParams.numOfParallelScans) + auditParallelRunner := utils.CreateAuditParallelRunner(auditParams.threads) JFrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) @@ -216,13 +205,12 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, serverDetails, auditParams, JFrogAppsConfig, threadId) }, auditParallelRunner.AddErrorToChan) if jasErr != nil { - auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to creat AM and jas scanners task: %s", jasErr.Error())) + auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create AM downloading task, skipping JAS scans...: %s", jasErr.Error())) } } // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. - scaScanErr := runScaScan(auditParallelRunner, auditParams, results) - if scaScanErr != nil { + if scaScanErr := runScaScan(auditParallelRunner, auditParams, results); scaScanErr != nil { auditParallelRunner.AddErrorToChan(scaScanErr) } go func() { @@ -256,10 +244,8 @@ func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.AuditParal defer func() { auditParallelRunner.JasWg.Done() }() - err = utils.DownloadAnalyzerManagerIfNeeded(threadId) - if err != nil { + if err = utils.DownloadAnalyzerManagerIfNeeded(threadId); err != nil { return } - err = RunJasScannersAndSetResults(auditParallelRunner, scanResults, serverDetails, auditParams, jfrogAppsConfig, scanResults.MultiScanId) - return + return RunJasScannersAndSetResults(auditParallelRunner, scanResults, serverDetails, auditParams, jfrogAppsConfig, scanResults.MultiScanId) } diff --git a/commands/audit/auditparams.go b/commands/audit/auditparams.go index 2b85474b..8d2be912 100644 --- a/commands/audit/auditparams.go +++ b/commands/audit/auditparams.go @@ -1,14 +1,14 @@ package audit import ( + "github.com/jfrog/jfrog-cli-security/scangraph" xrayutils "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-client-go/xray/services" ) type AuditParams struct { - xrayGraphScanParams *services.XrayGraphScanParams - // common params to all scan routines - commonGraphScanParams *CommonGraphScanParams + // Common params to all scan routines + commonGraphScanParams *scangraph.CommonGraphScanParams workingDirs []string installFunc func(tech string) error fixableOnly bool @@ -17,13 +17,12 @@ type AuditParams struct { xrayVersion string // Include third party dependencies source code in the applicability scan. thirdPartyApplicabilityScan bool - numOfParallelScans int + threads int } func NewAuditParams() *AuditParams { return &AuditParams{ - xrayGraphScanParams: &services.XrayGraphScanParams{}, - AuditBasicParams: &xrayutils.AuditBasicParams{}, + AuditBasicParams: &xrayutils.AuditBasicParams{}, } } @@ -31,10 +30,6 @@ func (params *AuditParams) InstallFunc() func(tech string) error { return params.installFunc } -func (params *AuditParams) XrayGraphScanParams() *services.XrayGraphScanParams { - return params.xrayGraphScanParams -} - func (params *AuditParams) WorkingDirs() []string { return params.workingDirs } @@ -86,12 +81,25 @@ func (params *AuditParams) SetDepsRepo(depsRepo string) *AuditParams { return params } -func (params *AuditParams) SetParallelScans(numOfParallelScans int) *AuditParams { - params.numOfParallelScans = numOfParallelScans +func (params *AuditParams) SetThreads(threads int) *AuditParams { + params.threads = threads return params } -func (params *AuditParams) SetCommonGraphScanParams(commonParams *CommonGraphScanParams) *AuditParams { +func (params *AuditParams) SetCommonGraphScanParams(commonParams *scangraph.CommonGraphScanParams) *AuditParams { params.commonGraphScanParams = commonParams return params } + +func (params *AuditParams) createXrayGraphScanParams() *services.XrayGraphScanParams { + return &services.XrayGraphScanParams{ + RepoPath: params.commonGraphScanParams.RepoPath, + Watches: params.commonGraphScanParams.Watches, + ScanType: params.commonGraphScanParams.ScanType, + ProjectKey: params.commonGraphScanParams.ProjectKey, + IncludeVulnerabilities: params.commonGraphScanParams.IncludeVulnerabilities, + IncludeLicenses: params.commonGraphScanParams.IncludeLicenses, + XscVersion: params.commonGraphScanParams.XscVersion, + MultiScanId: params.commonGraphScanParams.MultiScanId, + } +} diff --git a/commands/audit/jas/applicability/applicabilitymanager.go b/commands/audit/jas/applicability/applicabilitymanager.go index 0dd1d67d..e4999a4a 100644 --- a/commands/audit/jas/applicability/applicabilitymanager.go +++ b/commands/audit/jas/applicability/applicabilitymanager.go @@ -1,18 +1,17 @@ package applicability import ( + "github.com/jfrog/gofrog/datastructures" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" - "path/filepath" - "strconv" - - "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-security/utils" + clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray/services" "github.com/owenrumney/go-sarif/v2/sarif" "golang.org/x/exp/maps" "golang.org/x/exp/slices" + "path/filepath" ) const ( @@ -49,17 +48,17 @@ func RunApplicabilityScan(auditParallelRunner *utils.AuditParallelRunner, xrayRe } applicabilityScanManager := newApplicabilityScanManager(xrayResults, directDependencies, scanner, thirdPartyContextualAnalysis, scannerTempDir) if !applicabilityScanManager.cvesExists() { - log.Debug("[thread_id: " + strconv.Itoa(threadId) + "] We couldn't find any vulnerable dependencies. Skipping....") + log.Debug(clientutils.GetLogMsgPrefix(threadId, false), "We couldn't find any vulnerable dependencies. Skipping....") return } - log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running applicability scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false), "Running applicability scanning...") if err = applicabilityScanManager.scanner.Run(applicabilityScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Applicability, err) return } - auditParallelRunner.Mu.Lock() + auditParallelRunner.ResultsMu.Lock() extendedScanResults.ApplicabilityScanResults = applicabilityScanManager.applicabilityScanResults - auditParallelRunner.Mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jas/common.go b/commands/audit/jas/common.go index 5bca84fd..4a03e2a2 100644 --- a/commands/audit/jas/common.go +++ b/commands/audit/jas/common.go @@ -4,9 +4,9 @@ import ( "errors" "fmt" clientutils "github.com/jfrog/jfrog-client-go/utils" - "golang.org/x/exp/rand" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -211,7 +211,8 @@ var FakeBasicXrayResults = []services.ScanResponse{ func InitJasTest(t *testing.T, workingDirs ...string) (*JasScanner, func()) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) - jfrogAppsConfigForTest, _ := CreateJFrogAppsConfig(workingDirs) + jfrogAppsConfigForTest, err := CreateJFrogAppsConfig(workingDirs) + assert.NoError(t, err) scanner, err := NewJasScanner(&FakeServerDetails, jfrogAppsConfigForTest) assert.NoError(t, err) return scanner, func() { @@ -302,13 +303,7 @@ func CreateScannerTempDirectory(scanner *JasScanner, scanType string) (string, e if scanner.TempDir == "" { return "", errors.New("scanner temp dir cannot be created in an empty base dir") } - rand.Seed(uint64(time.Now().UnixNano())) - randomString := "" - for i := 0; i < 4; i++ { - randomDigit := rand.Intn(10) - randomString += fmt.Sprintf("%d", randomDigit) - } - scannerTempDir := scanner.TempDir + "/" + scanType + "_" + randomString + scannerTempDir := scanner.TempDir + "/" + scanType + "_" + strconv.FormatInt(time.Now().Unix(), 10) err := os.MkdirAll(scannerTempDir, 0777) if err != nil { return "", err diff --git a/commands/audit/jas/common_test.go b/commands/audit/jas/common_test.go index 3128e2f1..0b67f03a 100644 --- a/commands/audit/jas/common_test.go +++ b/commands/audit/jas/common_test.go @@ -131,8 +131,7 @@ func TestSetAnalyticsMetricsDataForAnalyzerManager(t *testing.T) { } func TestCreateScannerTempDirectory(t *testing.T) { - scanner, cleanUp := InitJasTest(t) - defer cleanUp() + scanner := &JasScanner{TempDir: "path"} tempDir, err := CreateScannerTempDirectory(scanner, string(utils.Applicability)) assert.NoError(t, err) assert.NotEmpty(t, tempDir) @@ -143,10 +142,7 @@ func TestCreateScannerTempDirectory(t *testing.T) { } func TestCreateScannerTempDirectory_baseDirIsEmpty(t *testing.T) { - scanner, cleanUp := InitJasTest(t) - defer cleanUp() - - scanner.TempDir = "" + scanner := &JasScanner{TempDir: ""} _, err := CreateScannerTempDirectory(scanner, string(utils.Applicability)) assert.Error(t, err) } diff --git a/commands/audit/jas/iac/iacscanner.go b/commands/audit/jas/iac/iacscanner.go index 0f524b9d..8e4798e2 100644 --- a/commands/audit/jas/iac/iacscanner.go +++ b/commands/audit/jas/iac/iacscanner.go @@ -1,11 +1,10 @@ package iac import ( - "path/filepath" - "strconv" - jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" + clientutils "github.com/jfrog/jfrog-client-go/utils" + "path/filepath" "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -39,18 +38,18 @@ func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Jas return } iacScanManager := newIacScanManager(scanner, scannerTempDir) - log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running IaC scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running IaC scan...") if err = iacScanManager.scanner.Run(iacScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.IaC, err) return } if len(iacScanManager.iacScannerResults) > 0 { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities") + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities") } results := iacScanManager.iacScannerResults - auditParallelRunner.Mu.Lock() + auditParallelRunner.ResultsMu.Lock() extendedScanResults.IacScanResults = append(extendedScanResults.IacScanResults, results...) - auditParallelRunner.Mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jas/sast/sastscanner.go b/commands/audit/jas/sast/sastscanner.go index 8d8f930d..82c78a8f 100644 --- a/commands/audit/jas/sast/sastscanner.go +++ b/commands/audit/jas/sast/sastscanner.go @@ -5,11 +5,11 @@ import ( jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" "github.com/jfrog/jfrog-cli-security/utils" + clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/owenrumney/go-sarif/v2/sarif" "golang.org/x/exp/maps" "path/filepath" - "strconv" ) const ( @@ -31,18 +31,18 @@ func RunSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Ja return } sastScanManager := newSastScanManager(scanner, scannerTempDir) - log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running SAST scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running SAST scanning...") if err = sastScanManager.scanner.Run(sastScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Sast, err) return } if len(sastScanManager.sastScannerResults) > 0 { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities") + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities") } results := sastScanManager.sastScannerResults - auditParallelRunner.Mu.Lock() + auditParallelRunner.ResultsMu.Lock() extendedScanResults.SastScanResults = append(extendedScanResults.SastScanResults, results...) - auditParallelRunner.Mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jas/secrets/secretsscanner.go b/commands/audit/jas/secrets/secretsscanner.go index 1d2e71bd..b52c90e8 100644 --- a/commands/audit/jas/secrets/secretsscanner.go +++ b/commands/audit/jas/secrets/secretsscanner.go @@ -1,8 +1,8 @@ package secrets import ( + clientutils "github.com/jfrog/jfrog-client-go/utils" "path/filepath" - "strconv" "strings" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" @@ -39,18 +39,18 @@ func RunSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas return } secretScanManager := newSecretsScanManager(scanner, scannerTempDir) - log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running secrets scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running secrets scanning...") if err = secretScanManager.scanner.Run(secretScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Secrets, err) return } results := secretScanManager.secretsScannerResults if len(results) > 0 { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(results...), "secrets") + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(results...), "secrets") } - auditParallelRunner.Mu.Lock() + auditParallelRunner.ResultsMu.Lock() extendedScanResults.SecretsScanResults = append(extendedScanResults.SecretsScanResults, results...) - auditParallelRunner.Mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jasrunner.go b/commands/audit/jasrunner.go index ad4a5f8a..1589644a 100644 --- a/commands/audit/jasrunner.go +++ b/commands/audit/jasrunner.go @@ -37,26 +37,14 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, // Don't execute other scanners when scanning third party dependencies. if !auditParams.thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { - if !jas.ShouldSkipScanner(module, utils.Secrets) { - auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runSecretsScan(auditParallelRunner, scanner, scanResults, module), auditParallelRunner.AddErrorToChan) - if err != nil { - return fmt.Errorf("failed to create secrets scan task: %s", err.Error()) - } + if err = addModuleJasScanTask(module, utils.Secrets, auditParallelRunner, runSecretsScan(auditParallelRunner, scanner, scanResults, module)); err != nil { + return } - if !jas.ShouldSkipScanner(module, utils.IaC) { - auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runIacScan(auditParallelRunner, scanner, scanResults, module), auditParallelRunner.AddErrorToChan) - if err != nil { - return fmt.Errorf("failed to create iac scan task: %s", err.Error()) - } + if err = addModuleJasScanTask(module, utils.IaC, auditParallelRunner, runIacScan(auditParallelRunner, scanner, scanResults, module)); err != nil { + return } - if !jas.ShouldSkipScanner(module, utils.Sast) { - auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runSastScan(auditParallelRunner, scanner, scanResults, module), auditParallelRunner.AddErrorToChan) - if err != nil { - return fmt.Errorf("failed to create sast scan task: %s", err.Error()) - } + if err = addModuleJasScanTask(module, utils.Sast, auditParallelRunner, runSastScan(auditParallelRunner, scanner, scanResults, module)); err != nil { + return } } } @@ -64,17 +52,24 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, // Wait for sca scan to complete auditParallelRunner.ScaScansWg.Wait() for _, module := range scanner.JFrogAppsConfig.Modules { - if !jas.ShouldSkipScanner(module, utils.Applicability) { - auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runContextualScan(auditParallelRunner, scanner, scanResults, module, auditParams), auditParallelRunner.AddErrorToChan) - if err != nil { - return fmt.Errorf("failed to create contextual scan task: %s", err.Error()) - } + if err = addModuleJasScanTask(module, utils.Applicability, auditParallelRunner, runContextualScan(auditParallelRunner, scanner, scanResults, module, auditParams)); err != nil { + return } } return err } +func addModuleJasScanTask(module jfrogappsconfig.Module, scanType utils.JasScanType, auditParallelRunner *utils.AuditParallelRunner, task parallel.TaskFunc) (err error) { + if jas.ShouldSkipScanner(module, scanType) { + return + } + auditParallelRunner.ScannersWg.Add(1) + if _, err = auditParallelRunner.Runner.AddTaskWithError(task, auditParallelRunner.AddErrorToChan); err != nil { + err = fmt.Errorf("failed to create %s scan task: %s", scanType, err.Error()) + } + return +} + func runSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index d185cb3f..86e7f7f6 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/jfrog/build-info-go/utils/pythonutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" - "strconv" "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/gofrog/parallel" @@ -68,11 +67,11 @@ func runScaScan(auditParallelRunner *utils.AuditParallelRunner, auditParams *Aud } // Create sca scan task auditParallelRunner.ScaScansWg.Add(1) - _, taskErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, auditParams, scan, *treeResult.FlatTree, treeResult.FullDepTrees), func(err error) { + _, taskErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, auditParams, scan, treeResult), func(err error) { auditParallelRunner.AddErrorToChan(fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, err.Error())) }) if taskErr != nil { - return fmt.Errorf("failed to creat sca scan task: %s", taskErr.Error()) + return fmt.Errorf("failed to create sca scan task for '%s': %s", scan.WorkingDirectory, taskErr.Error()) } // Add the scan to the results results.ScaResults = append(results.ScaResults, scan) @@ -119,32 +118,31 @@ func getRequestedDescriptors(params *AuditParams) map[coreutils.Technology][]str // Preform the SCA scan for the given scan information. func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetails *config.ServerDetails, auditParams *AuditParams, - scan *xrayutils.ScaScanResult, flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) parallel.TaskFunc { + scan *xrayutils.ScaScanResult, treeResult *DependencyTreeResult) parallel.TaskFunc { return func(threadId int) (err error) { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") defer func() { auditParallelRunner.ScaScansWg.Done() }() // Scan the dependency tree. - xrayGraphScanParams := createXrayGraphScanParams(auditParams) - scanResults, xrayErr := runScaWithTech(scan.Technology, auditParams, xrayGraphScanParams, serverDetails, flatTree, fullDependencyTrees) + scanResults, xrayErr := runScaWithTech(scan.Technology, auditParams, serverDetails, *treeResult.FlatTree, treeResult.FullDepTrees) if xrayErr != nil { return fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s", scan.Technology, xrayErr.Error()) } - scan.IsMultipleRootProject = clientutils.Pointer(len(fullDependencyTrees) > 1) - addThirdPartyDependenciesToParams(auditParams, scan.Technology, &flatTree, fullDependencyTrees) - auditParallelRunner.Mu.Lock() + scan.IsMultipleRootProject = clientutils.Pointer(len(treeResult.FullDepTrees) > 1) + addThirdPartyDependenciesToParams(auditParams, scan.Technology, treeResult.FlatTree, treeResult.FullDepTrees) + auditParallelRunner.ResultsMu.Lock() scan.XrayResults = append(scan.XrayResults, scanResults...) - auditParallelRunner.Mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() return } } -func runScaWithTech(tech coreutils.Technology, params *AuditParams, xrayGraphScanParams *services.XrayGraphScanParams, serverDetails *config.ServerDetails, +func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []services.ScanResponse, err error) { scanGraphParams := scangraph.NewScanGraphParams(). SetServerDetails(serverDetails). - SetXrayGraphScanParams(xrayGraphScanParams). + SetXrayGraphScanParams(params.createXrayGraphScanParams()). SetXrayVersion(params.xrayVersion). SetFixableOnly(params.fixableOnly). SetSeverityLevel(params.minSeverityFilter) @@ -406,16 +404,3 @@ func buildDependencyTree(scan *utils.ScaScanResult, params *AuditParams) (*Depen } return &treeResult, nil } - -func createXrayGraphScanParams(auditParams *AuditParams) *services.XrayGraphScanParams { - return &services.XrayGraphScanParams{ - RepoPath: auditParams.commonGraphScanParams.repoPath, - Watches: auditParams.commonGraphScanParams.watches, - ScanType: auditParams.commonGraphScanParams.scanType, - ProjectKey: auditParams.commonGraphScanParams.projectKey, - IncludeVulnerabilities: auditParams.commonGraphScanParams.includeVulnerabilities, - IncludeLicenses: auditParams.commonGraphScanParams.includeLicenses, - XscVersion: auditParams.commonGraphScanParams.xscVersion, - MultiScanId: auditParams.commonGraphScanParams.multiScanId, - } -} diff --git a/commands/curation/curationaudit.go b/commands/curation/curationaudit.go index 81ae6a34..044e9b1f 100644 --- a/commands/curation/curationaudit.go +++ b/commands/curation/curationaudit.go @@ -4,8 +4,8 @@ import ( "encoding/json" "errors" "fmt" + "github.com/jfrog/jfrog-cli-core/v2/common/cliutils" config "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-cli-security/commands" "net/http" "os" "path/filepath" @@ -55,8 +55,6 @@ const ( MinArtiPassThroughSupport = "7.82.0" MinXrayPassTHroughSupport = "3.92.0" - MinArtiMavenSupport = "7.82.0" - MinArtiXraySupport = "3.92.0" ) var CurationOutputFormats = []string{string(outFormat.Table), string(outFormat.Json)} @@ -322,7 +320,7 @@ func (ca *CurationAuditCommand) auditTree(tech coreutils.Technology, results map projectName = projectScope + "/" + projectName } if ca.parallelRequests == 0 { - ca.parallelRequests = commands.TotalConcurrentRequests + ca.parallelRequests = cliutils.Threads } var packagesStatus []*PackageStatus analyzer := treeAnalyzer{ diff --git a/scangraph/params.go b/scangraph/params.go index 76e0106e..e6547cd9 100644 --- a/scangraph/params.go +++ b/scangraph/params.go @@ -13,6 +13,17 @@ type ScanGraphParams struct { severityLevel int } +type CommonGraphScanParams struct { + RepoPath string + ProjectKey string + Watches []string + ScanType services.ScanType + IncludeVulnerabilities bool + IncludeLicenses bool + XscVersion string + MultiScanId string +} + func NewScanGraphParams() *ScanGraphParams { return &ScanGraphParams{} } diff --git a/utils/analyzermanager.go b/utils/analyzermanager.go index d8f3a500..7ec32a41 100644 --- a/utils/analyzermanager.go +++ b/utils/analyzermanager.go @@ -3,11 +3,11 @@ package utils import ( "errors" "fmt" + clientutils "github.com/jfrog/jfrog-client-go/utils" "os" "os/exec" "path" "path/filepath" - "strconv" "strings" "github.com/jfrog/jfrog-cli-core/v2/utils/config" @@ -257,7 +257,7 @@ func DownloadAnalyzerManagerIfNeeded(threadId int) error { } } // Download & unzip the analyzer manager files - log.Debug("[thread_id: " + strconv.Itoa(threadId) + "] The 'Analyzer Manager' app is not cached locally. Downloading it now...") + log.Debug(clientutils.GetLogMsgPrefix(threadId, false) + "The 'Analyzer Manager' app is not cached locally. Downloading it now...") if err = dependencies.DownloadDependency(artDetails, remotePath, filepath.Join(analyzerManagerDir, AnalyzerManagerZipName), true); err != nil { return err } diff --git a/utils/parallel_runner.go b/utils/parallel_runner.go index 22151fe2..93aa0454 100644 --- a/utils/parallel_runner.go +++ b/utils/parallel_runner.go @@ -8,7 +8,7 @@ import ( type AuditParallelRunner struct { Runner parallel.Runner ErrorsQueue chan error - Mu sync.Mutex + ResultsMu sync.Mutex ScaScansWg sync.WaitGroup // verify that the sca scan routines are done before running contextual scan ScannersWg sync.WaitGroup // verify that all scanners routines are done before cleaning temp dir JasWg sync.WaitGroup // verify that downloading analyzer manager and running all scanners are done From 5405e6d1a782991d5b61f513d72d079d4eaaa193 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 12 May 2024 12:58:10 +0300 Subject: [PATCH 12/59] Cr fixes --- audit_test.go | 18 +++++++++++ commands/audit/audit.go | 11 ++++--- .../jas/applicability/applicabilitymanager.go | 13 ++++---- commands/audit/jas/common.go | 1 + commands/audit/jas/iac/iacscanner.go | 9 ++---- commands/audit/jas/sast/sastscanner.go | 11 +++---- commands/audit/jas/secrets/secretsscanner.go | 10 ++---- commands/audit/jasrunner.go | 32 ++++++++++++++++--- commands/common.go | 14 -------- .../python_project_with_no_deps/main.py | 5 +++ utils/analyzermanager.go | 2 +- utils/sarifutils.go | 11 +++++++ 12 files changed, 88 insertions(+), 49 deletions(-) delete mode 100644 commands/common.go create mode 100644 tests/testdata/projects/empty_project/python_project_with_no_deps/main.py diff --git a/audit_test.go b/audit_test.go index df2e3421..89cd5c4d 100644 --- a/audit_test.go +++ b/audit_test.go @@ -550,3 +550,21 @@ func validateAnalyticsBasicEvent(t *testing.T, output string) { assert.NotEmpty(t, event.TotalScanDuration) assert.True(t, event.TotalFindings > 0) } + +func TestAuditOnEmptyProject(t *testing.T) { + securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion) + tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t) + defer createTempDirCallback() + projectDir := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), filepath.Join("projects", "empty_project", "python_project_with_no_deps")) + // Copy the multi project from the testdata to a temp dir + assert.NoError(t, biutils.CopyDir(projectDir, tempDirPath, true, nil)) + // Configure a new server named "default" + securityTestUtils.CreateJfrogHomeConfig(t, true) + defer securityTestUtils.CleanTestsHomeEnv() + baseWd, err := os.Getwd() + assert.NoError(t, err) + chdirCallback := clientTests.ChangeDirWithCallback(t, baseWd, tempDirPath) + defer chdirCallback() + output := securityTests.PlatformCli.WithoutCredentials().RunCliCmdWithOutput(t, "audit", "--format="+string(format.Json)) + assert.Equal(t, "null\n", output) +} diff --git a/commands/audit/audit.go b/commands/audit/audit.go index af205e7f..210a7635 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -240,12 +240,15 @@ func isEntitledForJas(xrayManager *xray.XrayServicesManager, xrayVersion string) } func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.AuditParallelRunner, scanResults *utils.Results, - serverDetails *config.ServerDetails, auditParams *AuditParams, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, threadId int) (err error) { + serverDetails *config.ServerDetails, auditParams *AuditParams, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, threadId int) error { defer func() { auditParallelRunner.JasWg.Done() }() - if err = utils.DownloadAnalyzerManagerIfNeeded(threadId); err != nil { - return + if err := utils.DownloadAnalyzerManagerIfNeeded(threadId); err != nil { + return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + } + if err := RunJasScannersAndSetResults(auditParallelRunner, scanResults, serverDetails, auditParams, jfrogAppsConfig, scanResults.MultiScanId); err != nil { + return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) } - return RunJasScannersAndSetResults(auditParallelRunner, scanResults, serverDetails, auditParams, jfrogAppsConfig, scanResults.MultiScanId) + return nil } diff --git a/commands/audit/jas/applicability/applicabilitymanager.go b/commands/audit/jas/applicability/applicabilitymanager.go index e4999a4a..2c96c93a 100644 --- a/commands/audit/jas/applicability/applicabilitymanager.go +++ b/commands/audit/jas/applicability/applicabilitymanager.go @@ -40,8 +40,8 @@ type ApplicabilityScanManager struct { // map[string]string: A map containing the applicability result of each XRAY CVE. // bool: true if the user is entitled to the applicability scan, false otherwise. // error: An error object (if any). -func RunApplicabilityScan(auditParallelRunner *utils.AuditParallelRunner, xrayResults []services.ScanResponse, directDependencies []string, - scanner *jas.JasScanner, thirdPartyContextualAnalysis bool, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module, threadId int) (err error) { +func RunApplicabilityScan(xrayResults []services.ScanResponse, directDependencies []string, scanner *jas.JasScanner, + thirdPartyContextualAnalysis bool, module jfrogappsconfig.Module, threadId int) (results []*sarif.Run, err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)); err != nil { return @@ -51,14 +51,15 @@ func RunApplicabilityScan(auditParallelRunner *utils.AuditParallelRunner, xrayRe log.Debug(clientutils.GetLogMsgPrefix(threadId, false), "We couldn't find any vulnerable dependencies. Skipping....") return } - log.Info(clientutils.GetLogMsgPrefix(threadId, false), "Running applicability scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running applicability scan...") if err = applicabilityScanManager.scanner.Run(applicabilityScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Applicability, err) return } - auditParallelRunner.ResultsMu.Lock() - extendedScanResults.ApplicabilityScanResults = applicabilityScanManager.applicabilityScanResults - auditParallelRunner.ResultsMu.Unlock() + results = applicabilityScanManager.applicabilityScanResults + if len(results) > 0 { + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetApplicableResultCountFromRule(results...), "applicable cves") + } return } diff --git a/commands/audit/jas/common.go b/commands/audit/jas/common.go index 4a03e2a2..73dc7c4f 100644 --- a/commands/audit/jas/common.go +++ b/commands/audit/jas/common.go @@ -75,6 +75,7 @@ func CreateJFrogAppsConfig(workingDirs []string) (*jfrogappsconfig.JFrogAppsConf } else if jfrogAppsConfig != nil { // jfrog-apps-config.yml exist in the workspace for _, module := range jfrogAppsConfig.Modules { + // converting to absolute path before starting the scan flow module.SourceRoot, err = filepath.Abs(module.SourceRoot) if err != nil { return nil, errorutils.CheckError(err) diff --git a/commands/audit/jas/iac/iacscanner.go b/commands/audit/jas/iac/iacscanner.go index 8e4798e2..45fea79a 100644 --- a/commands/audit/jas/iac/iacscanner.go +++ b/commands/audit/jas/iac/iacscanner.go @@ -32,7 +32,7 @@ type IacScanManager struct { // []utils.SourceCodeScanResult: a list of the iac violations that were found. // bool: true if the user is entitled to iac scan, false otherwise. // error: An error object (if any). -func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module, threadId int) (err error) { +func RunIacScan(scanner *jas.JasScanner, module jfrogappsconfig.Module, threadId int) (results []*sarif.Run, err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.IaC)); err != nil { return @@ -43,13 +43,10 @@ func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Jas err = utils.ParseAnalyzerManagerError(utils.IaC, err) return } - if len(iacScanManager.iacScannerResults) > 0 { + results = iacScanManager.iacScannerResults + if len(results) > 0 { log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities") } - results := iacScanManager.iacScannerResults - auditParallelRunner.ResultsMu.Lock() - extendedScanResults.IacScanResults = append(extendedScanResults.IacScanResults, results...) - auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jas/sast/sastscanner.go b/commands/audit/jas/sast/sastscanner.go index 82c78a8f..94b06744 100644 --- a/commands/audit/jas/sast/sastscanner.go +++ b/commands/audit/jas/sast/sastscanner.go @@ -25,24 +25,21 @@ type SastScanManager struct { resultsFileName string } -func RunSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module, threadId int) (err error) { +func RunSastScan(scanner *jas.JasScanner, module jfrogappsconfig.Module, threadId int) (results []*sarif.Run, err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Sast)); err != nil { return } sastScanManager := newSastScanManager(scanner, scannerTempDir) - log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running SAST scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running SAST scan...") if err = sastScanManager.scanner.Run(sastScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Sast, err) return } - if len(sastScanManager.sastScannerResults) > 0 { + results = sastScanManager.sastScannerResults + if len(results) > 0 { log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities") } - results := sastScanManager.sastScannerResults - auditParallelRunner.ResultsMu.Lock() - extendedScanResults.SastScanResults = append(extendedScanResults.SastScanResults, results...) - auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jas/secrets/secretsscanner.go b/commands/audit/jas/secrets/secretsscanner.go index b52c90e8..d3a23f68 100644 --- a/commands/audit/jas/secrets/secretsscanner.go +++ b/commands/audit/jas/secrets/secretsscanner.go @@ -32,25 +32,21 @@ type SecretScanManager struct { // Return values: // []utils.IacOrSecretResult: a list of the secrets that were found. // error: An error object (if any). -func RunSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, - module jfrogappsconfig.Module, threadId int) (err error) { +func RunSecretsScan(scanner *jas.JasScanner, module jfrogappsconfig.Module, threadId int) (results []*sarif.Run, err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Secrets)); err != nil { return } secretScanManager := newSecretsScanManager(scanner, scannerTempDir) - log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running secrets scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running secrets scan...") if err = secretScanManager.scanner.Run(secretScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Secrets, err) return } - results := secretScanManager.secretsScannerResults + results = secretScanManager.secretsScannerResults if len(results) > 0 { log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(results...), "secrets") } - auditParallelRunner.ResultsMu.Lock() - extendedScanResults.SecretsScanResults = append(extendedScanResults.SecretsScanResults, results...) - auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jasrunner.go b/commands/audit/jasrunner.go index 1589644a..16882f19 100644 --- a/commands/audit/jasrunner.go +++ b/commands/audit/jasrunner.go @@ -76,7 +76,13 @@ func runSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas defer func() { auditParallelRunner.ScannersWg.Done() }() - err = secrets.RunSecretsScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module, threadId) + results, err := secrets.RunSecretsScan(scanner, module, threadId) + if err != nil { + return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + } + auditParallelRunner.ResultsMu.Lock() + scanResults.ExtendedScanResults.SecretsScanResults = append(scanResults.ExtendedScanResults.SecretsScanResults, results...) + auditParallelRunner.ResultsMu.Unlock() return } } @@ -87,7 +93,13 @@ func runIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Jas defer func() { auditParallelRunner.ScannersWg.Done() }() - err = iac.RunIacScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module, threadId) + results, err := iac.RunIacScan(scanner, module, threadId) + if err != nil { + return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + } + auditParallelRunner.ResultsMu.Lock() + scanResults.ExtendedScanResults.IacScanResults = append(scanResults.ExtendedScanResults.IacScanResults, results...) + auditParallelRunner.ResultsMu.Unlock() return } } @@ -98,7 +110,13 @@ func runSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Ja defer func() { auditParallelRunner.ScannersWg.Done() }() - err = sast.RunSastScan(auditParallelRunner, scanner, scanResults.ExtendedScanResults, module, threadId) + results, err := sast.RunSastScan(scanner, module, threadId) + if err != nil { + return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + } + auditParallelRunner.ResultsMu.Lock() + scanResults.ExtendedScanResults.SastScanResults = append(scanResults.ExtendedScanResults.SastScanResults, results...) + auditParallelRunner.ResultsMu.Unlock() return } } @@ -109,7 +127,13 @@ func runContextualScan(auditParallelRunner *utils.AuditParallelRunner, scanner * defer func() { auditParallelRunner.ScannersWg.Done() }() - err = applicability.RunApplicabilityScan(auditParallelRunner, scanResults.GetScaScansXrayResults(), auditParams.DirectDependencies(), scanner, auditParams.thirdPartyApplicabilityScan, scanResults.ExtendedScanResults, module, threadId) + results, err := applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), auditParams.DirectDependencies(), scanner, auditParams.thirdPartyApplicabilityScan, module, threadId) + if err != nil { + return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + } + auditParallelRunner.ResultsMu.Lock() + scanResults.ExtendedScanResults.ApplicabilityScanResults = append(scanResults.ExtendedScanResults.ApplicabilityScanResults, results...) + auditParallelRunner.ResultsMu.Unlock() return } } diff --git a/commands/common.go b/commands/common.go deleted file mode 100644 index 1db74f38..00000000 --- a/commands/common.go +++ /dev/null @@ -1,14 +0,0 @@ -package commands - -import "github.com/jfrog/jfrog-client-go/utils/errorutils" - -const ( - TotalConcurrentRequests = 10 -) - -func DetectNumOfThreads(threadsCount int) (int, error) { - if threadsCount > TotalConcurrentRequests { - return 0, errorutils.CheckErrorf("number of threads crossed the maximum, the maximum threads allowed is %v", TotalConcurrentRequests) - } - return threadsCount, nil -} diff --git a/tests/testdata/projects/empty_project/python_project_with_no_deps/main.py b/tests/testdata/projects/empty_project/python_project_with_no_deps/main.py new file mode 100644 index 00000000..3a7c0141 --- /dev/null +++ b/tests/testdata/projects/empty_project/python_project_with_no_deps/main.py @@ -0,0 +1,5 @@ +def main(): + print('Empty project with no dependencies') + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/utils/analyzermanager.go b/utils/analyzermanager.go index 7ec32a41..c3d9ccff 100644 --- a/utils/analyzermanager.go +++ b/utils/analyzermanager.go @@ -257,7 +257,7 @@ func DownloadAnalyzerManagerIfNeeded(threadId int) error { } } // Download & unzip the analyzer manager files - log.Debug(clientutils.GetLogMsgPrefix(threadId, false) + "The 'Analyzer Manager' app is not cached locally. Downloading it now...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "The 'Analyzer Manager' app is not cached locally. Downloading it now...") if err = dependencies.DownloadDependency(artDetails, remotePath, filepath.Join(analyzerManagerDir, AnalyzerManagerZipName), true); err != nil { return err } diff --git a/utils/sarifutils.go b/utils/sarifutils.go index 0da57714..2baf101e 100644 --- a/utils/sarifutils.go +++ b/utils/sarifutils.go @@ -265,3 +265,14 @@ func GetInvocationWorkingDirectory(invocation *sarif.Invocation) string { } return "" } + +func GetApplicableResultCountFromRule(runs ...*sarif.Run) (count int) { + for _, run := range runs { + for _, rule := range run.Tool.Driver.Rules { + if rule.Properties["applicability"] != nil && rule.Properties["applicability"] == "applicable" { + count += 1 + } + } + } + return +} From 689958134562743b892de8a062ab5e89b1cd1852 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Wed, 5 Jun 2024 11:08:32 +0300 Subject: [PATCH 13/59] cr fixes --- commands/audit/jas/secrets/secretsscanner.go | 2 +- commands/audit/scarunner.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/audit/jas/secrets/secretsscanner.go b/commands/audit/jas/secrets/secretsscanner.go index d3a23f68..bc7baace 100644 --- a/commands/audit/jas/secrets/secretsscanner.go +++ b/commands/audit/jas/secrets/secretsscanner.go @@ -45,7 +45,7 @@ func RunSecretsScan(scanner *jas.JasScanner, module jfrogappsconfig.Module, thre } results = secretScanManager.secretsScannerResults if len(results) > 0 { - log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(results...), "secrets") + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(results...), "secrets vulnerabilities") } return } diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 86e7f7f6..ea387f55 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -127,7 +127,7 @@ func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetail // Scan the dependency tree. scanResults, xrayErr := runScaWithTech(scan.Technology, auditParams, serverDetails, *treeResult.FlatTree, treeResult.FullDepTrees) if xrayErr != nil { - return fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s", scan.Technology, xrayErr.Error()) + return fmt.Errorf("error from thread_id %d: '%s' Xray dependency tree scan request failed:\n%s", threadId, scan.Technology, xrayErr.Error()) } scan.IsMultipleRootProject = clientutils.Pointer(len(treeResult.FullDepTrees) > 1) addThirdPartyDependenciesToParams(auditParams, scan.Technology, treeResult.FlatTree, treeResult.FullDepTrees) From dad5eb1a9194b839432590595ee1dd658a830549 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 6 Jun 2024 13:55:46 +0300 Subject: [PATCH 14/59] cr fix --- commands/scan/scan.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/commands/scan/scan.go b/commands/scan/scan.go index cd2446ae..25d6253b 100644 --- a/commands/scan/scan.go +++ b/commands/scan/scan.go @@ -333,14 +333,14 @@ func (scanCmd *ScanCommand) CommandName() string { return "xr_scan" } -func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer parallel.Runner, JasFileProducerConsumer *utils.SecurityParallelRunner, entitledForJas bool, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors, jasErrors [][]formats.SimpleJsonError, fileCollectingErrorsQueue *clientutils.ErrorsQueue, xrayVersion string) { +func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer parallel.Runner, jasFileProducerConsumer *utils.SecurityParallelRunner, entitledForJas bool, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors, jasErrors [][]formats.SimpleJsonError, fileCollectingErrorsQueue *clientutils.ErrorsQueue, xrayVersion string) { go func() { defer fileProducer.Done() // Iterate over file-spec groups and produce indexing tasks. // When encountering an error, log and move to next group. specFiles := scanCmd.spec.Files for i := range specFiles { - artifactHandlerFunc := scanCmd.createIndexerHandlerFunc(&specFiles[i], entitledForJas, indexedFileProducer, JasFileProducerConsumer, resultsArr, fileErrors, indexedFileErrors, jasErrors, xrayVersion) + artifactHandlerFunc := scanCmd.createIndexerHandlerFunc(&specFiles[i], entitledForJas, indexedFileProducer, jasFileProducerConsumer, resultsArr, fileErrors, indexedFileErrors, jasErrors, xrayVersion) taskHandler := getAddTaskToProducerFunc(fileProducer, artifactHandlerFunc) err := collectFilesForIndexing(specFiles[i], taskHandler) @@ -352,7 +352,7 @@ func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer p }() } -func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledForJas bool, indexedFileProducer parallel.Runner, JasFileProducerConsumer *utils.SecurityParallelRunner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors, jasErrors [][]formats.SimpleJsonError, xrayVersion string) FileContext { +func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledForJas bool, indexedFileProducer parallel.Runner, jasFileProducerConsumer *utils.SecurityParallelRunner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors, jasErrors [][]formats.SimpleJsonError, xrayVersion string) FileContext { return func(filePath string) parallel.TaskFunc { return func(threadId int) (err error) { logMsgPrefix := clientutils.GetLogMsgPrefix(threadId, false) @@ -410,7 +410,7 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledFo jasErrors[threadId] = append(jasErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()}) } workingDirs := []string{filePath} - err = runner.RunJasScannersAndSetResults(JasFileProducerConsumer, &extendedScanResults, []techutils.Technology{techutils.Technology(scanResults.ScannedPackageType)}, []services.ScanResponse{*scanResults}, depsListFromVulnerabilities(*scanResults), scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) + err = runner.RunJasScannersAndSetResults(jasFileProducerConsumer, &extendedScanResults, []techutils.Technology{techutils.Technology(scanResults.ScannedPackageType)}, []services.ScanResponse{*scanResults}, depsListFromVulnerabilities(*scanResults), scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) if err != nil { log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error())) From 85ea02ce270229156eef5fc04f8ccbab81fc8236 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 6 Jun 2024 14:14:11 +0300 Subject: [PATCH 15/59] Merge remote-tracking branch 'upstream/dev' into run-jas-scanners-in-parallel # Conflicts: # utils/securityJobSummary.go --- utils/securityJobSummary.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/securityJobSummary.go b/utils/securityJobSummary.go index e848b7a6..17a3a96b 100644 --- a/utils/securityJobSummary.go +++ b/utils/securityJobSummary.go @@ -366,7 +366,7 @@ func getScanSummary(extendedScanResults *ExtendedScanResults, scaResults ...*Sca return } -func getScanViolationsSummary(scaResults ...ScaScanResult) (violations formats.TwoLevelSummaryCount) { +func getScanViolationsSummary(scaResults ...*ScaScanResult) (violations formats.TwoLevelSummaryCount) { vioUniqueFindings := map[string]IssueDetails{} if len(scaResults) == 0 { return @@ -392,7 +392,7 @@ func getScanViolationsSummary(scaResults ...ScaScanResult) (violations formats.T return issueDetailsToSummaryCount(vioUniqueFindings) } -func getScanSecurityVulnerabilitiesSummary(extendedScanResults *ExtendedScanResults, scaResults ...ScaScanResult) (summary *formats.ScanVulnerabilitiesSummary) { +func getScanSecurityVulnerabilitiesSummary(extendedScanResults *ExtendedScanResults, scaResults ...*ScaScanResult) (summary *formats.ScanVulnerabilitiesSummary) { summary = &formats.ScanVulnerabilitiesSummary{} if extendedScanResults == nil { summary.ScaScanResults = getScaSummaryResults(scaResults) From 25618600e778a09ae0d8587d054b6d0b66f5424e Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 6 Jun 2024 15:04:05 +0300 Subject: [PATCH 16/59] add debug flag --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 18c94b88..63e035b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,8 @@ jobs: env: GOPROXY: direct GRADLE_OPTS: -Dorg.gradle.daemon=false + CI: true + JFROG_CLI_LOG_LEVEL: DEBUG steps: # Install dependencies - name: Install Go From 63f718854843a7507956d1a56947d721665f12e9 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 6 Jun 2024 16:02:54 +0300 Subject: [PATCH 17/59] fix tests --- commands/audit/audit.go | 3 +-- jas/runner/jasrunner.go | 2 +- jas/runner/jasrunner_test.go | 12 ++++++------ jas/secrets/secretsscanner_test.go | 1 - utils/parallel_runner.go | 4 ++-- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index a0c1a40a..f9dd7df8 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -199,7 +199,6 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) - //JFrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) } @@ -211,7 +210,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) }, auditParallelRunner.AddErrorToChan) if jasErr != nil { auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create AM downloading task, skipping JAS scans...: %s", jasErr.Error())) - auditParallelRunner.JasWg.Done() + //auditParallelRunner.JasWg.Done() } } diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index 98752e84..06215ddd 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -76,7 +76,7 @@ func addModuleJasScanTask(module jfrogappsconfig.Module, scanType utils.JasScanT securityParallelRunner.ScannersWg.Add(1) if _, err = securityParallelRunner.Runner.AddTaskWithError(task, errHandlerFunc); err != nil { err = fmt.Errorf("failed to create %s scan task: %s", scanType, err.Error()) - securityParallelRunner.ScannersWg.Done() + //securityParallelRunner.ScannersWg.Done() } return } diff --git a/jas/runner/jasrunner_test.go b/jas/runner/jasrunner_test.go index 880a99f8..57635b2b 100644 --- a/jas/runner/jasrunner_test.go +++ b/jas/runner/jasrunner_test.go @@ -16,7 +16,7 @@ import ( ) func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { - auditParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) + securityParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) tmpDir, err := fileutils.CreateTempDir() defer func() { assert.NoError(t, fileutils.RemoveTempDir(tmpDir)) @@ -27,24 +27,24 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err = RunJasScannersAndSetResults(auditParallelRunnerForTest, scanResults.ExtendedScanResults, scanResults.GetScaScannedTechnologies(), scanResults.GetScaScansXrayResults(), []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunnerForTest.AddErrorToChan) + err = RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults.ExtendedScanResults, scanResults.GetScaScannedTechnologies(), scanResults.GetScaScansXrayResults(), []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) // Expect error: assert.Error(t, err) } func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { - auditParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) + securityParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := RunJasScannersAndSetResults(auditParallelRunnerForTest, scanResults.ExtendedScanResults, scanResults.GetScaScannedTechnologies(), scanResults.GetScaScansXrayResults(), []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunnerForTest.AddErrorToChan) + err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults.ExtendedScanResults, scanResults.GetScaScannedTechnologies(), scanResults.GetScaScansXrayResults(), []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) assert.NoError(t, err) } func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { - auditParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) + securityParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := RunJasScannersAndSetResults(auditParallelRunnerForTest, scanResults.ExtendedScanResults, scanResults.GetScaScannedTechnologies(), scanResults.GetScaScansXrayResults(), []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunnerForTest.AddErrorToChan) + err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults.ExtendedScanResults, scanResults.GetScaScannedTechnologies(), scanResults.GetScaScansXrayResults(), []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) // Expect error: assert.ErrorContains(t, err, "failed to run Applicability scan") diff --git a/jas/secrets/secretsscanner_test.go b/jas/secrets/secretsscanner_test.go index 7abcb213..cb3c2550 100644 --- a/jas/secrets/secretsscanner_test.go +++ b/jas/secrets/secretsscanner_test.go @@ -109,7 +109,6 @@ func TestParseResults_ResultsContainSecrets(t *testing.T) { } func TestGetSecretsScanResults_AnalyzerManagerReturnsError(t *testing.T) { - // auditParallelRunnerForTest := utils.NewSecurityParallelRunner(cliutils.Threads) scanner, cleanUp := jas.InitJasTest(t) defer cleanUp() scanResults, err := RunSecretsScan(scanner, SecretsScannerType, scanner.JFrogAppsConfig.Modules[0], 0) diff --git a/utils/parallel_runner.go b/utils/parallel_runner.go index dac218cb..b6f7ebc7 100644 --- a/utils/parallel_runner.go +++ b/utils/parallel_runner.go @@ -22,8 +22,8 @@ func NewSecurityParallelRunner(numOfParallelScans int) SecurityParallelRunner { } func CreateSecurityParallelRunner(numOfParallelScans int) *SecurityParallelRunner { - auditParallelRunner := NewSecurityParallelRunner(numOfParallelScans) - return &auditParallelRunner + securityParallelRunner := NewSecurityParallelRunner(numOfParallelScans) + return &securityParallelRunner } func (spr *SecurityParallelRunner) AddErrorToChan(err error) { From 08ea3cda70b08f8dd0a9bc791ff7272e37baf30c Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 6 Jun 2024 17:21:08 +0300 Subject: [PATCH 18/59] change xray results to pointers array --- commands/audit/audit.go | 1 - commands/audit/sca/common.go | 6 +++--- commands/audit/sca/common_test.go | 2 +- commands/audit/scarunner.go | 3 ++- commands/scan/buildscan.go | 2 +- commands/scan/scan.go | 4 ++-- jas/applicability/applicabilitymanager.go | 8 ++++---- jas/applicability/applicabilitymanager_test.go | 10 +++++----- jas/common.go | 2 +- jas/runner/jasrunner.go | 6 ++---- utils/analyticsmetrics_test.go | 2 +- utils/results.go | 12 ++++++------ utils/results_test.go | 4 ++-- utils/resultstable_test.go | 2 +- utils/resultwriter.go | 2 +- utils/resultwriter_test.go | 2 +- 16 files changed, 33 insertions(+), 35 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index f9dd7df8..47f7843b 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -210,7 +210,6 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) }, auditParallelRunner.AddErrorToChan) if jasErr != nil { auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create AM downloading task, skipping JAS scans...: %s", jasErr.Error())) - //auditParallelRunner.JasWg.Done() } } diff --git a/commands/audit/sca/common.go b/commands/audit/sca/common.go index 2046e7c5..75c07b7d 100644 --- a/commands/audit/sca/common.go +++ b/commands/audit/sca/common.go @@ -32,7 +32,7 @@ func GetExcludePattern(params utils.AuditParams) string { return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, params.IsRecursiveScan()) } -func RunXrayDependenciesTreeScanGraph(dependencyTree xrayUtils.GraphNode, technology techutils.Technology, scanGraphParams *scangraph.ScanGraphParams) (results []services.ScanResponse, err error) { +func RunXrayDependenciesTreeScanGraph(dependencyTree xrayUtils.GraphNode, technology techutils.Technology, scanGraphParams *scangraph.ScanGraphParams) (results []*services.ScanResponse, err error) { scanGraphParams.XrayGraphScanParams().DependenciesGraph = &dependencyTree xscGitInfoContext := scanGraphParams.XrayGraphScanParams().XscGitInfoContext if xscGitInfoContext != nil { @@ -56,7 +56,7 @@ func RunXrayDependenciesTreeScanGraph(dependencyTree xrayUtils.GraphNode, techno for i := range scanResults.Violations { scanResults.Violations[i].Technology = technology.String() } - results = append(results, *scanResults) + results = append(results, scanResults) return } @@ -82,7 +82,7 @@ func LogExecutableVersion(executable string) { // BuildImpactPathsForScanResponse builds the full impact paths for each vulnerability found in the scanResult argument, using the dependencyTrees argument. // Returns the updated services.ScanResponse slice. -func BuildImpactPathsForScanResponse(scanResult []services.ScanResponse, dependencyTree []*xrayUtils.GraphNode) []services.ScanResponse { +func BuildImpactPathsForScanResponse(scanResult []*services.ScanResponse, dependencyTree []*xrayUtils.GraphNode) []*services.ScanResponse { for _, result := range scanResult { if len(result.Vulnerabilities) > 0 { buildVulnerabilitiesImpactPaths(result.Vulnerabilities, dependencyTree) diff --git a/commands/audit/sca/common_test.go b/commands/audit/sca/common_test.go index be8b42fc..a093ed5b 100644 --- a/commands/audit/sca/common_test.go +++ b/commands/audit/sca/common_test.go @@ -180,7 +180,7 @@ func TestUpdateVulnerableComponent(t *testing.T) { func TestBuildImpactPaths(t *testing.T) { // create sample scan result and dependency trees - scanResult := []services.ScanResponse{ + scanResult := []*services.ScanResponse{ { Vulnerabilities: []services.Vulnerability{ { diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 64b2724f..b59683f8 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -140,12 +140,13 @@ func executeScaScan(auditParallelRunner *utils.SecurityParallelRunner, serverDet auditParallelRunner.ResultsMu.Lock() scan.XrayResults = append(scan.XrayResults, scanResults...) auditParallelRunner.ResultsMu.Unlock() + log.Info("finish running sca scan, got results from xray. the first cve is: " + scanResults[0].Vulnerabilities[0].IssueId) return } } func runScaWithTech(tech techutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, - flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []services.ScanResponse, err error) { + flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []*services.ScanResponse, err error) { scanGraphParams := scangraph.NewScanGraphParams(). SetServerDetails(serverDetails). SetXrayGraphScanParams(params.createXrayGraphScanParams()). diff --git a/commands/scan/buildscan.go b/commands/scan/buildscan.go index 43e9f2bb..7b1a5c97 100644 --- a/commands/scan/buildscan.go +++ b/commands/scan/buildscan.go @@ -123,7 +123,7 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS log.Info("The scan data is available at: " + buildScanResults.MoreDetailsUrl) isFailBuildResponse = buildScanResults.FailBuild - scanResponse := []services.ScanResponse{{ + scanResponse := []*services.ScanResponse{{ Violations: buildScanResults.Violations, Vulnerabilities: buildScanResults.Vulnerabilities, XrayDataUrl: buildScanResults.MoreDetailsUrl, diff --git a/commands/scan/scan.go b/commands/scan/scan.go index 25d6253b..5c870d30 100644 --- a/commands/scan/scan.go +++ b/commands/scan/scan.go @@ -264,7 +264,7 @@ func (scanCmd *ScanCommand) Run() (err error) { for _, arr := range resultsArr { for _, res := range arr { - flatResults = append(flatResults, &xrutils.ScaScanResult{Target: res.Target, XrayResults: []services.ScanResponse{*res.Result}}) + flatResults = append(flatResults, &xrutils.ScaScanResult{Target: res.Target, XrayResults: []*services.ScanResponse{res.Result}}) scanResults.ExtendedScanResults.ApplicabilityScanResults = append(scanResults.ExtendedScanResults.ApplicabilityScanResults, res.ExtendedScanResults.ApplicabilityScanResults...) scanResults.ExtendedScanResults.SecretsScanResults = append(scanResults.ExtendedScanResults.SecretsScanResults, res.ExtendedScanResults.SecretsScanResults...) } @@ -410,7 +410,7 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledFo jasErrors[threadId] = append(jasErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()}) } workingDirs := []string{filePath} - err = runner.RunJasScannersAndSetResults(jasFileProducerConsumer, &extendedScanResults, []techutils.Technology{techutils.Technology(scanResults.ScannedPackageType)}, []services.ScanResponse{*scanResults}, depsListFromVulnerabilities(*scanResults), scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) + err = runner.RunJasScannersAndSetResults(jasFileProducerConsumer, &extendedScanResults, []techutils.Technology{techutils.Technology(scanResults.ScannedPackageType)}, []*services.ScanResponse{scanResults}, depsListFromVulnerabilities(*scanResults), scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) if err != nil { log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error())) diff --git a/jas/applicability/applicabilitymanager.go b/jas/applicability/applicabilitymanager.go index ad385b55..b9ae5533 100644 --- a/jas/applicability/applicabilitymanager.go +++ b/jas/applicability/applicabilitymanager.go @@ -28,7 +28,7 @@ type ApplicabilityScanManager struct { applicabilityScanResults []*sarif.Run directDependenciesCves []string indirectDependenciesCves []string - xrayResults []services.ScanResponse + xrayResults []*services.ScanResponse scanner *jas.JasScanner thirdPartyScan bool commandType string @@ -45,7 +45,7 @@ type ApplicabilityScanManager struct { // map[string]string: A map containing the applicability result of each XRAY CVE. // bool: true if the user is entitled to the applicability scan, false otherwise. // error: An error object (if any). -func RunApplicabilityScan(xrayResults []services.ScanResponse, directDependencies []string, +func RunApplicabilityScan(xrayResults []*services.ScanResponse, directDependencies []string, scanner *jas.JasScanner, thirdPartyContextualAnalysis bool, scanType ApplicabilityScanType, module jfrogappsconfig.Module, threadId int) (results []*sarif.Run, err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)); err != nil { @@ -68,7 +68,7 @@ func RunApplicabilityScan(xrayResults []services.ScanResponse, directDependencie return } -func newApplicabilityScanManager(xrayScanResults []services.ScanResponse, directDependencies []string, scanner *jas.JasScanner, thirdPartyScan bool, scanType ApplicabilityScanType, scannerTempDir string) (manager *ApplicabilityScanManager) { +func newApplicabilityScanManager(xrayScanResults []*services.ScanResponse, directDependencies []string, scanner *jas.JasScanner, thirdPartyScan bool, scanType ApplicabilityScanType, scannerTempDir string) (manager *ApplicabilityScanManager) { directDependenciesCves, indirectDependenciesCves := extractDependenciesCvesFromScan(xrayScanResults, directDependencies) return &ApplicabilityScanManager{ applicabilityScanResults: []*sarif.Run{}, @@ -93,7 +93,7 @@ func addCvesToSet(cves []services.Cve, set *datastructures.Set[string]) { // This function gets a list of xray scan responses that contain direct and indirect vulnerabilities and returns separate // lists of the direct and indirect CVEs -func extractDependenciesCvesFromScan(xrayScanResults []services.ScanResponse, directDependencies []string) (directCves []string, indirectCves []string) { +func extractDependenciesCvesFromScan(xrayScanResults []*services.ScanResponse, directDependencies []string) (directCves []string, indirectCves []string) { directCvesSet := datastructures.MakeSet[string]() indirectCvesSet := datastructures.MakeSet[string]() for _, scanResult := range xrayScanResults { diff --git a/jas/applicability/applicabilitymanager_test.go b/jas/applicability/applicabilitymanager_test.go index d838ff2d..e2be2e69 100644 --- a/jas/applicability/applicabilitymanager_test.go +++ b/jas/applicability/applicabilitymanager_test.go @@ -50,7 +50,7 @@ func TestNewApplicabilityScanManager_DependencyTreeDoesntExist(t *testing.T) { func TestNewApplicabilityScanManager_NoDirectDependenciesInScan(t *testing.T) { // Arrange - var noDirectDependenciesResults = []services.ScanResponse{ + var noDirectDependenciesResults = []*services.ScanResponse{ { ScanId: "scanId_1", Vulnerabilities: []services.Vulnerability{ @@ -106,7 +106,7 @@ func TestNewApplicabilityScanManager_MultipleDependencyTrees(t *testing.T) { func TestNewApplicabilityScanManager_ViolationsDontExistInResults(t *testing.T) { // Arrange - noViolationScanResponse := []services.ScanResponse{ + noViolationScanResponse := []*services.ScanResponse{ { ScanId: "scanId_1", Vulnerabilities: []services.Vulnerability{ @@ -132,7 +132,7 @@ func TestNewApplicabilityScanManager_ViolationsDontExistInResults(t *testing.T) func TestNewApplicabilityScanManager_VulnerabilitiesDontExist(t *testing.T) { // Arrange - noVulnerabilitiesScanResponse := []services.ScanResponse{ + noVulnerabilitiesScanResponse := []*services.ScanResponse{ { ScanId: "scanId_1", Violations: []services.Violation{ @@ -157,7 +157,7 @@ func TestNewApplicabilityScanManager_VulnerabilitiesDontExist(t *testing.T) { } func TestExtractXrayDirectViolations(t *testing.T) { - var xrayResponseForDirectViolationsTest = []services.ScanResponse{ + var xrayResponseForDirectViolationsTest = []*services.ScanResponse{ { Violations: []services.Violation{ {IssueId: "issueId_2", Technology: techutils.Pipenv.String(), @@ -194,7 +194,7 @@ func TestExtractXrayDirectViolations(t *testing.T) { } func TestExtractXrayDirectVulnerabilities(t *testing.T) { - var xrayResponseForDirectVulnerabilitiesTest = []services.ScanResponse{ + var xrayResponseForDirectVulnerabilitiesTest = []*services.ScanResponse{ { ScanId: "scanId_1", Vulnerabilities: []services.Vulnerability{ diff --git a/jas/common.go b/jas/common.go index 4433f882..04214d7c 100644 --- a/jas/common.go +++ b/jas/common.go @@ -198,7 +198,7 @@ var FakeServerDetails = config.ServerDetails{ User: "user", } -var FakeBasicXrayResults = []services.ScanResponse{ +var FakeBasicXrayResults = []*services.ScanResponse{ { ScanId: "scanId_1", Vulnerabilities: []services.Vulnerability{ diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index 06215ddd..e15fe923 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/jfrog/gofrog/parallel" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" - "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-security/jas" "github.com/jfrog/jfrog-cli-security/jas/applicability" @@ -18,7 +17,7 @@ import ( "github.com/jfrog/jfrog-client-go/xray/services" ) -func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelRunner, extendedScanResults *utils.ExtendedScanResults, technologiesList []techutils.Technology, xrayScanResults []services.ScanResponse, directDependencies []string, +func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelRunner, extendedScanResults *utils.ExtendedScanResults, technologiesList []techutils.Technology, xrayScanResults []*services.ScanResponse, directDependencies []string, serverDetails *config.ServerDetails, workingDirs []string, thirdPartyApplicabilityScan bool, msi string, scanType applicability.ApplicabilityScanType, secretsScanType secrets.SecretsScanType, errHandlerFunc func(error)) (err error) { if serverDetails == nil || len(serverDetails.Url) == 0 { log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.") @@ -76,7 +75,6 @@ func addModuleJasScanTask(module jfrogappsconfig.Module, scanType utils.JasScanT securityParallelRunner.ScannersWg.Add(1) if _, err = securityParallelRunner.Runner.AddTaskWithError(task, errHandlerFunc); err != nil { err = fmt.Errorf("failed to create %s scan task: %s", scanType, err.Error()) - //securityParallelRunner.ScannersWg.Done() } return } @@ -133,7 +131,7 @@ func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner * } func runContextualScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, - xrayScanResults []services.ScanResponse, module jfrogappsconfig.Module, directDependencies []string, thirdPartyApplicabilityScan bool, scanType applicability.ApplicabilityScanType) parallel.TaskFunc { + xrayScanResults []*services.ScanResponse, module jfrogappsconfig.Module, directDependencies []string, thirdPartyApplicabilityScan bool, scanType applicability.ApplicabilityScanType) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { securityParallelRunner.ScannersWg.Done() diff --git a/utils/analyticsmetrics_test.go b/utils/analyticsmetrics_test.go index 7e8b8890..6f77a711 100644 --- a/utils/analyticsmetrics_test.go +++ b/utils/analyticsmetrics_test.go @@ -75,7 +75,7 @@ func TestAnalyticsMetricsService_createAuditResultsFromXscAnalyticsBasicGeneralE usageCallback := tests.SetEnvWithCallbackAndAssert(t, coreutils.ReportUsage, "true") defer usageCallback() vulnerabilities := []services.Vulnerability{{IssueId: "XRAY-ID", Cves: []services.Cve{{Id: "CVE-123"}}, Components: map[string]services.Component{"issueId_2_direct_dependency": {}}}} - scaResults := []*ScaScanResult{{XrayResults: []services.ScanResponse{{Vulnerabilities: vulnerabilities}}}} + scaResults := []*ScaScanResult{{XrayResults: []*services.ScanResponse{{Vulnerabilities: vulnerabilities}}}} auditResults := Results{ ScaResults: scaResults, ExtendedScanResults: &ExtendedScanResults{ diff --git a/utils/results.go b/utils/results.go index 0023edd1..cd93f6e0 100644 --- a/utils/results.go +++ b/utils/results.go @@ -22,7 +22,7 @@ func NewAuditResults() *Results { return &Results{ExtendedScanResults: &ExtendedScanResults{}} } -func (r *Results) GetScaScansXrayResults() (results []services.ScanResponse) { +func (r *Results) GetScaScansXrayResults() (results []*services.ScanResponse) { for _, scaResult := range r.ScaResults { results = append(results, scaResult.XrayResults...) } @@ -119,11 +119,11 @@ func (r *Results) getScanSummaryByTargets(targets ...string) (summaries []format type ScaScanResult struct { // Could be working directory (audit), file path (binary scan) or build name+number (build scan) - Target string `json:"Target"` - Technology techutils.Technology `json:"Technology,omitempty"` - XrayResults []services.ScanResponse `json:"XrayResults,omitempty"` - Descriptors []string `json:"Descriptors,omitempty"` - IsMultipleRootProject *bool `json:"IsMultipleRootProject,omitempty"` + Target string `json:"Target"` + Technology techutils.Technology `json:"Technology,omitempty"` + XrayResults []*services.ScanResponse `json:"XrayResults,omitempty"` + Descriptors []string `json:"Descriptors,omitempty"` + IsMultipleRootProject *bool `json:"IsMultipleRootProject,omitempty"` } func (s ScaScanResult) HasInformation() bool { diff --git a/utils/results_test.go b/utils/results_test.go index 9ca45923..a4d35487 100644 --- a/utils/results_test.go +++ b/utils/results_test.go @@ -172,7 +172,7 @@ func TestGetSummary(t *testing.T) { } } -func getDummyScaTestResults(vulnerability, violation bool) (responses []services.ScanResponse) { +func getDummyScaTestResults(vulnerability, violation bool) (responses []*services.ScanResponse) { response := services.ScanResponse{} switch { case vulnerability && violation: @@ -197,6 +197,6 @@ func getDummyScaTestResults(vulnerability, violation bool) (responses []services {ViolationType: formats.ViolationTypeLicense.String(), WatchName: "test-watch-name", IssueId: "MIT", Severity: "High", LicenseKey: "MIT", Components: map[string]services.Component{"issueId_direct_dependency": {}}}, } } - responses = append(responses, response) + responses = append(responses, &response) return } diff --git a/utils/resultstable_test.go b/utils/resultstable_test.go index af83a690..7fc5c0e3 100644 --- a/utils/resultstable_test.go +++ b/utils/resultstable_test.go @@ -27,7 +27,7 @@ func TestPrintViolationsTable(t *testing.T) { for _, test := range tests { err := PrintViolationsTable(test.violations, NewAuditResults(), false, true, services.Binary) assert.NoError(t, err) - if CheckIfFailBuild([]services.ScanResponse{{Violations: test.violations}}) { + if CheckIfFailBuild([]*services.ScanResponse{{Violations: test.violations}}) { err = NewFailBuildError() } assert.Equal(t, test.expectedError, err != nil) diff --git a/utils/resultwriter.go b/utils/resultwriter.go index 91d76d99..662e563d 100644 --- a/utils/resultwriter.go +++ b/utils/resultwriter.go @@ -548,7 +548,7 @@ func PrintSarif(results *Results, isMultipleRoots, includeLicenses bool) error { return nil } -func CheckIfFailBuild(results []services.ScanResponse) bool { +func CheckIfFailBuild(results []*services.ScanResponse) bool { for _, result := range results { for _, violation := range result.Violations { if violation.FailBuild { diff --git a/utils/resultwriter_test.go b/utils/resultwriter_test.go index 1180070a..40bf96bc 100644 --- a/utils/resultwriter_test.go +++ b/utils/resultwriter_test.go @@ -389,7 +389,7 @@ func TestConvertXrayScanToSimpleJson(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { results := NewAuditResults() - scaScanResult := ScaScanResult{XrayResults: []services.ScanResponse{tc.result}} + scaScanResult := ScaScanResult{XrayResults: []*services.ScanResponse{&tc.result}} results.ScaResults = append(results.ScaResults, &scaScanResult) output, err := ConvertXrayScanToSimpleJson(results, false, tc.includeLicenses, true, tc.allowedLicenses) if assert.NoError(t, err) { From 1ee4f40da8a2fc535fe5c9ef4472d37ae3bdc529 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 6 Jun 2024 17:30:35 +0300 Subject: [PATCH 19/59] change xray results to pointers array --- commands/audit/scarunner.go | 1 - utils/resultwriter_test.go | 14 +++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index b59683f8..bda474d1 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -140,7 +140,6 @@ func executeScaScan(auditParallelRunner *utils.SecurityParallelRunner, serverDet auditParallelRunner.ResultsMu.Lock() scan.XrayResults = append(scan.XrayResults, scanResults...) auditParallelRunner.ResultsMu.Unlock() - log.Info("finish running sca scan, got results from xray. the first cve is: " + scanResults[0].Vulnerabilities[0].IssueId) return } } diff --git a/utils/resultwriter_test.go b/utils/resultwriter_test.go index 40bf96bc..cc25664f 100644 --- a/utils/resultwriter_test.go +++ b/utils/resultwriter_test.go @@ -337,7 +337,7 @@ func TestConvertXrayScanToSimpleJson(t *testing.T) { testCases := []struct { name string - result services.ScanResponse + result *services.ScanResponse includeLicenses bool allowedLicenses []string expectedOutput formats.SimpleJsonResults @@ -346,21 +346,21 @@ func TestConvertXrayScanToSimpleJson(t *testing.T) { name: "Vulnerabilities only", includeLicenses: false, allowedLicenses: nil, - result: services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, + result: &services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, expectedOutput: formats.SimpleJsonResults{Vulnerabilities: expectedVulnerabilities}, }, { name: "Vulnerabilities with licenses", includeLicenses: true, allowedLicenses: nil, - result: services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, + result: &services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, expectedOutput: formats.SimpleJsonResults{Vulnerabilities: expectedVulnerabilities, Licenses: expectedLicenses}, }, { name: "Vulnerabilities only - with allowed licenses", includeLicenses: false, allowedLicenses: []string{"license-1"}, - result: services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, + result: &services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, expectedOutput: formats.SimpleJsonResults{ Vulnerabilities: expectedVulnerabilities, LicensesViolations: []formats.LicenseRow{ @@ -375,21 +375,21 @@ func TestConvertXrayScanToSimpleJson(t *testing.T) { name: "Violations only", includeLicenses: false, allowedLicenses: nil, - result: services.ScanResponse{Violations: violations, Licenses: licenses}, + result: &services.ScanResponse{Violations: violations, Licenses: licenses}, expectedOutput: formats.SimpleJsonResults{SecurityViolations: expectedSecViolations, LicensesViolations: expectedLicViolations}, }, { name: "Violations - override allowed licenses", includeLicenses: false, allowedLicenses: []string{"license-1"}, - result: services.ScanResponse{Violations: violations, Licenses: licenses}, + result: &services.ScanResponse{Violations: violations, Licenses: licenses}, expectedOutput: formats.SimpleJsonResults{SecurityViolations: expectedSecViolations, LicensesViolations: expectedLicViolations}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { results := NewAuditResults() - scaScanResult := ScaScanResult{XrayResults: []*services.ScanResponse{&tc.result}} + scaScanResult := ScaScanResult{XrayResults: []*services.ScanResponse{tc.result}} results.ScaResults = append(results.ScaResults, &scaScanResult) output, err := ConvertXrayScanToSimpleJson(results, false, tc.includeLicenses, true, tc.allowedLicenses) if assert.NoError(t, err) { From 9b84810611df0121cc123ebdc085e33404ce592b Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Fri, 7 Jun 2024 20:57:31 +0300 Subject: [PATCH 20/59] fix pointers issue --- commands/audit/audit.go | 2 +- commands/audit/sca/common.go | 6 +++--- commands/audit/sca/common_test.go | 2 +- commands/audit/scarunner.go | 2 +- commands/scan/buildscan.go | 2 +- commands/scan/scan.go | 13 ++++++++----- jas/applicability/applicabilitymanager.go | 8 ++++---- .../applicabilitymanager_test.go | 10 +++++----- jas/common.go | 2 +- jas/runner/jasrunner.go | 19 +++++++++---------- jas/runner/jasrunner_test.go | 6 +++--- utils/analyticsmetrics_test.go | 2 +- utils/results.go | 12 ++++++------ utils/results_test.go | 4 ++-- utils/resultstable_test.go | 2 +- utils/resultwriter.go | 2 +- utils/resultwriter_test.go | 14 +++++++------- 17 files changed, 55 insertions(+), 53 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 47f7843b..d7e90c93 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -242,7 +242,7 @@ func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.SecurityPa if err := xrayutils.DownloadAnalyzerManagerIfNeeded(threadId); err != nil { return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) } - if err := runner.RunJasScannersAndSetResults(auditParallelRunner, scanResults.ExtendedScanResults, scanResults.GetScaScannedTechnologies(), scanResults.GetScaScansXrayResults(), auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.thirdPartyApplicabilityScan, auditParams.commonGraphScanParams.MultiScanId, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan); err != nil { + if err := runner.RunJasScannersAndSetResults(auditParallelRunner, scanResults, scanResults.GetScaScannedTechnologies(), auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.thirdPartyApplicabilityScan, auditParams.commonGraphScanParams.MultiScanId, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan); err != nil { return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) } return nil diff --git a/commands/audit/sca/common.go b/commands/audit/sca/common.go index 75c07b7d..2046e7c5 100644 --- a/commands/audit/sca/common.go +++ b/commands/audit/sca/common.go @@ -32,7 +32,7 @@ func GetExcludePattern(params utils.AuditParams) string { return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, params.IsRecursiveScan()) } -func RunXrayDependenciesTreeScanGraph(dependencyTree xrayUtils.GraphNode, technology techutils.Technology, scanGraphParams *scangraph.ScanGraphParams) (results []*services.ScanResponse, err error) { +func RunXrayDependenciesTreeScanGraph(dependencyTree xrayUtils.GraphNode, technology techutils.Technology, scanGraphParams *scangraph.ScanGraphParams) (results []services.ScanResponse, err error) { scanGraphParams.XrayGraphScanParams().DependenciesGraph = &dependencyTree xscGitInfoContext := scanGraphParams.XrayGraphScanParams().XscGitInfoContext if xscGitInfoContext != nil { @@ -56,7 +56,7 @@ func RunXrayDependenciesTreeScanGraph(dependencyTree xrayUtils.GraphNode, techno for i := range scanResults.Violations { scanResults.Violations[i].Technology = technology.String() } - results = append(results, scanResults) + results = append(results, *scanResults) return } @@ -82,7 +82,7 @@ func LogExecutableVersion(executable string) { // BuildImpactPathsForScanResponse builds the full impact paths for each vulnerability found in the scanResult argument, using the dependencyTrees argument. // Returns the updated services.ScanResponse slice. -func BuildImpactPathsForScanResponse(scanResult []*services.ScanResponse, dependencyTree []*xrayUtils.GraphNode) []*services.ScanResponse { +func BuildImpactPathsForScanResponse(scanResult []services.ScanResponse, dependencyTree []*xrayUtils.GraphNode) []services.ScanResponse { for _, result := range scanResult { if len(result.Vulnerabilities) > 0 { buildVulnerabilitiesImpactPaths(result.Vulnerabilities, dependencyTree) diff --git a/commands/audit/sca/common_test.go b/commands/audit/sca/common_test.go index a093ed5b..be8b42fc 100644 --- a/commands/audit/sca/common_test.go +++ b/commands/audit/sca/common_test.go @@ -180,7 +180,7 @@ func TestUpdateVulnerableComponent(t *testing.T) { func TestBuildImpactPaths(t *testing.T) { // create sample scan result and dependency trees - scanResult := []*services.ScanResponse{ + scanResult := []services.ScanResponse{ { Vulnerabilities: []services.Vulnerability{ { diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index bda474d1..64b2724f 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -145,7 +145,7 @@ func executeScaScan(auditParallelRunner *utils.SecurityParallelRunner, serverDet } func runScaWithTech(tech techutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, - flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []*services.ScanResponse, err error) { + flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []services.ScanResponse, err error) { scanGraphParams := scangraph.NewScanGraphParams(). SetServerDetails(serverDetails). SetXrayGraphScanParams(params.createXrayGraphScanParams()). diff --git a/commands/scan/buildscan.go b/commands/scan/buildscan.go index 7b1a5c97..43e9f2bb 100644 --- a/commands/scan/buildscan.go +++ b/commands/scan/buildscan.go @@ -123,7 +123,7 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS log.Info("The scan data is available at: " + buildScanResults.MoreDetailsUrl) isFailBuildResponse = buildScanResults.FailBuild - scanResponse := []*services.ScanResponse{{ + scanResponse := []services.ScanResponse{{ Violations: buildScanResults.Violations, Vulnerabilities: buildScanResults.Vulnerabilities, XrayDataUrl: buildScanResults.MoreDetailsUrl, diff --git a/commands/scan/scan.go b/commands/scan/scan.go index 5c870d30..283275b0 100644 --- a/commands/scan/scan.go +++ b/commands/scan/scan.go @@ -264,7 +264,7 @@ func (scanCmd *ScanCommand) Run() (err error) { for _, arr := range resultsArr { for _, res := range arr { - flatResults = append(flatResults, &xrutils.ScaScanResult{Target: res.Target, XrayResults: []*services.ScanResponse{res.Result}}) + flatResults = append(flatResults, &xrutils.ScaScanResult{Target: res.Target, XrayResults: []services.ScanResponse{*res.Result}}) scanResults.ExtendedScanResults.ApplicabilityScanResults = append(scanResults.ExtendedScanResults.ApplicabilityScanResults, res.ExtendedScanResults.ApplicabilityScanResults...) scanResults.ExtendedScanResults.SecretsScanResults = append(scanResults.ExtendedScanResults.SecretsScanResults, res.ExtendedScanResults.SecretsScanResults...) } @@ -396,21 +396,24 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledFo if err != nil { return err } - scanResults, err := scangraph.RunScanGraphAndGetResults(scanGraphParams, xrayManager) + graphScanResults, err := scangraph.RunScanGraphAndGetResults(scanGraphParams, xrayManager) if err != nil { log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error())) indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()}) return } - extendedScanResults := utils.ExtendedScanResults{} + scanResults := utils.Results{ + ScaResults: []*utils.ScaScanResult{{XrayResults: []services.ScanResponse{*graphScanResults}}}, + ExtendedScanResults: &utils.ExtendedScanResults{}, + } if entitledForJas && scanCmd.commandSupportsJAS { // Run Jas scans jasErrHandlerFunc := func(err error) { jasErrors[threadId] = append(jasErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()}) } workingDirs := []string{filePath} - err = runner.RunJasScannersAndSetResults(jasFileProducerConsumer, &extendedScanResults, []techutils.Technology{techutils.Technology(scanResults.ScannedPackageType)}, []*services.ScanResponse{scanResults}, depsListFromVulnerabilities(*scanResults), scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) + err = runner.RunJasScannersAndSetResults(jasFileProducerConsumer, &scanResults, []techutils.Technology{techutils.Technology(graphScanResults.ScannedPackageType)}, depsListFromVulnerabilities(*graphScanResults), scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) if err != nil { log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error())) @@ -418,7 +421,7 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledFo return } } - resultsArr[threadId] = append(resultsArr[threadId], &ScanInfo{Target: filePath, Result: scanResults, ExtendedScanResults: &extendedScanResults}) + resultsArr[threadId] = append(resultsArr[threadId], &ScanInfo{Target: filePath, Result: graphScanResults, ExtendedScanResults: scanResults.ExtendedScanResults}) return } diff --git a/jas/applicability/applicabilitymanager.go b/jas/applicability/applicabilitymanager.go index b9ae5533..ad385b55 100644 --- a/jas/applicability/applicabilitymanager.go +++ b/jas/applicability/applicabilitymanager.go @@ -28,7 +28,7 @@ type ApplicabilityScanManager struct { applicabilityScanResults []*sarif.Run directDependenciesCves []string indirectDependenciesCves []string - xrayResults []*services.ScanResponse + xrayResults []services.ScanResponse scanner *jas.JasScanner thirdPartyScan bool commandType string @@ -45,7 +45,7 @@ type ApplicabilityScanManager struct { // map[string]string: A map containing the applicability result of each XRAY CVE. // bool: true if the user is entitled to the applicability scan, false otherwise. // error: An error object (if any). -func RunApplicabilityScan(xrayResults []*services.ScanResponse, directDependencies []string, +func RunApplicabilityScan(xrayResults []services.ScanResponse, directDependencies []string, scanner *jas.JasScanner, thirdPartyContextualAnalysis bool, scanType ApplicabilityScanType, module jfrogappsconfig.Module, threadId int) (results []*sarif.Run, err error) { var scannerTempDir string if scannerTempDir, err = jas.CreateScannerTempDirectory(scanner, string(utils.Applicability)); err != nil { @@ -68,7 +68,7 @@ func RunApplicabilityScan(xrayResults []*services.ScanResponse, directDependenci return } -func newApplicabilityScanManager(xrayScanResults []*services.ScanResponse, directDependencies []string, scanner *jas.JasScanner, thirdPartyScan bool, scanType ApplicabilityScanType, scannerTempDir string) (manager *ApplicabilityScanManager) { +func newApplicabilityScanManager(xrayScanResults []services.ScanResponse, directDependencies []string, scanner *jas.JasScanner, thirdPartyScan bool, scanType ApplicabilityScanType, scannerTempDir string) (manager *ApplicabilityScanManager) { directDependenciesCves, indirectDependenciesCves := extractDependenciesCvesFromScan(xrayScanResults, directDependencies) return &ApplicabilityScanManager{ applicabilityScanResults: []*sarif.Run{}, @@ -93,7 +93,7 @@ func addCvesToSet(cves []services.Cve, set *datastructures.Set[string]) { // This function gets a list of xray scan responses that contain direct and indirect vulnerabilities and returns separate // lists of the direct and indirect CVEs -func extractDependenciesCvesFromScan(xrayScanResults []*services.ScanResponse, directDependencies []string) (directCves []string, indirectCves []string) { +func extractDependenciesCvesFromScan(xrayScanResults []services.ScanResponse, directDependencies []string) (directCves []string, indirectCves []string) { directCvesSet := datastructures.MakeSet[string]() indirectCvesSet := datastructures.MakeSet[string]() for _, scanResult := range xrayScanResults { diff --git a/jas/applicability/applicabilitymanager_test.go b/jas/applicability/applicabilitymanager_test.go index e2be2e69..d838ff2d 100644 --- a/jas/applicability/applicabilitymanager_test.go +++ b/jas/applicability/applicabilitymanager_test.go @@ -50,7 +50,7 @@ func TestNewApplicabilityScanManager_DependencyTreeDoesntExist(t *testing.T) { func TestNewApplicabilityScanManager_NoDirectDependenciesInScan(t *testing.T) { // Arrange - var noDirectDependenciesResults = []*services.ScanResponse{ + var noDirectDependenciesResults = []services.ScanResponse{ { ScanId: "scanId_1", Vulnerabilities: []services.Vulnerability{ @@ -106,7 +106,7 @@ func TestNewApplicabilityScanManager_MultipleDependencyTrees(t *testing.T) { func TestNewApplicabilityScanManager_ViolationsDontExistInResults(t *testing.T) { // Arrange - noViolationScanResponse := []*services.ScanResponse{ + noViolationScanResponse := []services.ScanResponse{ { ScanId: "scanId_1", Vulnerabilities: []services.Vulnerability{ @@ -132,7 +132,7 @@ func TestNewApplicabilityScanManager_ViolationsDontExistInResults(t *testing.T) func TestNewApplicabilityScanManager_VulnerabilitiesDontExist(t *testing.T) { // Arrange - noVulnerabilitiesScanResponse := []*services.ScanResponse{ + noVulnerabilitiesScanResponse := []services.ScanResponse{ { ScanId: "scanId_1", Violations: []services.Violation{ @@ -157,7 +157,7 @@ func TestNewApplicabilityScanManager_VulnerabilitiesDontExist(t *testing.T) { } func TestExtractXrayDirectViolations(t *testing.T) { - var xrayResponseForDirectViolationsTest = []*services.ScanResponse{ + var xrayResponseForDirectViolationsTest = []services.ScanResponse{ { Violations: []services.Violation{ {IssueId: "issueId_2", Technology: techutils.Pipenv.String(), @@ -194,7 +194,7 @@ func TestExtractXrayDirectViolations(t *testing.T) { } func TestExtractXrayDirectVulnerabilities(t *testing.T) { - var xrayResponseForDirectVulnerabilitiesTest = []*services.ScanResponse{ + var xrayResponseForDirectVulnerabilitiesTest = []services.ScanResponse{ { ScanId: "scanId_1", Vulnerabilities: []services.Vulnerability{ diff --git a/jas/common.go b/jas/common.go index 04214d7c..4433f882 100644 --- a/jas/common.go +++ b/jas/common.go @@ -198,7 +198,7 @@ var FakeServerDetails = config.ServerDetails{ User: "user", } -var FakeBasicXrayResults = []*services.ScanResponse{ +var FakeBasicXrayResults = []services.ScanResponse{ { ScanId: "scanId_1", Vulnerabilities: []services.Vulnerability{ diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index e15fe923..4f7f536c 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -14,10 +14,9 @@ import ( "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-cli-security/utils/techutils" "github.com/jfrog/jfrog-client-go/utils/log" - "github.com/jfrog/jfrog-client-go/xray/services" ) -func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelRunner, extendedScanResults *utils.ExtendedScanResults, technologiesList []techutils.Technology, xrayScanResults []*services.ScanResponse, directDependencies []string, +func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, technologiesList []techutils.Technology, directDependencies []string, serverDetails *config.ServerDetails, workingDirs []string, thirdPartyApplicabilityScan bool, msi string, scanType applicability.ApplicabilityScanType, secretsScanType secrets.SecretsScanType, errHandlerFunc func(error)) (err error) { if serverDetails == nil || len(serverDetails.Url) == 0 { log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.") @@ -44,14 +43,14 @@ func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelR // Don't execute other scanners when scanning third party dependencies. if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { - if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, extendedScanResults, module, secretsScanType), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module, secretsScanType), errHandlerFunc); err != nil { return } if runAllScanners { - if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, extendedScanResults, module), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { return } - if err = addModuleJasScanTask(module, utils.Sast, securityParallelRunner, runSastScan(securityParallelRunner, scanner, extendedScanResults, module), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.Sast, securityParallelRunner, runSastScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { return } } @@ -61,7 +60,7 @@ func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelR // Wait for sca scan to complete securityParallelRunner.ScaScansWg.Wait() for _, module := range scanner.JFrogAppsConfig.Modules { - if err = addModuleJasScanTask(module, utils.Applicability, securityParallelRunner, runContextualScan(securityParallelRunner, scanner, extendedScanResults, xrayScanResults, module, directDependencies, thirdPartyApplicabilityScan, scanType), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.Applicability, securityParallelRunner, runContextualScan(securityParallelRunner, scanner, scanResults, module, directDependencies, thirdPartyApplicabilityScan, scanType), errHandlerFunc); err != nil { return } } @@ -130,18 +129,18 @@ func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner * } } -func runContextualScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, - xrayScanResults []*services.ScanResponse, module jfrogappsconfig.Module, directDependencies []string, thirdPartyApplicabilityScan bool, scanType applicability.ApplicabilityScanType) parallel.TaskFunc { +func runContextualScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, + module jfrogappsconfig.Module, directDependencies []string, thirdPartyApplicabilityScan bool, scanType applicability.ApplicabilityScanType) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { securityParallelRunner.ScannersWg.Done() }() - results, err := applicability.RunApplicabilityScan(xrayScanResults, directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) + results, err := applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) if err != nil { return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) } securityParallelRunner.ResultsMu.Lock() - extendedScanResults.ApplicabilityScanResults = append(extendedScanResults.ApplicabilityScanResults, results...) + scanResults.ExtendedScanResults.ApplicabilityScanResults = append(scanResults.ExtendedScanResults.ApplicabilityScanResults, results...) securityParallelRunner.ResultsMu.Unlock() return } diff --git a/jas/runner/jasrunner_test.go b/jas/runner/jasrunner_test.go index 57635b2b..8c10cc72 100644 --- a/jas/runner/jasrunner_test.go +++ b/jas/runner/jasrunner_test.go @@ -27,7 +27,7 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err = RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults.ExtendedScanResults, scanResults.GetScaScannedTechnologies(), scanResults.GetScaScansXrayResults(), []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + err = RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) // Expect error: assert.Error(t, err) } @@ -35,7 +35,7 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { securityParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults.ExtendedScanResults, scanResults.GetScaScannedTechnologies(), scanResults.GetScaScansXrayResults(), []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) assert.NoError(t, err) } @@ -44,7 +44,7 @@ func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults.ExtendedScanResults, scanResults.GetScaScannedTechnologies(), scanResults.GetScaScansXrayResults(), []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) // Expect error: assert.ErrorContains(t, err, "failed to run Applicability scan") diff --git a/utils/analyticsmetrics_test.go b/utils/analyticsmetrics_test.go index 6f77a711..7e8b8890 100644 --- a/utils/analyticsmetrics_test.go +++ b/utils/analyticsmetrics_test.go @@ -75,7 +75,7 @@ func TestAnalyticsMetricsService_createAuditResultsFromXscAnalyticsBasicGeneralE usageCallback := tests.SetEnvWithCallbackAndAssert(t, coreutils.ReportUsage, "true") defer usageCallback() vulnerabilities := []services.Vulnerability{{IssueId: "XRAY-ID", Cves: []services.Cve{{Id: "CVE-123"}}, Components: map[string]services.Component{"issueId_2_direct_dependency": {}}}} - scaResults := []*ScaScanResult{{XrayResults: []*services.ScanResponse{{Vulnerabilities: vulnerabilities}}}} + scaResults := []*ScaScanResult{{XrayResults: []services.ScanResponse{{Vulnerabilities: vulnerabilities}}}} auditResults := Results{ ScaResults: scaResults, ExtendedScanResults: &ExtendedScanResults{ diff --git a/utils/results.go b/utils/results.go index cd93f6e0..0023edd1 100644 --- a/utils/results.go +++ b/utils/results.go @@ -22,7 +22,7 @@ func NewAuditResults() *Results { return &Results{ExtendedScanResults: &ExtendedScanResults{}} } -func (r *Results) GetScaScansXrayResults() (results []*services.ScanResponse) { +func (r *Results) GetScaScansXrayResults() (results []services.ScanResponse) { for _, scaResult := range r.ScaResults { results = append(results, scaResult.XrayResults...) } @@ -119,11 +119,11 @@ func (r *Results) getScanSummaryByTargets(targets ...string) (summaries []format type ScaScanResult struct { // Could be working directory (audit), file path (binary scan) or build name+number (build scan) - Target string `json:"Target"` - Technology techutils.Technology `json:"Technology,omitempty"` - XrayResults []*services.ScanResponse `json:"XrayResults,omitempty"` - Descriptors []string `json:"Descriptors,omitempty"` - IsMultipleRootProject *bool `json:"IsMultipleRootProject,omitempty"` + Target string `json:"Target"` + Technology techutils.Technology `json:"Technology,omitempty"` + XrayResults []services.ScanResponse `json:"XrayResults,omitempty"` + Descriptors []string `json:"Descriptors,omitempty"` + IsMultipleRootProject *bool `json:"IsMultipleRootProject,omitempty"` } func (s ScaScanResult) HasInformation() bool { diff --git a/utils/results_test.go b/utils/results_test.go index a4d35487..9ca45923 100644 --- a/utils/results_test.go +++ b/utils/results_test.go @@ -172,7 +172,7 @@ func TestGetSummary(t *testing.T) { } } -func getDummyScaTestResults(vulnerability, violation bool) (responses []*services.ScanResponse) { +func getDummyScaTestResults(vulnerability, violation bool) (responses []services.ScanResponse) { response := services.ScanResponse{} switch { case vulnerability && violation: @@ -197,6 +197,6 @@ func getDummyScaTestResults(vulnerability, violation bool) (responses []*service {ViolationType: formats.ViolationTypeLicense.String(), WatchName: "test-watch-name", IssueId: "MIT", Severity: "High", LicenseKey: "MIT", Components: map[string]services.Component{"issueId_direct_dependency": {}}}, } } - responses = append(responses, &response) + responses = append(responses, response) return } diff --git a/utils/resultstable_test.go b/utils/resultstable_test.go index 7fc5c0e3..af83a690 100644 --- a/utils/resultstable_test.go +++ b/utils/resultstable_test.go @@ -27,7 +27,7 @@ func TestPrintViolationsTable(t *testing.T) { for _, test := range tests { err := PrintViolationsTable(test.violations, NewAuditResults(), false, true, services.Binary) assert.NoError(t, err) - if CheckIfFailBuild([]*services.ScanResponse{{Violations: test.violations}}) { + if CheckIfFailBuild([]services.ScanResponse{{Violations: test.violations}}) { err = NewFailBuildError() } assert.Equal(t, test.expectedError, err != nil) diff --git a/utils/resultwriter.go b/utils/resultwriter.go index 662e563d..91d76d99 100644 --- a/utils/resultwriter.go +++ b/utils/resultwriter.go @@ -548,7 +548,7 @@ func PrintSarif(results *Results, isMultipleRoots, includeLicenses bool) error { return nil } -func CheckIfFailBuild(results []*services.ScanResponse) bool { +func CheckIfFailBuild(results []services.ScanResponse) bool { for _, result := range results { for _, violation := range result.Violations { if violation.FailBuild { diff --git a/utils/resultwriter_test.go b/utils/resultwriter_test.go index cc25664f..1180070a 100644 --- a/utils/resultwriter_test.go +++ b/utils/resultwriter_test.go @@ -337,7 +337,7 @@ func TestConvertXrayScanToSimpleJson(t *testing.T) { testCases := []struct { name string - result *services.ScanResponse + result services.ScanResponse includeLicenses bool allowedLicenses []string expectedOutput formats.SimpleJsonResults @@ -346,21 +346,21 @@ func TestConvertXrayScanToSimpleJson(t *testing.T) { name: "Vulnerabilities only", includeLicenses: false, allowedLicenses: nil, - result: &services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, + result: services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, expectedOutput: formats.SimpleJsonResults{Vulnerabilities: expectedVulnerabilities}, }, { name: "Vulnerabilities with licenses", includeLicenses: true, allowedLicenses: nil, - result: &services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, + result: services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, expectedOutput: formats.SimpleJsonResults{Vulnerabilities: expectedVulnerabilities, Licenses: expectedLicenses}, }, { name: "Vulnerabilities only - with allowed licenses", includeLicenses: false, allowedLicenses: []string{"license-1"}, - result: &services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, + result: services.ScanResponse{Vulnerabilities: vulnerabilities, Licenses: licenses}, expectedOutput: formats.SimpleJsonResults{ Vulnerabilities: expectedVulnerabilities, LicensesViolations: []formats.LicenseRow{ @@ -375,21 +375,21 @@ func TestConvertXrayScanToSimpleJson(t *testing.T) { name: "Violations only", includeLicenses: false, allowedLicenses: nil, - result: &services.ScanResponse{Violations: violations, Licenses: licenses}, + result: services.ScanResponse{Violations: violations, Licenses: licenses}, expectedOutput: formats.SimpleJsonResults{SecurityViolations: expectedSecViolations, LicensesViolations: expectedLicViolations}, }, { name: "Violations - override allowed licenses", includeLicenses: false, allowedLicenses: []string{"license-1"}, - result: &services.ScanResponse{Violations: violations, Licenses: licenses}, + result: services.ScanResponse{Violations: violations, Licenses: licenses}, expectedOutput: formats.SimpleJsonResults{SecurityViolations: expectedSecViolations, LicensesViolations: expectedLicViolations}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { results := NewAuditResults() - scaScanResult := ScaScanResult{XrayResults: []*services.ScanResponse{tc.result}} + scaScanResult := ScaScanResult{XrayResults: []services.ScanResponse{tc.result}} results.ScaResults = append(results.ScaResults, &scaScanResult) output, err := ConvertXrayScanToSimpleJson(results, false, tc.includeLicenses, true, tc.allowedLicenses) if assert.NoError(t, err) { From 955299d4e63d05bcbd5fd9885ce686de58ea7c66 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Fri, 7 Jun 2024 22:40:11 +0300 Subject: [PATCH 21/59] change direct dependency to pointer --- commands/scan/scan.go | 3 ++- jas/runner/jasrunner.go | 4 ++-- jas/runner/jasrunner_test.go | 6 +++--- utils/auditbasicparams.go | 6 +++--- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/commands/scan/scan.go b/commands/scan/scan.go index 02177be3..a16bde91 100644 --- a/commands/scan/scan.go +++ b/commands/scan/scan.go @@ -419,7 +419,8 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledFo jasErrors[threadId] = append(jasErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()}) } workingDirs := []string{filePath} - err = runner.RunJasScannersAndSetResults(jasFileProducerConsumer, &scanResults, []techutils.Technology{techutils.Technology(graphScanResults.ScannedPackageType)}, depsListFromVulnerabilities(*graphScanResults), scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) + depsList := depsListFromVulnerabilities(*graphScanResults) + err = runner.RunJasScannersAndSetResults(jasFileProducerConsumer, &scanResults, []techutils.Technology{techutils.Technology(graphScanResults.ScannedPackageType)}, &depsList, scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) if err != nil { log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error())) diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index 4f7f536c..ed9a7f65 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -16,7 +16,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" ) -func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, technologiesList []techutils.Technology, directDependencies []string, +func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, technologiesList []techutils.Technology, directDependencies *[]string, serverDetails *config.ServerDetails, workingDirs []string, thirdPartyApplicabilityScan bool, msi string, scanType applicability.ApplicabilityScanType, secretsScanType secrets.SecretsScanType, errHandlerFunc func(error)) (err error) { if serverDetails == nil || len(serverDetails.Url) == 0 { log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.") @@ -60,7 +60,7 @@ func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelR // Wait for sca scan to complete securityParallelRunner.ScaScansWg.Wait() for _, module := range scanner.JFrogAppsConfig.Modules { - if err = addModuleJasScanTask(module, utils.Applicability, securityParallelRunner, runContextualScan(securityParallelRunner, scanner, scanResults, module, directDependencies, thirdPartyApplicabilityScan, scanType), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.Applicability, securityParallelRunner, runContextualScan(securityParallelRunner, scanner, scanResults, module, *directDependencies, thirdPartyApplicabilityScan, scanType), errHandlerFunc); err != nil { return } } diff --git a/jas/runner/jasrunner_test.go b/jas/runner/jasrunner_test.go index 8c10cc72..dd898e65 100644 --- a/jas/runner/jasrunner_test.go +++ b/jas/runner/jasrunner_test.go @@ -27,7 +27,7 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err = RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + err = RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) // Expect error: assert.Error(t, err) } @@ -35,7 +35,7 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { securityParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) assert.NoError(t, err) } @@ -44,7 +44,7 @@ func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) // Expect error: assert.ErrorContains(t, err, "failed to run Applicability scan") diff --git a/utils/auditbasicparams.go b/utils/auditbasicparams.go index e66a618f..b3d882a9 100644 --- a/utils/auditbasicparams.go +++ b/utils/auditbasicparams.go @@ -7,7 +7,7 @@ import ( ) type AuditParams interface { - DirectDependencies() []string + DirectDependencies() *[]string AppendDependenciesForApplicabilityScan(directDependencies []string) *AuditBasicParams ServerDetails() (*config.ServerDetails, error) SetServerDetails(serverDetails *config.ServerDetails) *AuditBasicParams @@ -63,8 +63,8 @@ type AuditBasicParams struct { isRecursiveScan bool } -func (abp *AuditBasicParams) DirectDependencies() []string { - return abp.dependenciesForApplicabilityScan +func (abp *AuditBasicParams) DirectDependencies() *[]string { + return &abp.dependenciesForApplicabilityScan } func (abp *AuditBasicParams) AppendDependenciesForApplicabilityScan(directDependencies []string) *AuditBasicParams { From afa30109c9791cda26e4e80c556811971faf45f5 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sat, 8 Jun 2024 08:47:57 +0300 Subject: [PATCH 22/59] fix unit test --- jas/runner/jasrunner_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jas/runner/jasrunner_test.go b/jas/runner/jasrunner_test.go index dd898e65..0d75117e 100644 --- a/jas/runner/jasrunner_test.go +++ b/jas/runner/jasrunner_test.go @@ -40,11 +40,12 @@ func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { } func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { - securityParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) - scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + jfrogAppsConfigForTest, _ := jas.CreateJFrogAppsConfig(nil) + scanner, _ := jas.NewJasScanner(nil, &jas.FakeServerDetails) + _, err := applicability.RunApplicabilityScan(jas.FakeBasicXrayResults, []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, + scanner, false, applicability.ApplicabilityScannerType, jfrogAppsConfigForTest.Modules[0], 0) // Expect error: assert.ErrorContains(t, err, "failed to run Applicability scan") From a565415f98fa0627b7eec56cecb51b83177ac26e Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Mon, 10 Jun 2024 13:36:40 +0300 Subject: [PATCH 23/59] cr changes --- audit_test.go | 4 ++-- commands/audit/audit.go | 14 ++++++-------- commands/audit/scarunner.go | 8 ++++---- jas/runner/jasrunner.go | 12 +++++++----- utils/parallel_runner.go | 6 +++--- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/audit_test.go b/audit_test.go index 7d6a3c61..4cd5b54d 100644 --- a/audit_test.go +++ b/audit_test.go @@ -568,6 +568,6 @@ func TestAuditOnEmptyProject(t *testing.T) { assert.NoError(t, err) chdirCallback := clientTests.ChangeDirWithCallback(t, baseWd, tempDirPath) defer chdirCallback() - output := securityTests.PlatformCli.WithoutCredentials().RunCliCmdWithOutput(t, "audit", "--format="+string(format.Json)) - assert.Equal(t, "null\n", output) + output := securityTests.PlatformCli.WithoutCredentials().RunCliCmdWithOutput(t, "audit", "--format="+string(format.SimpleJson)) + securityTestUtils.VerifySimpleJsonJasResults(t, output, 0, 0, 0, 0, 0, 0, 0) } diff --git a/commands/audit/audit.go b/commands/audit/audit.go index d7e90c93..59a647e7 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -199,23 +199,21 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) - if err != nil { - return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) - } if results.ExtendedScanResults.EntitledForJas { // Download (if needed) the analyzer manager and run scanners. auditParallelRunner.JasWg.Add(1) - _, jasErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { + if _, jasErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, serverDetails, auditParams, threadId) - }, auditParallelRunner.AddErrorToChan) - if jasErr != nil { + }, auditParallelRunner.AddErrorToChan); jasErr != nil { auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create AM downloading task, skipping JAS scans...: %s", jasErr.Error())) } } // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. - if scaScanErr := runScaScan(auditParallelRunner, auditParams, results); scaScanErr != nil { - auditParallelRunner.AddErrorToChan(scaScanErr) + if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { + return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) + }, auditParallelRunner.AddErrorToChan); scaErr != nil { + auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create sca scan task %s", scaErr.Error())) } go func() { auditParallelRunner.JasWg.Wait() diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 64b2724f..01beeffb 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -34,7 +34,7 @@ import ( "time" ) -func runScaScan(auditParallelRunner *utils.SecurityParallelRunner, auditParams *AuditParams, results *xrayutils.Results) (err error) { +func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner, auditParams *AuditParams, results *xrayutils.Results, threadId int) (err error) { // Prepare currentWorkingDir, err := os.Getwd() if errorutils.CheckError(err) != nil { @@ -54,7 +54,7 @@ func runScaScan(auditParallelRunner *utils.SecurityParallelRunner, auditParams * if err != nil { return } - log.Info(fmt.Sprintf("Preforming %d SCA scans:\n%s", len(scans), scanInfo)) + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Preforming", len(scans), "SCA scans:\n", scanInfo) defer func() { // Make sure to return to the original working directory, building the dependency tree may change it @@ -69,7 +69,7 @@ func runScaScan(auditParallelRunner *utils.SecurityParallelRunner, auditParams * } // Create sca scan task auditParallelRunner.ScaScansWg.Add(1) - _, taskErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, auditParams, scan, treeResult), func(err error) { + _, taskErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScanTask(auditParallelRunner, serverDetails, auditParams, scan, treeResult), func(err error) { auditParallelRunner.AddErrorToChan(fmt.Errorf("audit command in '%s' failed:\n%s", scan.Target, err.Error())) }) if taskErr != nil { @@ -123,7 +123,7 @@ func getRequestedDescriptors(params *AuditParams) map[techutils.Technology][]str } // Preform the SCA scan for the given scan information. -func executeScaScan(auditParallelRunner *utils.SecurityParallelRunner, serverDetails *config.ServerDetails, auditParams *AuditParams, +func executeScaScanTask(auditParallelRunner *utils.SecurityParallelRunner, serverDetails *config.ServerDetails, auditParams *AuditParams, scan *xrayutils.ScaScanResult, treeResult *DependencyTreeResult) parallel.TaskFunc { return func(threadId int) (err error) { log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Running SCA scan for", scan.Target, "vulnerable dependencies in", scan.Target, "directory...") diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index ed9a7f65..0c7f6b6a 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -13,6 +13,7 @@ import ( "github.com/jfrog/jfrog-cli-security/jas/secrets" "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-cli-security/utils/techutils" + clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" ) @@ -34,6 +35,7 @@ func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelR // Set environments variables for analytics in analyzers manager. callback := jas.SetAnalyticsMetricsDataForAnalyzerManager(msi, technologiesList) defer func() { + // Wait for all scanners to complete before cleaning up temp dir securityParallelRunner.ScannersWg.Wait() callback() cleanup := scanner.ScannerDirCleanupFunc @@ -57,7 +59,7 @@ func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelR } } - // Wait for sca scan to complete + // Wait for sca scan to complete before running contextual scan securityParallelRunner.ScaScansWg.Wait() for _, module := range scanner.JFrogAppsConfig.Modules { if err = addModuleJasScanTask(module, utils.Applicability, securityParallelRunner, runContextualScan(securityParallelRunner, scanner, scanResults, module, *directDependencies, thirdPartyApplicabilityScan, scanType), errHandlerFunc); err != nil { @@ -86,7 +88,7 @@ func runSecretsScan(securityParallelRunner *utils.SecurityParallelRunner, scanne }() results, err := secrets.RunSecretsScan(scanner, secretsScanType, module, threadId) if err != nil { - return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() extendedScanResults.SecretsScanResults = append(extendedScanResults.SecretsScanResults, results...) @@ -103,7 +105,7 @@ func runIacScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *j }() results, err := iac.RunIacScan(scanner, module, threadId) if err != nil { - return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() extendedScanResults.IacScanResults = append(extendedScanResults.IacScanResults, results...) @@ -120,7 +122,7 @@ func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner * }() results, err := sast.RunSastScan(scanner, module, threadId) if err != nil { - return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() extendedScanResults.SastScanResults = append(extendedScanResults.SastScanResults, results...) @@ -137,7 +139,7 @@ func runContextualScan(securityParallelRunner *utils.SecurityParallelRunner, sca }() results, err := applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) if err != nil { - return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() scanResults.ExtendedScanResults.ApplicabilityScanResults = append(scanResults.ExtendedScanResults.ApplicabilityScanResults, results...) diff --git a/utils/parallel_runner.go b/utils/parallel_runner.go index b6f7ebc7..368152a3 100644 --- a/utils/parallel_runner.go +++ b/utils/parallel_runner.go @@ -9,9 +9,9 @@ type SecurityParallelRunner struct { Runner parallel.Runner ErrorsQueue chan error ResultsMu sync.Mutex - ScaScansWg sync.WaitGroup // verify that the sca scan routines are done before running contextual scan - ScannersWg sync.WaitGroup // verify that all scanners routines are done before cleaning temp dir - JasWg sync.WaitGroup // verify that downloading analyzer manager and running all scanners are done + ScaScansWg sync.WaitGroup // Verify that the sca scan routines are done before running contextual scan + ScannersWg sync.WaitGroup // Verify that all scanners routines are done before cleaning temp dir + JasWg sync.WaitGroup // Verify that downloading analyzer manager and running all scanners are done } func NewSecurityParallelRunner(numOfParallelScans int) SecurityParallelRunner { From 917871e83d4ae2be99848be8ac8df90c3c2bc61f Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Mon, 10 Jun 2024 14:06:19 +0300 Subject: [PATCH 24/59] add wg --- commands/audit/audit.go | 1 + commands/audit/scarunner.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 59a647e7..c611755d 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -210,6 +210,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. + auditParallelRunner.ScaScansWg.Add(1) if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) }, auditParallelRunner.AddErrorToChan); scaErr != nil { diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 3e6085f8..8676023a 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -36,6 +36,9 @@ import ( func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner, auditParams *AuditParams, results *xrayutils.Results, threadId int) (err error) { // Prepare + defer func() { + auditParallelRunner.ScaScansWg.Done() + }() currentWorkingDir, err := os.Getwd() if errorutils.CheckError(err) != nil { return From ce5c1f4d7c526bd0789a52319d4d3c5ec393ceb5 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Mon, 10 Jun 2024 17:01:49 +0300 Subject: [PATCH 25/59] add debug logs --- commands/audit/audit.go | 6 +++--- commands/audit/scarunner.go | 2 +- commands/scan/scan.go | 2 +- jas/runner/jasrunner.go | 12 +++++++++--- jas/runner/jasrunner_test.go | 4 ++-- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index c611755d..8cb5b8e5 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -239,10 +239,10 @@ func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.SecurityPa auditParallelRunner.JasWg.Done() }() if err := xrayutils.DownloadAnalyzerManagerIfNeeded(threadId); err != nil { - return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + return fmt.Errorf("%s failed to download analyzer manager: %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } - if err := runner.RunJasScannersAndSetResults(auditParallelRunner, scanResults, scanResults.GetScaScannedTechnologies(), auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.thirdPartyApplicabilityScan, auditParams.commonGraphScanParams.MultiScanId, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan); err != nil { - return fmt.Errorf("error from thread_id %d: %s", threadId, err.Error()) + if err := runner.AddJasScannersTasks(auditParallelRunner, scanResults, scanResults.GetScaScannedTechnologies(), auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.thirdPartyApplicabilityScan, auditParams.commonGraphScanParams.MultiScanId, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan); err != nil { + return fmt.Errorf("%s failed to run JAS scanners: %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } return nil } diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 8676023a..50e8961a 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -136,7 +136,7 @@ func executeScaScanTask(auditParallelRunner *utils.SecurityParallelRunner, serve // Scan the dependency tree. scanResults, xrayErr := runScaWithTech(scan.Technology, auditParams, serverDetails, *treeResult.FlatTree, treeResult.FullDepTrees) if xrayErr != nil { - return fmt.Errorf("error from thread_id %d: '%s' Xray dependency tree scan request failed:\n%s", threadId, scan.Technology, xrayErr.Error()) + return fmt.Errorf("%s Xray dependency tree scan request on '%s' failed:\n%s", clientutils.GetLogMsgPrefix(threadId, false), scan.Technology, xrayErr.Error()) } scan.IsMultipleRootProject = clientutils.Pointer(len(treeResult.FullDepTrees) > 1) addThirdPartyDependenciesToParams(auditParams, scan.Technology, treeResult.FlatTree, treeResult.FullDepTrees) diff --git a/commands/scan/scan.go b/commands/scan/scan.go index a16bde91..5f419bea 100644 --- a/commands/scan/scan.go +++ b/commands/scan/scan.go @@ -420,7 +420,7 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledFo } workingDirs := []string{filePath} depsList := depsListFromVulnerabilities(*graphScanResults) - err = runner.RunJasScannersAndSetResults(jasFileProducerConsumer, &scanResults, []techutils.Technology{techutils.Technology(graphScanResults.ScannedPackageType)}, &depsList, scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) + err = runner.AddJasScannersTasks(jasFileProducerConsumer, &scanResults, []techutils.Technology{techutils.Technology(graphScanResults.ScannedPackageType)}, &depsList, scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) if err != nil { log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error())) diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index 0c7f6b6a..b5ad72c5 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -17,7 +17,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" ) -func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, technologiesList []techutils.Technology, directDependencies *[]string, +func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, technologiesList []techutils.Technology, directDependencies *[]string, serverDetails *config.ServerDetails, workingDirs []string, thirdPartyApplicabilityScan bool, msi string, scanType applicability.ApplicabilityScanType, secretsScanType secrets.SecretsScanType, errHandlerFunc func(error)) (err error) { if serverDetails == nil || len(serverDetails.Url) == 0 { log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.") @@ -34,12 +34,14 @@ func RunJasScannersAndSetResults(securityParallelRunner *utils.SecurityParallelR } // Set environments variables for analytics in analyzers manager. callback := jas.SetAnalyticsMetricsDataForAnalyzerManager(msi, technologiesList) - defer func() { + go func() { // Wait for all scanners to complete before cleaning up temp dir securityParallelRunner.ScannersWg.Wait() + log.Debug("done scannersWg waiting") callback() cleanup := scanner.ScannerDirCleanupFunc err = errors.Join(err, cleanup()) + log.Debug("done cleanup folder") }() // Don't execute other scanners when scanning third party dependencies. @@ -85,10 +87,11 @@ func runSecretsScan(securityParallelRunner *utils.SecurityParallelRunner, scanne return func(threadId int) (err error) { defer func() { securityParallelRunner.ScannersWg.Done() + log.Debug("done run secrets scan waiting") }() results, err := secrets.RunSecretsScan(scanner, secretsScanType, module, threadId) if err != nil { - return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) + return fmt.Errorf("%s%s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() extendedScanResults.SecretsScanResults = append(extendedScanResults.SecretsScanResults, results...) @@ -102,6 +105,7 @@ func runIacScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *j return func(threadId int) (err error) { defer func() { securityParallelRunner.ScannersWg.Done() + log.Debug("done run iac scan waiting") }() results, err := iac.RunIacScan(scanner, module, threadId) if err != nil { @@ -119,6 +123,7 @@ func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner * return func(threadId int) (err error) { defer func() { securityParallelRunner.ScannersWg.Done() + log.Debug("done run sast scan waiting") }() results, err := sast.RunSastScan(scanner, module, threadId) if err != nil { @@ -136,6 +141,7 @@ func runContextualScan(securityParallelRunner *utils.SecurityParallelRunner, sca return func(threadId int) (err error) { defer func() { securityParallelRunner.ScannersWg.Done() + log.Debug("done run contextual scan waiting") }() results, err := applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) if err != nil { diff --git a/jas/runner/jasrunner_test.go b/jas/runner/jasrunner_test.go index 0d75117e..9e99dd10 100644 --- a/jas/runner/jasrunner_test.go +++ b/jas/runner/jasrunner_test.go @@ -27,7 +27,7 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err = RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + err = AddJasScannersTasks(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) // Expect error: assert.Error(t, err) } @@ -35,7 +35,7 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { securityParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := RunJasScannersAndSetResults(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + err := AddJasScannersTasks(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) assert.NoError(t, err) } From 247552163e629d2806e5bbed1b52b649af3633e0 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Mon, 10 Jun 2024 17:34:57 +0300 Subject: [PATCH 26/59] add more debug logs --- jas/runner/jasrunner.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index b5ad72c5..cfb0c566 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -34,7 +34,7 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s } // Set environments variables for analytics in analyzers manager. callback := jas.SetAnalyticsMetricsDataForAnalyzerManager(msi, technologiesList) - go func() { + defer func() { // Wait for all scanners to complete before cleaning up temp dir securityParallelRunner.ScannersWg.Wait() log.Debug("done scannersWg waiting") @@ -63,6 +63,7 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s // Wait for sca scan to complete before running contextual scan securityParallelRunner.ScaScansWg.Wait() + log.Debug("done waiting for sca") for _, module := range scanner.JFrogAppsConfig.Modules { if err = addModuleJasScanTask(module, utils.Applicability, securityParallelRunner, runContextualScan(securityParallelRunner, scanner, scanResults, module, *directDependencies, thirdPartyApplicabilityScan, scanType), errHandlerFunc); err != nil { return @@ -76,6 +77,7 @@ func addModuleJasScanTask(module jfrogappsconfig.Module, scanType utils.JasScanT return } securityParallelRunner.ScannersWg.Add(1) + log.Debug("add 1 to scannersWg") if _, err = securityParallelRunner.Runner.AddTaskWithError(task, errHandlerFunc); err != nil { err = fmt.Errorf("failed to create %s scan task: %s", scanType, err.Error()) } From 79a97b1a2002fee1a742d6840fd768531cc8615a Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Tue, 11 Jun 2024 09:31:19 +0300 Subject: [PATCH 27/59] create sca tasks first --- commands/audit/audit.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 8cb5b8e5..c00cde32 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -197,8 +197,15 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId - auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) + // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. + auditParallelRunner.ScaScansWg.Add(1) + if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { + return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) + }, auditParallelRunner.AddErrorToChan); scaErr != nil { + auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create sca scan task %s", scaErr.Error())) + } + if results.ExtendedScanResults.EntitledForJas { // Download (if needed) the analyzer manager and run scanners. auditParallelRunner.JasWg.Add(1) @@ -208,14 +215,6 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create AM downloading task, skipping JAS scans...: %s", jasErr.Error())) } } - - // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. - auditParallelRunner.ScaScansWg.Add(1) - if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { - return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) - }, auditParallelRunner.AddErrorToChan); scaErr != nil { - auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create sca scan task %s", scaErr.Error())) - } go func() { auditParallelRunner.JasWg.Wait() auditParallelRunner.ScaScansWg.Wait() From 12a3f2c38fe724611ceb27aacc26209beb362964 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Wed, 12 Jun 2024 10:35:31 +0300 Subject: [PATCH 28/59] remove sca scan as task --- commands/audit/audit.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index c00cde32..2dcdad18 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -198,14 +198,6 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) - // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. - auditParallelRunner.ScaScansWg.Add(1) - if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { - return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) - }, auditParallelRunner.AddErrorToChan); scaErr != nil { - auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create sca scan task %s", scaErr.Error())) - } - if results.ExtendedScanResults.EntitledForJas { // Download (if needed) the analyzer manager and run scanners. auditParallelRunner.JasWg.Add(1) @@ -215,6 +207,17 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create AM downloading task, skipping JAS scans...: %s", jasErr.Error())) } } + + // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. + //auditParallelRunner.ScaScansWg.Add(1) + //if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { + // return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) + //}, auditParallelRunner.AddErrorToChan); scaErr != nil { + // auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create sca scan task %s", scaErr.Error())) + //} + if scaScanErr := buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, 0); scaScanErr != nil { + auditParallelRunner.AddErrorToChan(scaScanErr) + } go func() { auditParallelRunner.JasWg.Wait() auditParallelRunner.ScaScansWg.Wait() From 9866706da4e3e6d9fd5cb93ad21919f31e508fc4 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Wed, 12 Jun 2024 10:42:08 +0300 Subject: [PATCH 29/59] remove sca scan as task 2 --- commands/audit/scarunner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 50e8961a..070ebb11 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -36,9 +36,9 @@ import ( func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner, auditParams *AuditParams, results *xrayutils.Results, threadId int) (err error) { // Prepare - defer func() { - auditParallelRunner.ScaScansWg.Done() - }() + //defer func() { + // auditParallelRunner.ScaScansWg.Done() + //}() currentWorkingDir, err := os.Getwd() if errorutils.CheckError(err) != nil { return From 17cbfbcdd6bde7ba1d7220a2f3d3a65a11b11b96 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Wed, 12 Jun 2024 11:44:16 +0300 Subject: [PATCH 30/59] bring back sca scan as task --- commands/audit/audit.go | 21 +++++++++++---------- commands/audit/scarunner.go | 6 +++--- jas/common.go | 1 + 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 2dcdad18..2dd0329e 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -198,6 +198,14 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) + // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. + auditParallelRunner.ScaScansWg.Add(1) + if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { + return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) + }, auditParallelRunner.AddErrorToChan); scaErr != nil { + auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create sca scan task %s", scaErr.Error())) + } + if results.ExtendedScanResults.EntitledForJas { // Download (if needed) the analyzer manager and run scanners. auditParallelRunner.JasWg.Add(1) @@ -208,16 +216,9 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } } - // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. - //auditParallelRunner.ScaScansWg.Add(1) - //if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { - // return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) - //}, auditParallelRunner.AddErrorToChan); scaErr != nil { - // auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create sca scan task %s", scaErr.Error())) - //} - if scaScanErr := buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, 0); scaScanErr != nil { - auditParallelRunner.AddErrorToChan(scaScanErr) - } + // if scaScanErr := buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, 0); scaScanErr != nil { + // auditParallelRunner.AddErrorToChan(scaScanErr) + // } go func() { auditParallelRunner.JasWg.Wait() auditParallelRunner.ScaScansWg.Wait() diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 070ebb11..39368bef 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -35,10 +35,10 @@ import ( ) func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner, auditParams *AuditParams, results *xrayutils.Results, threadId int) (err error) { + defer func() { + auditParallelRunner.ScaScansWg.Done() + }() // Prepare - //defer func() { - // auditParallelRunner.ScaScansWg.Done() - //}() currentWorkingDir, err := os.Getwd() if errorutils.CheckError(err) != nil { return diff --git a/jas/common.go b/jas/common.go index 4433f882..b2ce5695 100644 --- a/jas/common.go +++ b/jas/common.go @@ -65,6 +65,7 @@ func NewJasScanner(workingDirs []string, serverDetails *config.ServerDetails) (s return } scanner.TempDir = tempDir + log.Debug("created tempdir") scanner.ScannerDirCleanupFunc = func() error { return fileutils.RemoveTempDir(tempDir) } From d31233811a578a4fb66655a6400b343ea0de5f67 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 13 Jun 2024 11:30:44 +0300 Subject: [PATCH 31/59] create jfrogAppsConfig at beginning of audit --- commands/audit/audit.go | 12 +++++++++--- commands/scan/scan.go | 8 ++++++-- jas/common.go | 8 +++++--- jas/runner/jasrunner.go | 4 ++-- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 2dd0329e..76308889 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -3,6 +3,7 @@ package audit import ( "errors" "fmt" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/jas/applicability" "github.com/jfrog/jfrog-cli-security/jas/runner" "github.com/jfrog/jfrog-cli-security/jas/secrets" @@ -198,6 +199,11 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) + jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) + if err != nil { + return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) + } + // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. auditParallelRunner.ScaScansWg.Add(1) if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { @@ -210,7 +216,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) // Download (if needed) the analyzer manager and run scanners. auditParallelRunner.JasWg.Add(1) if _, jasErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { - return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, serverDetails, auditParams, threadId) + return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, serverDetails, auditParams, jfrogAppsConfig, threadId) }, auditParallelRunner.AddErrorToChan); jasErr != nil { auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create AM downloading task, skipping JAS scans...: %s", jasErr.Error())) } @@ -237,14 +243,14 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, - serverDetails *config.ServerDetails, auditParams *AuditParams, threadId int) error { + serverDetails *config.ServerDetails, auditParams *AuditParams, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, threadId int) error { defer func() { auditParallelRunner.JasWg.Done() }() if err := xrayutils.DownloadAnalyzerManagerIfNeeded(threadId); err != nil { return fmt.Errorf("%s failed to download analyzer manager: %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } - if err := runner.AddJasScannersTasks(auditParallelRunner, scanResults, scanResults.GetScaScannedTechnologies(), auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.thirdPartyApplicabilityScan, auditParams.commonGraphScanParams.MultiScanId, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan); err != nil { + if err := runner.AddJasScannersTasks(auditParallelRunner, scanResults, scanResults.GetScaScannedTechnologies(), auditParams.DirectDependencies(), serverDetails, jfrogAppsConfig, auditParams.thirdPartyApplicabilityScan, auditParams.commonGraphScanParams.MultiScanId, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan); err != nil { return fmt.Errorf("%s failed to run JAS scanners: %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } return nil diff --git a/commands/scan/scan.go b/commands/scan/scan.go index 5f419bea..a0f9607c 100644 --- a/commands/scan/scan.go +++ b/commands/scan/scan.go @@ -420,8 +420,12 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledFo } workingDirs := []string{filePath} depsList := depsListFromVulnerabilities(*graphScanResults) - err = runner.AddJasScannersTasks(jasFileProducerConsumer, &scanResults, []techutils.Technology{techutils.Technology(graphScanResults.ScannedPackageType)}, &depsList, scanCmd.serverDetails, workingDirs, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) - + jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(workingDirs) + if err != nil { + log.Error(fmt.Sprintf("failed to create JFrogAppsConfig: %s", err.Error())) + indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()}) + } + err = runner.AddJasScannersTasks(jasFileProducerConsumer, &scanResults, []techutils.Technology{techutils.Technology(graphScanResults.ScannedPackageType)}, &depsList, scanCmd.serverDetails, jfrogAppsConfig, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) if err != nil { log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error())) indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()}) diff --git a/jas/common.go b/jas/common.go index b2ce5695..aaae76f4 100644 --- a/jas/common.go +++ b/jas/common.go @@ -55,7 +55,7 @@ type JasScanner struct { ScannerDirCleanupFunc func() error } -func NewJasScanner(workingDirs []string, serverDetails *config.ServerDetails) (scanner *JasScanner, err error) { +func NewJasScanner(jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, serverDetails *config.ServerDetails) (scanner *JasScanner, err error) { scanner = &JasScanner{} if scanner.AnalyzerManager.AnalyzerManagerFullPath, err = utils.GetAnalyzerManagerExecutable(); err != nil { return @@ -70,7 +70,7 @@ func NewJasScanner(workingDirs []string, serverDetails *config.ServerDetails) (s return fileutils.RemoveTempDir(tempDir) } scanner.ServerDetails = serverDetails - scanner.JFrogAppsConfig, err = CreateJFrogAppsConfig(workingDirs) + scanner.JFrogAppsConfig = jfrogAppsConfig return } @@ -217,7 +217,9 @@ var FakeBasicXrayResults = []services.ScanResponse{ func InitJasTest(t *testing.T, workingDirs ...string) (*JasScanner, func()) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) - scanner, err := NewJasScanner(workingDirs, &FakeServerDetails) + jfrogAppsConfigForTest, err := CreateJFrogAppsConfig(workingDirs) + assert.NoError(t, err) + scanner, err := NewJasScanner(jfrogAppsConfigForTest, &FakeServerDetails) assert.NoError(t, err) return scanner, func() { assert.NoError(t, scanner.ScannerDirCleanupFunc()) diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index cfb0c566..a13925c6 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -18,7 +18,7 @@ import ( ) func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, technologiesList []techutils.Technology, directDependencies *[]string, - serverDetails *config.ServerDetails, workingDirs []string, thirdPartyApplicabilityScan bool, msi string, scanType applicability.ApplicabilityScanType, secretsScanType secrets.SecretsScanType, errHandlerFunc func(error)) (err error) { + serverDetails *config.ServerDetails, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, thirdPartyApplicabilityScan bool, msi string, scanType applicability.ApplicabilityScanType, secretsScanType secrets.SecretsScanType, errHandlerFunc func(error)) (err error) { if serverDetails == nil || len(serverDetails.Url) == 0 { log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.") return @@ -28,7 +28,7 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s if scanType == applicability.ApplicabilityScannerType || secretsScanType == secrets.SecretsScannerType { runAllScanners = true } - scanner, err := jas.NewJasScanner(workingDirs, serverDetails) + scanner, err := jas.NewJasScanner(jfrogAppsConfig, serverDetails) if err != nil { return } From b3cb67de0628dae4731cadf9365b5cfd8059f577 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 13 Jun 2024 12:23:13 +0300 Subject: [PATCH 32/59] adding debugs logs --- commands/audit/audit.go | 2 ++ jas/runner/jasrunner.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 76308889..23515bb8 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -200,6 +200,8 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) + log.Debug("the num of modules created: " + string(rune(len(jfrogAppsConfig.Modules)))) + log.Debug("the name of the first module: " + jfrogAppsConfig.Modules[0].Name) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) } diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index a13925c6..c2509afa 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -47,10 +47,12 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s // Don't execute other scanners when scanning third party dependencies. if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { + log.Debug("the module for secrets when creating task: " + module.Name) if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module, secretsScanType), errHandlerFunc); err != nil { return } if runAllScanners { + log.Debug("the module for iac when creating task: " + module.Name) if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { return } From b02eb22f4a5f685115eb791645c5c1e2fa84c9a7 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 13 Jun 2024 13:05:34 +0300 Subject: [PATCH 33/59] adding debugs logs 2 --- commands/audit/audit.go | 4 ++-- jas/runner/jasrunner.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 23515bb8..73b12c5f 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -200,8 +200,8 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) - log.Debug("the num of modules created: " + string(rune(len(jfrogAppsConfig.Modules)))) - log.Debug("the name of the first module: " + jfrogAppsConfig.Modules[0].Name) + log.Debug(fmt.Sprintf("the num of modules created: %s", string(rune(len(jfrogAppsConfig.Modules))))) + log.Debug(fmt.Sprintf("the name of the first module: %s", jfrogAppsConfig.Modules[0].Name)) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) } diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index c2509afa..e80daa30 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -47,12 +47,12 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s // Don't execute other scanners when scanning third party dependencies. if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { - log.Debug("the module for secrets when creating task: " + module.Name) + log.Debug(fmt.Sprintf("the module for secrets when creating task: %s", module.Name)) if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module, secretsScanType), errHandlerFunc); err != nil { return } if runAllScanners { - log.Debug("the module for iac when creating task: " + module.Name) + log.Debug(fmt.Sprintf("the module for secrets when creating task: %s", module.Name)) if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { return } From e0b19f77ae620948463345d4ab0495d2d09bf55d Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 13 Jun 2024 13:43:59 +0300 Subject: [PATCH 34/59] more logs --- commands/audit/audit.go | 3 ++- commands/audit/scarunner.go | 4 ++++ jas/common.go | 2 ++ jas/runner/jasrunner.go | 4 ++-- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 73b12c5f..409f8415 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -200,8 +200,9 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) - log.Debug(fmt.Sprintf("the num of modules created: %s", string(rune(len(jfrogAppsConfig.Modules))))) + log.Debug(fmt.Sprintf("the num of modules created: %d", len(jfrogAppsConfig.Modules))) log.Debug(fmt.Sprintf("the name of the first module: %s", jfrogAppsConfig.Modules[0].Name)) + log.Debug(fmt.Sprintf("the source root of the first module: %s", jfrogAppsConfig.Modules[0].SourceRoot)) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) } diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 39368bef..d8ffb405 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -405,9 +405,13 @@ func logDeps(uniqueDeps any) (err error) { // This method will change the working directory to the scan's working directory. func buildDependencyTree(scan *utils.ScaScanResult, params *AuditParams) (*DependencyTreeResult, error) { + cuurentDir, _ := os.Getwd() + log.Debug(fmt.Sprintf("current dir is: %s", cuurentDir)) if err := os.Chdir(scan.Target); err != nil { return nil, errorutils.CheckError(err) } + cuurentDir, _ = os.Getwd() + log.Debug(fmt.Sprintf("current dir is: %s", cuurentDir)) treeResult, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology) if techErr != nil { return nil, fmt.Errorf("failed while building '%s' dependency tree:\n%s", scan.Technology, techErr.Error()) diff --git a/jas/common.go b/jas/common.go index aaae76f4..87eccda4 100644 --- a/jas/common.go +++ b/jas/common.go @@ -241,6 +241,8 @@ func ShouldSkipScanner(module jfrogappsconfig.Module, scanType utils.JasScanType func GetSourceRoots(module jfrogappsconfig.Module, scanner *jfrogappsconfig.Scanner) ([]string, error) { root, err := filepath.Abs(module.SourceRoot) + cuurentDir, _ := os.Getwd() + log.Debug(fmt.Sprintf("current dir is: %s", cuurentDir)) if err != nil { return []string{}, errorutils.CheckError(err) } diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index e80daa30..20646258 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -47,12 +47,12 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s // Don't execute other scanners when scanning third party dependencies. if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { - log.Debug(fmt.Sprintf("the module for secrets when creating task: %s", module.Name)) + log.Debug(fmt.Sprintf("the source root of the first module: %s", jfrogAppsConfig.Modules[0].SourceRoot)) if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module, secretsScanType), errHandlerFunc); err != nil { return } if runAllScanners { - log.Debug(fmt.Sprintf("the module for secrets when creating task: %s", module.Name)) + log.Debug(fmt.Sprintf("the source root of the first module: %s", jfrogAppsConfig.Modules[0].SourceRoot)) if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { return } From 59ce3b7e8c360072fe6d5b4acac2c1effeb97e53 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 13 Jun 2024 13:50:35 +0300 Subject: [PATCH 35/59] more logs --- commands/audit/scarunner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index d8ffb405..961e19c3 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -406,12 +406,12 @@ func logDeps(uniqueDeps any) (err error) { // This method will change the working directory to the scan's working directory. func buildDependencyTree(scan *utils.ScaScanResult, params *AuditParams) (*DependencyTreeResult, error) { cuurentDir, _ := os.Getwd() - log.Debug(fmt.Sprintf("current dir is: %s", cuurentDir)) + log.Debug(fmt.Sprintf("current dir before changing to scan target is: %s", cuurentDir)) if err := os.Chdir(scan.Target); err != nil { return nil, errorutils.CheckError(err) } cuurentDir, _ = os.Getwd() - log.Debug(fmt.Sprintf("current dir is: %s", cuurentDir)) + log.Debug(fmt.Sprintf("current dir after changing to scan target is: %s", cuurentDir)) treeResult, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology) if techErr != nil { return nil, fmt.Errorf("failed while building '%s' dependency tree:\n%s", scan.Technology, techErr.Error()) From 23f003d01f735e066ae7efd549087980ef4a8d7e Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 13 Jun 2024 14:40:37 +0300 Subject: [PATCH 36/59] more logs with current dir --- jas/runner/jasrunner.go | 7 +++++++ jas/secrets/secretsscanner.go | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index 20646258..af9baed3 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -15,6 +15,7 @@ import ( "github.com/jfrog/jfrog-cli-security/utils/techutils" clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" + "os" ) func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, technologiesList []techutils.Technology, directDependencies *[]string, @@ -48,11 +49,15 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { log.Debug(fmt.Sprintf("the source root of the first module: %s", jfrogAppsConfig.Modules[0].SourceRoot)) + cuurentDir, _ := os.Getwd() + log.Debug(fmt.Sprintf("current dir is (secrets): %s", cuurentDir)) if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module, secretsScanType), errHandlerFunc); err != nil { return } if runAllScanners { log.Debug(fmt.Sprintf("the source root of the first module: %s", jfrogAppsConfig.Modules[0].SourceRoot)) + cuurentDir, _ := os.Getwd() + log.Debug(fmt.Sprintf("current dir is (iac): %s", cuurentDir)) if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { return } @@ -93,6 +98,8 @@ func runSecretsScan(securityParallelRunner *utils.SecurityParallelRunner, scanne securityParallelRunner.ScannersWg.Done() log.Debug("done run secrets scan waiting") }() + cuurentDir, _ := os.Getwd() + log.Debug(fmt.Sprintf("current dir is: %s", cuurentDir)) results, err := secrets.RunSecretsScan(scanner, secretsScanType, module, threadId) if err != nil { return fmt.Errorf("%s%s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) diff --git a/jas/secrets/secretsscanner.go b/jas/secrets/secretsscanner.go index 43ab4d6e..6111a38c 100644 --- a/jas/secrets/secretsscanner.go +++ b/jas/secrets/secretsscanner.go @@ -1,7 +1,9 @@ package secrets import ( + "fmt" clientutils "github.com/jfrog/jfrog-client-go/utils" + "os" "path/filepath" "strings" @@ -43,6 +45,8 @@ func RunSecretsScan(scanner *jas.JasScanner, scanType SecretsScanType, module jf return } secretScanManager := newSecretsScanManager(scanner, scanType, scannerTempDir) + cuurentDir, _ := os.Getwd() + log.Debug(fmt.Sprintf("current dir is: %s", cuurentDir)) log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running secrets scan...") if err = secretScanManager.scanner.Run(secretScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Secrets, err) From 749cfc6dbbfe844975431ffec3101e39efe9ac18 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 13 Jun 2024 14:42:54 +0300 Subject: [PATCH 37/59] more logs with current dir 2 --- jas/common.go | 2 +- jas/secrets/secretsscanner.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/jas/common.go b/jas/common.go index 87eccda4..c8d16ed0 100644 --- a/jas/common.go +++ b/jas/common.go @@ -242,7 +242,7 @@ func ShouldSkipScanner(module jfrogappsconfig.Module, scanType utils.JasScanType func GetSourceRoots(module jfrogappsconfig.Module, scanner *jfrogappsconfig.Scanner) ([]string, error) { root, err := filepath.Abs(module.SourceRoot) cuurentDir, _ := os.Getwd() - log.Debug(fmt.Sprintf("current dir is: %s", cuurentDir)) + log.Debug(fmt.Sprintf("current dir in GetSourceRoots: %s", cuurentDir)) if err != nil { return []string{}, errorutils.CheckError(err) } diff --git a/jas/secrets/secretsscanner.go b/jas/secrets/secretsscanner.go index 6111a38c..8195b6b4 100644 --- a/jas/secrets/secretsscanner.go +++ b/jas/secrets/secretsscanner.go @@ -96,6 +96,8 @@ type secretsScanConfiguration struct { } func (s *SecretScanManager) createConfigFile(module jfrogappsconfig.Module) error { + cuurentDir, _ := os.Getwd() + log.Debug(fmt.Sprintf("current dir while creating config file: %s", cuurentDir)) roots, err := jas.GetSourceRoots(module, module.Scanners.Secrets) if err != nil { return err From 6da2edd8ba6d65ddaf305f27862f8228054b7a8e Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 13 Jun 2024 15:08:09 +0300 Subject: [PATCH 38/59] more logs with current dir 2 --- commands/audit/scarunner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 961e19c3..0b5c06a7 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -61,6 +61,7 @@ func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner defer func() { // Make sure to return to the original working directory, building the dependency tree may change it + log.Debug(fmt.Sprintf("turning back to original wk: %s", currentWorkingDir)) err = errors.Join(err, os.Chdir(currentWorkingDir)) }() for _, scan := range scans { From 81e8908cb4b672342ddc00c1399c6cef211f7b34 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 13 Jun 2024 15:13:45 +0300 Subject: [PATCH 39/59] add wait group before running scanners --- commands/audit/audit.go | 1 + commands/audit/scarunner.go | 15 +++++++-------- jas/runner/jasrunner.go | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 409f8415..069e8b6f 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -209,6 +209,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. auditParallelRunner.ScaScansWg.Add(1) + auditParallelRunner.ScannersWg.Add(1) if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) }, auditParallelRunner.AddErrorToChan); scaErr != nil { diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 0b5c06a7..e4d57f1f 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -35,14 +35,18 @@ import ( ) func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner, auditParams *AuditParams, results *xrayutils.Results, threadId int) (err error) { - defer func() { - auditParallelRunner.ScaScansWg.Done() - }() // Prepare currentWorkingDir, err := os.Getwd() if errorutils.CheckError(err) != nil { return } + defer func() { + auditParallelRunner.ScaScansWg.Done() + auditParallelRunner.ScannersWg.Done() + // Make sure to return to the original working directory, buildDependencyTree may change it + log.Debug(fmt.Sprintf("turning back to original wk: %s", currentWorkingDir)) + err = errors.Join(err, os.Chdir(currentWorkingDir)) + }() serverDetails, err := auditParams.ServerDetails() if err != nil { return @@ -59,11 +63,6 @@ func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner } log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Preforming", len(scans), "SCA scans:\n", scanInfo) - defer func() { - // Make sure to return to the original working directory, building the dependency tree may change it - log.Debug(fmt.Sprintf("turning back to original wk: %s", currentWorkingDir)) - err = errors.Join(err, os.Chdir(currentWorkingDir)) - }() for _, scan := range scans { // Get the dependency tree for the technology in the working directory. treeResult, bdtErr := buildDependencyTree(scan, auditParams) diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index af9baed3..53505baa 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -45,6 +45,8 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s log.Debug("done cleanup folder") }() + // wait for the building of the dependency tree to finish, so the current dir would the root project + securityParallelRunner.ScannersWg.Wait() // Don't execute other scanners when scanning third party dependencies. if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { From c5b3cca814a391bb6210e54ac6988966c09ac1b9 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Thu, 13 Jun 2024 17:32:22 +0300 Subject: [PATCH 40/59] remove logs and move jas scanner location --- audit_test.go | 15 +++++++---- commands/audit/audit.go | 32 +++++++++++----------- commands/audit/scarunner.go | 9 ++----- commands/scan/scan.go | 8 +++++- jas/common.go | 16 +++++------ jas/runner/jasrunner.go | 50 +++++++++-------------------------- jas/runner/jasrunner_test.go | 13 ++++++--- jas/secrets/secretsscanner.go | 6 ----- utils/parallel_runner.go | 12 ++++----- 9 files changed, 71 insertions(+), 90 deletions(-) diff --git a/audit_test.go b/audit_test.go index 4cd5b54d..313c68a8 100644 --- a/audit_test.go +++ b/audit_test.go @@ -435,22 +435,27 @@ func addDummyPackageDescriptor(t *testing.T, hasPackageJson bool) { // JAS func TestXrayAuditJasSimpleJson(t *testing.T) { - output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("jas", "jas")) + output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("jas", "jas"), "3") + securityTestUtils.VerifySimpleJsonJasResults(t, output, 1, 9, 7, 3, 0, 2, 2) +} + +func TestXrayAuditJasSimpleJsonWithOneThread(t *testing.T) { + output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("jas", "jas"), "1") securityTestUtils.VerifySimpleJsonJasResults(t, output, 1, 9, 7, 3, 0, 2, 2) } func TestXrayAuditJasSimpleJsonWithConfig(t *testing.T) { - output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("jas", "jas-config")) + output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("jas", "jas-config"), "3") securityTestUtils.VerifySimpleJsonJasResults(t, output, 0, 0, 1, 3, 0, 2, 2) } func TestXrayAuditJasNoViolationsSimpleJson(t *testing.T) { - output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("package-managers", "npm", "npm")) + output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("package-managers", "npm", "npm"), "3") securityTestUtils.VerifySimpleJsonScanResults(t, output, 0, 1, 0) securityTestUtils.VerifySimpleJsonJasResults(t, output, 0, 0, 0, 0, 0, 0, 1) } -func testXrayAuditJas(t *testing.T, format string, project string) string { +func testXrayAuditJas(t *testing.T, format string, project string, threads string) string { securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion) tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t) defer createTempDirCallback() @@ -464,7 +469,7 @@ func testXrayAuditJas(t *testing.T, format string, project string) string { assert.NoError(t, err) chdirCallback := clientTests.ChangeDirWithCallback(t, baseWd, tempDirPath) defer chdirCallback() - return securityTests.PlatformCli.WithoutCredentials().RunCliCmdWithOutput(t, "audit", "--format="+format) + return securityTests.PlatformCli.WithoutCredentials().RunCliCmdWithOutput(t, "audit", "--format="+format, "--threads="+threads) } func TestXrayAuditDetectTech(t *testing.T) { diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 069e8b6f..db2ab72a 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -196,42 +196,40 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) if err != nil { return } - results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId + auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) - log.Debug(fmt.Sprintf("the num of modules created: %d", len(jfrogAppsConfig.Modules))) - log.Debug(fmt.Sprintf("the name of the first module: %s", jfrogAppsConfig.Modules[0].Name)) - log.Debug(fmt.Sprintf("the source root of the first module: %s", jfrogAppsConfig.Modules[0].SourceRoot)) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) } - // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. auditParallelRunner.ScaScansWg.Add(1) - auditParallelRunner.ScannersWg.Add(1) + auditParallelRunner.JasScannersWg.Add(1) if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) }, auditParallelRunner.AddErrorToChan); scaErr != nil { auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create sca scan task %s", scaErr.Error())) } + jasScanner := &jas.JasScanner{} if results.ExtendedScanResults.EntitledForJas { // Download (if needed) the analyzer manager and run scanners. auditParallelRunner.JasWg.Add(1) if _, jasErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { - return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, serverDetails, auditParams, jfrogAppsConfig, threadId) + return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, serverDetails, auditParams, jasScanner, jfrogAppsConfig, threadId) }, auditParallelRunner.AddErrorToChan); jasErr != nil { auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create AM downloading task, skipping JAS scans...: %s", jasErr.Error())) } } - // if scaScanErr := buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, 0); scaScanErr != nil { - // auditParallelRunner.AddErrorToChan(scaScanErr) - // } go func() { - auditParallelRunner.JasWg.Wait() auditParallelRunner.ScaScansWg.Wait() + auditParallelRunner.JasWg.Wait() + // Wait for all jas scanners to complete before cleaning up scanners temp dir + auditParallelRunner.JasScannersWg.Wait() + cleanup := jasScanner.ScannerDirCleanupFunc + auditParallelRunner.AddErrorToChan(cleanup()) auditParallelRunner.Runner.Done() }() go func() { @@ -247,15 +245,19 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, - serverDetails *config.ServerDetails, auditParams *AuditParams, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, threadId int) error { + serverDetails *config.ServerDetails, auditParams *AuditParams, scanner *jas.JasScanner, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, threadId int) (err error) { defer func() { auditParallelRunner.JasWg.Done() }() - if err := xrayutils.DownloadAnalyzerManagerIfNeeded(threadId); err != nil { + if err = xrayutils.DownloadAnalyzerManagerIfNeeded(threadId); err != nil { return fmt.Errorf("%s failed to download analyzer manager: %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } - if err := runner.AddJasScannersTasks(auditParallelRunner, scanResults, scanResults.GetScaScannedTechnologies(), auditParams.DirectDependencies(), serverDetails, jfrogAppsConfig, auditParams.thirdPartyApplicabilityScan, auditParams.commonGraphScanParams.MultiScanId, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan); err != nil { + scanner, err = jas.CreateJasScanner(scanner, jfrogAppsConfig, serverDetails) + if err != nil { + return fmt.Errorf("failed to create jas scanner: %s", err.Error()) + } + if err = runner.AddJasScannersTasks(auditParallelRunner, scanResults, scanResults.GetScaScannedTechnologies(), auditParams.DirectDependencies(), serverDetails, auditParams.thirdPartyApplicabilityScan, auditParams.commonGraphScanParams.MultiScanId, scanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan); err != nil { return fmt.Errorf("%s failed to run JAS scanners: %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } - return nil + return } diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index e4d57f1f..f6b795e9 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -41,11 +41,10 @@ func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner return } defer func() { - auditParallelRunner.ScaScansWg.Done() - auditParallelRunner.ScannersWg.Done() // Make sure to return to the original working directory, buildDependencyTree may change it - log.Debug(fmt.Sprintf("turning back to original wk: %s", currentWorkingDir)) err = errors.Join(err, os.Chdir(currentWorkingDir)) + auditParallelRunner.ScaScansWg.Done() + auditParallelRunner.JasScannersWg.Done() }() serverDetails, err := auditParams.ServerDetails() if err != nil { @@ -405,13 +404,9 @@ func logDeps(uniqueDeps any) (err error) { // This method will change the working directory to the scan's working directory. func buildDependencyTree(scan *utils.ScaScanResult, params *AuditParams) (*DependencyTreeResult, error) { - cuurentDir, _ := os.Getwd() - log.Debug(fmt.Sprintf("current dir before changing to scan target is: %s", cuurentDir)) if err := os.Chdir(scan.Target); err != nil { return nil, errorutils.CheckError(err) } - cuurentDir, _ = os.Getwd() - log.Debug(fmt.Sprintf("current dir after changing to scan target is: %s", cuurentDir)) treeResult, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology) if techErr != nil { return nil, fmt.Errorf("failed while building '%s' dependency tree:\n%s", scan.Technology, techErr.Error()) diff --git a/commands/scan/scan.go b/commands/scan/scan.go index a0f9607c..f5426251 100644 --- a/commands/scan/scan.go +++ b/commands/scan/scan.go @@ -425,7 +425,13 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledFo log.Error(fmt.Sprintf("failed to create JFrogAppsConfig: %s", err.Error())) indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()}) } - err = runner.AddJasScannersTasks(jasFileProducerConsumer, &scanResults, []techutils.Technology{techutils.Technology(graphScanResults.ScannedPackageType)}, &depsList, scanCmd.serverDetails, jfrogAppsConfig, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) + scanner := &jas.JasScanner{} + scanner, err = jas.CreateJasScanner(scanner, jfrogAppsConfig, scanCmd.serverDetails) + if err != nil { + log.Error(fmt.Sprintf("failed to create jas scanner: %s", err.Error())) + indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()}) + } + err = runner.AddJasScannersTasks(jasFileProducerConsumer, &scanResults, []techutils.Technology{techutils.Technology(graphScanResults.ScannedPackageType)}, &depsList, scanCmd.serverDetails, false, "", scanner, applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc) if err != nil { log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error())) indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()}) diff --git a/jas/common.go b/jas/common.go index c8d16ed0..dac0497a 100644 --- a/jas/common.go +++ b/jas/common.go @@ -55,23 +55,22 @@ type JasScanner struct { ScannerDirCleanupFunc func() error } -func NewJasScanner(jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, serverDetails *config.ServerDetails) (scanner *JasScanner, err error) { - scanner = &JasScanner{} +func CreateJasScanner(scanner *JasScanner, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, serverDetails *config.ServerDetails) (*JasScanner, error) { + var err error if scanner.AnalyzerManager.AnalyzerManagerFullPath, err = utils.GetAnalyzerManagerExecutable(); err != nil { - return + return scanner, err } var tempDir string if tempDir, err = fileutils.CreateTempDir(); err != nil { - return + return scanner, err } scanner.TempDir = tempDir - log.Debug("created tempdir") scanner.ScannerDirCleanupFunc = func() error { return fileutils.RemoveTempDir(tempDir) } scanner.ServerDetails = serverDetails scanner.JFrogAppsConfig = jfrogAppsConfig - return + return scanner, err } func CreateJFrogAppsConfig(workingDirs []string) (*jfrogappsconfig.JFrogAppsConfig, error) { @@ -219,7 +218,8 @@ func InitJasTest(t *testing.T, workingDirs ...string) (*JasScanner, func()) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) jfrogAppsConfigForTest, err := CreateJFrogAppsConfig(workingDirs) assert.NoError(t, err) - scanner, err := NewJasScanner(jfrogAppsConfigForTest, &FakeServerDetails) + scanner := &JasScanner{} + scanner, err = CreateJasScanner(scanner, jfrogAppsConfigForTest, &FakeServerDetails) assert.NoError(t, err) return scanner, func() { assert.NoError(t, scanner.ScannerDirCleanupFunc()) @@ -241,8 +241,6 @@ func ShouldSkipScanner(module jfrogappsconfig.Module, scanType utils.JasScanType func GetSourceRoots(module jfrogappsconfig.Module, scanner *jfrogappsconfig.Scanner) ([]string, error) { root, err := filepath.Abs(module.SourceRoot) - cuurentDir, _ := os.Getwd() - log.Debug(fmt.Sprintf("current dir in GetSourceRoots: %s", cuurentDir)) if err != nil { return []string{}, errorutils.CheckError(err) } diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index 53505baa..eaa28ddf 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -1,7 +1,6 @@ package runner import ( - "errors" "fmt" "github.com/jfrog/gofrog/parallel" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" @@ -15,11 +14,10 @@ import ( "github.com/jfrog/jfrog-cli-security/utils/techutils" clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" - "os" ) func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, technologiesList []techutils.Technology, directDependencies *[]string, - serverDetails *config.ServerDetails, jfrogAppsConfig *jfrogappsconfig.JFrogAppsConfig, thirdPartyApplicabilityScan bool, msi string, scanType applicability.ApplicabilityScanType, secretsScanType secrets.SecretsScanType, errHandlerFunc func(error)) (err error) { + serverDetails *config.ServerDetails, thirdPartyApplicabilityScan bool, msi string, scanner *jas.JasScanner, scanType applicability.ApplicabilityScanType, secretsScanType secrets.SecretsScanType, errHandlerFunc func(error)) (err error) { if serverDetails == nil || len(serverDetails.Url) == 0 { log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.") return @@ -29,37 +27,21 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s if scanType == applicability.ApplicabilityScannerType || secretsScanType == secrets.SecretsScannerType { runAllScanners = true } - scanner, err := jas.NewJasScanner(jfrogAppsConfig, serverDetails) - if err != nil { - return - } // Set environments variables for analytics in analyzers manager. callback := jas.SetAnalyticsMetricsDataForAnalyzerManager(msi, technologiesList) defer func() { - // Wait for all scanners to complete before cleaning up temp dir - securityParallelRunner.ScannersWg.Wait() - log.Debug("done scannersWg waiting") callback() - cleanup := scanner.ScannerDirCleanupFunc - err = errors.Join(err, cleanup()) - log.Debug("done cleanup folder") }() // wait for the building of the dependency tree to finish, so the current dir would the root project - securityParallelRunner.ScannersWg.Wait() + securityParallelRunner.JasScannersWg.Wait() // Don't execute other scanners when scanning third party dependencies. if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { - log.Debug(fmt.Sprintf("the source root of the first module: %s", jfrogAppsConfig.Modules[0].SourceRoot)) - cuurentDir, _ := os.Getwd() - log.Debug(fmt.Sprintf("current dir is (secrets): %s", cuurentDir)) if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module, secretsScanType), errHandlerFunc); err != nil { return } if runAllScanners { - log.Debug(fmt.Sprintf("the source root of the first module: %s", jfrogAppsConfig.Modules[0].SourceRoot)) - cuurentDir, _ := os.Getwd() - log.Debug(fmt.Sprintf("current dir is (iac): %s", cuurentDir)) if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { return } @@ -70,11 +52,8 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s } } - // Wait for sca scan to complete before running contextual scan - securityParallelRunner.ScaScansWg.Wait() - log.Debug("done waiting for sca") for _, module := range scanner.JFrogAppsConfig.Modules { - if err = addModuleJasScanTask(module, utils.Applicability, securityParallelRunner, runContextualScan(securityParallelRunner, scanner, scanResults, module, *directDependencies, thirdPartyApplicabilityScan, scanType), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.Applicability, securityParallelRunner, runContextualScan(securityParallelRunner, scanner, scanResults, module, directDependencies, thirdPartyApplicabilityScan, scanType), errHandlerFunc); err != nil { return } } @@ -85,8 +64,7 @@ func addModuleJasScanTask(module jfrogappsconfig.Module, scanType utils.JasScanT if jas.ShouldSkipScanner(module, scanType) { return } - securityParallelRunner.ScannersWg.Add(1) - log.Debug("add 1 to scannersWg") + securityParallelRunner.JasScannersWg.Add(1) if _, err = securityParallelRunner.Runner.AddTaskWithError(task, errHandlerFunc); err != nil { err = fmt.Errorf("failed to create %s scan task: %s", scanType, err.Error()) } @@ -97,11 +75,8 @@ func runSecretsScan(securityParallelRunner *utils.SecurityParallelRunner, scanne module jfrogappsconfig.Module, secretsScanType secrets.SecretsScanType) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { - securityParallelRunner.ScannersWg.Done() - log.Debug("done run secrets scan waiting") + securityParallelRunner.JasScannersWg.Done() }() - cuurentDir, _ := os.Getwd() - log.Debug(fmt.Sprintf("current dir is: %s", cuurentDir)) results, err := secrets.RunSecretsScan(scanner, secretsScanType, module, threadId) if err != nil { return fmt.Errorf("%s%s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) @@ -117,8 +92,7 @@ func runIacScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *j module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { - securityParallelRunner.ScannersWg.Done() - log.Debug("done run iac scan waiting") + securityParallelRunner.JasScannersWg.Done() }() results, err := iac.RunIacScan(scanner, module, threadId) if err != nil { @@ -135,8 +109,7 @@ func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner * module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { - securityParallelRunner.ScannersWg.Done() - log.Debug("done run sast scan waiting") + securityParallelRunner.JasScannersWg.Done() }() results, err := sast.RunSastScan(scanner, module, threadId) if err != nil { @@ -150,13 +123,14 @@ func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner * } func runContextualScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, - module jfrogappsconfig.Module, directDependencies []string, thirdPartyApplicabilityScan bool, scanType applicability.ApplicabilityScanType) parallel.TaskFunc { + module jfrogappsconfig.Module, directDependencies *[]string, thirdPartyApplicabilityScan bool, scanType applicability.ApplicabilityScanType) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { - securityParallelRunner.ScannersWg.Done() - log.Debug("done run contextual scan waiting") + securityParallelRunner.JasScannersWg.Done() }() - results, err := applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) + // Wait for sca scan to complete before running contextual scan + securityParallelRunner.ScaScansWg.Wait() + results, err := applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), *directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) if err != nil { return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } diff --git a/jas/runner/jasrunner_test.go b/jas/runner/jasrunner_test.go index 9e99dd10..279f1102 100644 --- a/jas/runner/jasrunner_test.go +++ b/jas/runner/jasrunner_test.go @@ -26,16 +26,22 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { defer func() { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() + scanner := &jas.JasScanner{} + jasScanner, err := jas.CreateJasScanner(scanner, nil, &jas.FakeServerDetails) + assert.NoError(t, err) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err = AddJasScannersTasks(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + err = AddJasScannersTasks(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, false, "", jasScanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) // Expect error: assert.Error(t, err) } func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { securityParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) + scanner := &jas.JasScanner{} + jasScanner, err := jas.CreateJasScanner(scanner, nil, &jas.FakeServerDetails) + assert.NoError(t, err) scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := AddJasScannersTasks(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, false, "", applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) + err = AddJasScannersTasks(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, false, "", jasScanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) assert.NoError(t, err) } @@ -43,7 +49,8 @@ func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) jfrogAppsConfigForTest, _ := jas.CreateJFrogAppsConfig(nil) - scanner, _ := jas.NewJasScanner(nil, &jas.FakeServerDetails) + scanner := &jas.JasScanner{} + scanner, _ = jas.CreateJasScanner(scanner, nil, &jas.FakeServerDetails) _, err := applicability.RunApplicabilityScan(jas.FakeBasicXrayResults, []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, scanner, false, applicability.ApplicabilityScannerType, jfrogAppsConfigForTest.Modules[0], 0) diff --git a/jas/secrets/secretsscanner.go b/jas/secrets/secretsscanner.go index 8195b6b4..43ab4d6e 100644 --- a/jas/secrets/secretsscanner.go +++ b/jas/secrets/secretsscanner.go @@ -1,9 +1,7 @@ package secrets import ( - "fmt" clientutils "github.com/jfrog/jfrog-client-go/utils" - "os" "path/filepath" "strings" @@ -45,8 +43,6 @@ func RunSecretsScan(scanner *jas.JasScanner, scanType SecretsScanType, module jf return } secretScanManager := newSecretsScanManager(scanner, scanType, scannerTempDir) - cuurentDir, _ := os.Getwd() - log.Debug(fmt.Sprintf("current dir is: %s", cuurentDir)) log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running secrets scan...") if err = secretScanManager.scanner.Run(secretScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Secrets, err) @@ -96,8 +92,6 @@ type secretsScanConfiguration struct { } func (s *SecretScanManager) createConfigFile(module jfrogappsconfig.Module) error { - cuurentDir, _ := os.Getwd() - log.Debug(fmt.Sprintf("current dir while creating config file: %s", cuurentDir)) roots, err := jas.GetSourceRoots(module, module.Scanners.Secrets) if err != nil { return err diff --git a/utils/parallel_runner.go b/utils/parallel_runner.go index 368152a3..875c6dbc 100644 --- a/utils/parallel_runner.go +++ b/utils/parallel_runner.go @@ -6,12 +6,12 @@ import ( ) type SecurityParallelRunner struct { - Runner parallel.Runner - ErrorsQueue chan error - ResultsMu sync.Mutex - ScaScansWg sync.WaitGroup // Verify that the sca scan routines are done before running contextual scan - ScannersWg sync.WaitGroup // Verify that all scanners routines are done before cleaning temp dir - JasWg sync.WaitGroup // Verify that downloading analyzer manager and running all scanners are done + Runner parallel.Runner + ErrorsQueue chan error + ResultsMu sync.Mutex + ScaScansWg sync.WaitGroup // Verify that the sca scan routines are done before running contextual scan + JasScannersWg sync.WaitGroup // Verify that all scanners routines are done before cleaning temp dir + JasWg sync.WaitGroup // Verify that downloading analyzer manager and running all scanners are done } func NewSecurityParallelRunner(numOfParallelScans int) SecurityParallelRunner { From 2986a0d2e24b53e7398a62eb17adea760b667b2d Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sat, 15 Jun 2024 11:55:22 +0300 Subject: [PATCH 41/59] rollback to sca scan not as task --- commands/audit/audit.go | 13 ++++--------- commands/audit/scarunner.go | 14 ++++++-------- jas/runner/jasrunner.go | 5 +---- jas/runner/jasrunner_test.go | 10 +++------- 4 files changed, 14 insertions(+), 28 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index db2ab72a..7fe483ef 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -203,15 +203,6 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) } - - auditParallelRunner.ScaScansWg.Add(1) - auditParallelRunner.JasScannersWg.Add(1) - if _, scaErr := auditParallelRunner.Runner.AddTaskWithError(func(threadId int) error { - return buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results, threadId) - }, auditParallelRunner.AddErrorToChan); scaErr != nil { - auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create sca scan task %s", scaErr.Error())) - } - jasScanner := &jas.JasScanner{} if results.ExtendedScanResults.EntitledForJas { // Download (if needed) the analyzer manager and run scanners. @@ -223,6 +214,10 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } } + // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. + if scaScanErr := buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results); scaScanErr != nil { + auditParallelRunner.AddErrorToChan(scaScanErr) + } go func() { auditParallelRunner.ScaScansWg.Wait() auditParallelRunner.JasWg.Wait() diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index f6b795e9..a33e1a89 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -34,18 +34,12 @@ import ( "time" ) -func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner, auditParams *AuditParams, results *xrayutils.Results, threadId int) (err error) { +func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner, auditParams *AuditParams, results *xrayutils.Results) (err error) { // Prepare currentWorkingDir, err := os.Getwd() if errorutils.CheckError(err) != nil { return } - defer func() { - // Make sure to return to the original working directory, buildDependencyTree may change it - err = errors.Join(err, os.Chdir(currentWorkingDir)) - auditParallelRunner.ScaScansWg.Done() - auditParallelRunner.JasScannersWg.Done() - }() serverDetails, err := auditParams.ServerDetails() if err != nil { return @@ -60,8 +54,12 @@ func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner if err != nil { return } - log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Preforming", len(scans), "SCA scans:\n", scanInfo) + log.Info(fmt.Sprintf("Preforming %d SCA scans:\n%s", len(scans), scanInfo)) + defer func() { + // Make sure to return to the original working directory, buildDependencyTree may change it + err = errors.Join(err, os.Chdir(currentWorkingDir)) + }() for _, scan := range scans { // Get the dependency tree for the technology in the working directory. treeResult, bdtErr := buildDependencyTree(scan, auditParams) diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index eaa28ddf..1a476d60 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -32,9 +32,6 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s defer func() { callback() }() - - // wait for the building of the dependency tree to finish, so the current dir would the root project - securityParallelRunner.JasScannersWg.Wait() // Don't execute other scanners when scanning third party dependencies. if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { @@ -128,7 +125,7 @@ func runContextualScan(securityParallelRunner *utils.SecurityParallelRunner, sca defer func() { securityParallelRunner.JasScannersWg.Done() }() - // Wait for sca scan to complete before running contextual scan + // Wait for sca scans to complete before running contextual scan securityParallelRunner.ScaScansWg.Wait() results, err := applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), *directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) if err != nil { diff --git a/jas/runner/jasrunner_test.go b/jas/runner/jasrunner_test.go index 279f1102..16ea0390 100644 --- a/jas/runner/jasrunner_test.go +++ b/jas/runner/jasrunner_test.go @@ -16,23 +16,19 @@ import ( ) func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { - securityParallelRunnerForTest := utils.CreateSecurityParallelRunner(cliutils.Threads) tmpDir, err := fileutils.CreateTempDir() + assert.NoError(t, err) defer func() { assert.NoError(t, fileutils.RemoveTempDir(tmpDir)) }() - assert.NoError(t, err) assert.NoError(t, os.Setenv(coreutils.HomeDir, tmpDir)) defer func() { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() scanner := &jas.JasScanner{} - jasScanner, err := jas.CreateJasScanner(scanner, nil, &jas.FakeServerDetails) - assert.NoError(t, err) - scanResults := &utils.Results{ScaResults: []*utils.ScaScanResult{{Technology: techutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err = AddJasScannersTasks(securityParallelRunnerForTest, scanResults, scanResults.GetScaScannedTechnologies(), &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, false, "", jasScanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan) - // Expect error: + _, err = jas.CreateJasScanner(scanner, nil, &jas.FakeServerDetails) assert.Error(t, err) + assert.ErrorContains(t, err, "unable to locate the analyzer manager package. Advanced security scans cannot be performed without this package") } func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { From affc60eb43e467262c7b9457fc71d68b67ce1588 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sat, 15 Jun 2024 16:48:01 +0300 Subject: [PATCH 42/59] add mutex to results --- commands/audit/audit.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 7fe483ef..45bfad67 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -229,7 +229,9 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) }() go func() { for e := range auditParallelRunner.ErrorsQueue { + auditParallelRunner.ResultsMu.Lock() results.ScansErr = errors.Join(results.ScansErr, e) + auditParallelRunner.ResultsMu.Unlock() } }() if auditParams.Progress() != nil { From 021a5d1b9d7f813b0983d2c9bd9b82110ccd9363 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sat, 15 Jun 2024 17:25:25 +0300 Subject: [PATCH 43/59] add mutex to results --- commands/audit/scarunner.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index a33e1a89..734af86e 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -76,7 +76,9 @@ func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner return fmt.Errorf("failed to create sca scan task for '%s': %s", scan.Target, taskErr.Error()) } // Add the scan to the results + auditParallelRunner.ResultsMu.Lock() results.ScaResults = append(results.ScaResults, scan) + auditParallelRunner.ResultsMu.Unlock() } return } From e9787db007f74b9bcbd4b1545b116d7de3cde183 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sat, 15 Jun 2024 18:20:37 +0300 Subject: [PATCH 44/59] add mutex to results --- commands/audit/audit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 45bfad67..ea065c33 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -230,7 +230,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) go func() { for e := range auditParallelRunner.ErrorsQueue { auditParallelRunner.ResultsMu.Lock() - results.ScansErr = errors.Join(results.ScansErr, e) + errors.Join(results.ScansErr, e) auditParallelRunner.ResultsMu.Unlock() } }() From 4250a6cc174c0e998970630a15996188569173c5 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sat, 15 Jun 2024 18:28:59 +0300 Subject: [PATCH 45/59] add mutex to results --- commands/audit/audit.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index ea065c33..f987d894 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -229,9 +229,8 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) }() go func() { for e := range auditParallelRunner.ErrorsQueue { - auditParallelRunner.ResultsMu.Lock() - errors.Join(results.ScansErr, e) - auditParallelRunner.ResultsMu.Unlock() + newErrs := errors.Join(results.ScansErr, e) + results.ScansErr = newErrs } }() if auditParams.Progress() != nil { From 28f8c33dd2402ee795d51103f44b9c8def9eb0da Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sat, 15 Jun 2024 18:47:30 +0300 Subject: [PATCH 46/59] add mutex to results --- commands/audit/audit.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index f987d894..805d88b2 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -229,8 +229,10 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) }() go func() { for e := range auditParallelRunner.ErrorsQueue { + auditParallelRunner.ResultsMu.Lock() newErrs := errors.Join(results.ScansErr, e) results.ScansErr = newErrs + auditParallelRunner.ResultsMu.Unlock() } }() if auditParams.Progress() != nil { From 212d4fb4f35adefd80bfbc933d8618ce964b3e99 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 09:44:36 +0300 Subject: [PATCH 47/59] create audit parallel runner in Run func --- commands/audit/audit.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 805d88b2..d6294d04 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -133,7 +133,8 @@ func (auditCmd *AuditCommand) Run() (err error) { SetThreads(auditCmd.Threads) auditParams.SetIsRecursiveScan(isRecursiveScan).SetExclusions(auditCmd.Exclusions()) - auditResults, err := RunAudit(auditParams) + auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) + auditResults, err := RunAudit(auditParams, auditParallelRunner) if err != nil { return } @@ -159,8 +160,11 @@ func (auditCmd *AuditCommand) Run() (err error) { return } - if auditResults.ScansErr != nil { - return auditResults.ScansErr + auditParallelRunner.ResultsMu.Lock() + err = auditResults.ScansErr + auditParallelRunner.ResultsMu.Unlock() + if err != nil { + return err } // Only in case Xray's context was given (!auditCmd.IncludeVulnerabilities), and the user asked to fail the build accordingly, do so. @@ -177,7 +181,7 @@ func (auditCmd *AuditCommand) CommandName() string { // Runs an audit scan based on the provided auditParams. // Returns an audit Results object containing all the scan results. // If the current server is entitled for JAS, the advanced security results will be included in the scan results. -func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) { +func RunAudit(auditParams *AuditParams, auditParallelRunner *utils.SecurityParallelRunner) (results *xrayutils.Results, err error) { // Initialize Results struct results = xrayutils.NewAuditResults() serverDetails, err := auditParams.ServerDetails() @@ -198,7 +202,6 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId - auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) From cb2ac87a2201339ec43b3ff3d5e4b6814177eaa6 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 10:22:30 +0300 Subject: [PATCH 48/59] move back parallelRunner to runAudit --- commands/audit/audit.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index d6294d04..805d88b2 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -133,8 +133,7 @@ func (auditCmd *AuditCommand) Run() (err error) { SetThreads(auditCmd.Threads) auditParams.SetIsRecursiveScan(isRecursiveScan).SetExclusions(auditCmd.Exclusions()) - auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) - auditResults, err := RunAudit(auditParams, auditParallelRunner) + auditResults, err := RunAudit(auditParams) if err != nil { return } @@ -160,11 +159,8 @@ func (auditCmd *AuditCommand) Run() (err error) { return } - auditParallelRunner.ResultsMu.Lock() - err = auditResults.ScansErr - auditParallelRunner.ResultsMu.Unlock() - if err != nil { - return err + if auditResults.ScansErr != nil { + return auditResults.ScansErr } // Only in case Xray's context was given (!auditCmd.IncludeVulnerabilities), and the user asked to fail the build accordingly, do so. @@ -181,7 +177,7 @@ func (auditCmd *AuditCommand) CommandName() string { // Runs an audit scan based on the provided auditParams. // Returns an audit Results object containing all the scan results. // If the current server is entitled for JAS, the advanced security results will be included in the scan results. -func RunAudit(auditParams *AuditParams, auditParallelRunner *utils.SecurityParallelRunner) (results *xrayutils.Results, err error) { +func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) { // Initialize Results struct results = xrayutils.NewAuditResults() serverDetails, err := auditParams.ServerDetails() @@ -202,6 +198,7 @@ func RunAudit(auditParams *AuditParams, auditParallelRunner *utils.SecurityParal } results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId + auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) From 5752db71e9800499d56d1e27584141fdcedda58c Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 10:39:12 +0300 Subject: [PATCH 49/59] change to select --- commands/audit/audit.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 805d88b2..9a696854 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -228,11 +228,16 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) auditParallelRunner.Runner.Done() }() go func() { - for e := range auditParallelRunner.ErrorsQueue { + select { + case e, ok := <-auditParallelRunner.ErrorsQueue: + if !ok { + return + } auditParallelRunner.ResultsMu.Lock() - newErrs := errors.Join(results.ScansErr, e) - results.ScansErr = newErrs + results.ScansErr = errors.Join(results.ScansErr, e) auditParallelRunner.ResultsMu.Unlock() + default: + return } }() if auditParams.Progress() != nil { From cfaa371ea7d48e4e92cbee4588f444447d495672 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 12:03:12 +0300 Subject: [PATCH 50/59] fix err handle --- commands/audit/audit.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 9a696854..72c967c8 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -228,16 +228,16 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) auditParallelRunner.Runner.Done() }() go func() { - select { - case e, ok := <-auditParallelRunner.ErrorsQueue: - if !ok { - return + for { + select { + case e, ok := <-auditParallelRunner.ErrorsQueue: + if !ok { + return + } + auditParallelRunner.ResultsMu.Lock() + results.ScansErr = errors.Join(results.ScansErr, e) + auditParallelRunner.ResultsMu.Unlock() } - auditParallelRunner.ResultsMu.Lock() - results.ScansErr = errors.Join(results.ScansErr, e) - auditParallelRunner.ResultsMu.Unlock() - default: - return } }() if auditParams.Progress() != nil { From cd11c22a734b75f7ece42717d8d52a6d1d2aa4dd Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 13:45:49 +0300 Subject: [PATCH 51/59] add mutex before accessing results struct --- jas/runner/jasrunner.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index 1a476d60..c26babbc 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -35,14 +35,14 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s // Don't execute other scanners when scanning third party dependencies. if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { - if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module, secretsScanType), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults, module, secretsScanType), errHandlerFunc); err != nil { return } if runAllScanners { - if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults, module), errHandlerFunc); err != nil { return } - if err = addModuleJasScanTask(module, utils.Sast, securityParallelRunner, runSastScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.Sast, securityParallelRunner, runSastScan(securityParallelRunner, scanner, scanResults, module), errHandlerFunc); err != nil { return } } @@ -68,7 +68,7 @@ func addModuleJasScanTask(module jfrogappsconfig.Module, scanType utils.JasScanT return } -func runSecretsScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, +func runSecretsScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, module jfrogappsconfig.Module, secretsScanType secrets.SecretsScanType) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { @@ -79,13 +79,13 @@ func runSecretsScan(securityParallelRunner *utils.SecurityParallelRunner, scanne return fmt.Errorf("%s%s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() - extendedScanResults.SecretsScanResults = append(extendedScanResults.SecretsScanResults, results...) + scanResults.ExtendedScanResults.SecretsScanResults = append(scanResults.ExtendedScanResults.SecretsScanResults, results...) securityParallelRunner.ResultsMu.Unlock() return } } -func runIacScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, +func runIacScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { @@ -96,13 +96,13 @@ func runIacScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *j return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() - extendedScanResults.IacScanResults = append(extendedScanResults.IacScanResults, results...) + scanResults.ExtendedScanResults.IacScanResults = append(scanResults.ExtendedScanResults.IacScanResults, results...) securityParallelRunner.ResultsMu.Unlock() return } } -func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, +func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { @@ -113,7 +113,7 @@ func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner * return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() - extendedScanResults.SastScanResults = append(extendedScanResults.SastScanResults, results...) + scanResults.ExtendedScanResults.SastScanResults = append(scanResults.ExtendedScanResults.SastScanResults, results...) securityParallelRunner.ResultsMu.Unlock() return } @@ -127,7 +127,12 @@ func runContextualScan(securityParallelRunner *utils.SecurityParallelRunner, sca }() // Wait for sca scans to complete before running contextual scan securityParallelRunner.ScaScansWg.Wait() - results, err := applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), *directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) + + securityParallelRunner.ResultsMu.Lock() + xrayScanResults := scanResults.GetScaScansXrayResults() + securityParallelRunner.ResultsMu.Unlock() + + results, err := applicability.RunApplicabilityScan(xrayScanResults, *directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) if err != nil { return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } From c1f79a87d6629c820d28f74d020a1f94c0e3491b Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 13:56:54 +0300 Subject: [PATCH 52/59] new mutex only for err handling --- commands/audit/audit.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 72c967c8..868970ff 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -8,6 +8,7 @@ import ( "github.com/jfrog/jfrog-cli-security/jas/runner" "github.com/jfrog/jfrog-cli-security/jas/secrets" "os" + "sync" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -218,6 +219,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) if scaScanErr := buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results); scaScanErr != nil { auditParallelRunner.AddErrorToChan(scaScanErr) } + mu := sync.Mutex{} go func() { auditParallelRunner.ScaScansWg.Wait() auditParallelRunner.JasWg.Wait() @@ -234,9 +236,9 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) if !ok { return } - auditParallelRunner.ResultsMu.Lock() + mu.Lock() results.ScansErr = errors.Join(results.ScansErr, e) - auditParallelRunner.ResultsMu.Unlock() + mu.Unlock() } } }() From 196c6fa26d9066e618dfcd32b4013f14d10a2256 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 14:30:02 +0300 Subject: [PATCH 53/59] add mutex to all results.ScanErr --- commands/audit/audit.go | 21 +++++++++++---------- utils/analyticsmetrics.go | 4 +++- utils/analyticsmetrics_test.go | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 868970ff..475e8452 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -8,7 +8,6 @@ import ( "github.com/jfrog/jfrog-cli-security/jas/runner" "github.com/jfrog/jfrog-cli-security/jas/secrets" "os" - "sync" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -134,11 +133,12 @@ func (auditCmd *AuditCommand) Run() (err error) { SetThreads(auditCmd.Threads) auditParams.SetIsRecursiveScan(isRecursiveScan).SetExclusions(auditCmd.Exclusions()) - auditResults, err := RunAudit(auditParams) + auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) + auditResults, err := RunAudit(auditParams, auditParallelRunner) if err != nil { return } - auditCmd.analyticsMetricsService.UpdateGeneralEvent(auditCmd.analyticsMetricsService.CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(auditResults)) + auditCmd.analyticsMetricsService.UpdateGeneralEvent(auditCmd.analyticsMetricsService.CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(auditResults, auditParallelRunner)) if auditCmd.Progress() != nil { if err = auditCmd.Progress().Quit(); err != nil { return @@ -160,8 +160,11 @@ func (auditCmd *AuditCommand) Run() (err error) { return } - if auditResults.ScansErr != nil { - return auditResults.ScansErr + auditParallelRunner.ResultsMu.Lock() + errs := auditResults.ScansErr + auditParallelRunner.ResultsMu.Unlock() + if errs != nil { + return errs } // Only in case Xray's context was given (!auditCmd.IncludeVulnerabilities), and the user asked to fail the build accordingly, do so. @@ -178,7 +181,7 @@ func (auditCmd *AuditCommand) CommandName() string { // Runs an audit scan based on the provided auditParams. // Returns an audit Results object containing all the scan results. // If the current server is entitled for JAS, the advanced security results will be included in the scan results. -func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) { +func RunAudit(auditParams *AuditParams, auditParallelRunner *xrayutils.SecurityParallelRunner) (results *xrayutils.Results, err error) { // Initialize Results struct results = xrayutils.NewAuditResults() serverDetails, err := auditParams.ServerDetails() @@ -199,7 +202,6 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId - auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) @@ -219,7 +221,6 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) if scaScanErr := buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results); scaScanErr != nil { auditParallelRunner.AddErrorToChan(scaScanErr) } - mu := sync.Mutex{} go func() { auditParallelRunner.ScaScansWg.Wait() auditParallelRunner.JasWg.Wait() @@ -236,9 +237,9 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) if !ok { return } - mu.Lock() + auditParallelRunner.ResultsMu.Lock() results.ScansErr = errors.Join(results.ScansErr, e) - mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() } } }() diff --git a/utils/analyticsmetrics.go b/utils/analyticsmetrics.go index b9b57beb..f1814fb2 100644 --- a/utils/analyticsmetrics.go +++ b/utils/analyticsmetrics.go @@ -154,12 +154,14 @@ func (ams *AnalyticsMetricsService) GetGeneralEvent(msi string) (*xscservices.Xs return event, err } -func (ams *AnalyticsMetricsService) CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(auditResults *Results) *xscservices.XscAnalyticsGeneralEventFinalize { +func (ams *AnalyticsMetricsService) CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(auditResults *Results, auditParallelRunner *SecurityParallelRunner) *xscservices.XscAnalyticsGeneralEventFinalize { totalDuration := time.Since(ams.GetStartTime()) eventStatus := xscservices.Completed + auditParallelRunner.ResultsMu.Lock() if auditResults.ScansErr != nil { eventStatus = xscservices.Failed } + auditParallelRunner.ResultsMu.Unlock() basicEvent := xscservices.XscAnalyticsBasicGeneralEvent{ EventStatus: eventStatus, diff --git a/utils/analyticsmetrics_test.go b/utils/analyticsmetrics_test.go index 7e8b8890..a0a9746d 100644 --- a/utils/analyticsmetrics_test.go +++ b/utils/analyticsmetrics_test.go @@ -111,7 +111,7 @@ func TestAnalyticsMetricsService_createAuditResultsFromXscAnalyticsBasicGeneralE time.Sleep(time.Millisecond) for _, tt := range testStruct { t.Run(tt.name, func(t *testing.T) { - event := am.CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(tt.auditResults) + event := am.CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(tt.auditResults, &SecurityParallelRunner{}) assert.Equal(t, tt.want.TotalFindings, event.TotalFindings) assert.Equal(t, tt.want.EventStatus, event.EventStatus) totalDuration, err := time.ParseDuration(event.TotalScanDuration) From 423a4a418280f107f7df09d2e9c4417e67020cc0 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 15:06:45 +0300 Subject: [PATCH 54/59] add wg for errors --- commands/audit/audit.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 475e8452..ef1bcdb4 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -8,6 +8,7 @@ import ( "github.com/jfrog/jfrog-cli-security/jas/runner" "github.com/jfrog/jfrog-cli-security/jas/secrets" "os" + "sync" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -221,6 +222,7 @@ func RunAudit(auditParams *AuditParams, auditParallelRunner *xrayutils.SecurityP if scaScanErr := buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results); scaScanErr != nil { auditParallelRunner.AddErrorToChan(scaScanErr) } + testWG := sync.WaitGroup{} go func() { auditParallelRunner.ScaScansWg.Wait() auditParallelRunner.JasWg.Wait() @@ -231,6 +233,8 @@ func RunAudit(auditParams *AuditParams, auditParallelRunner *xrayutils.SecurityP auditParallelRunner.Runner.Done() }() go func() { + testWG.Add(1) + defer testWG.Done() for { select { case e, ok := <-auditParallelRunner.ErrorsQueue: @@ -247,6 +251,7 @@ func RunAudit(auditParams *AuditParams, auditParallelRunner *xrayutils.SecurityP auditParams.Progress().SetHeadlineMsg("Scanning for issues") } auditParallelRunner.Runner.Run() + testWG.Wait() return } From 1de40e7e1464feb24a7557029053d65849701d44 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 15:31:55 +0300 Subject: [PATCH 55/59] remove all mutex to results object --- commands/audit/audit.go | 32 ++++++++++++++------------------ jas/runner/jasrunner.go | 18 +++++++++--------- utils/analyticsmetrics.go | 4 +--- utils/analyticsmetrics_test.go | 2 +- utils/parallel_runner.go | 1 + 5 files changed, 26 insertions(+), 31 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index ef1bcdb4..7c0f9ed8 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -4,15 +4,13 @@ import ( "errors" "fmt" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-security/jas/applicability" "github.com/jfrog/jfrog-cli-security/jas/runner" "github.com/jfrog/jfrog-cli-security/jas/secrets" - "os" - "sync" - - "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-security/scangraph" + "os" "github.com/jfrog/jfrog-cli-security/jas" "github.com/jfrog/jfrog-cli-security/utils" @@ -134,12 +132,11 @@ func (auditCmd *AuditCommand) Run() (err error) { SetThreads(auditCmd.Threads) auditParams.SetIsRecursiveScan(isRecursiveScan).SetExclusions(auditCmd.Exclusions()) - auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) - auditResults, err := RunAudit(auditParams, auditParallelRunner) + auditResults, err := RunAudit(auditParams) if err != nil { return } - auditCmd.analyticsMetricsService.UpdateGeneralEvent(auditCmd.analyticsMetricsService.CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(auditResults, auditParallelRunner)) + auditCmd.analyticsMetricsService.UpdateGeneralEvent(auditCmd.analyticsMetricsService.CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(auditResults)) if auditCmd.Progress() != nil { if err = auditCmd.Progress().Quit(); err != nil { return @@ -161,11 +158,8 @@ func (auditCmd *AuditCommand) Run() (err error) { return } - auditParallelRunner.ResultsMu.Lock() - errs := auditResults.ScansErr - auditParallelRunner.ResultsMu.Unlock() - if errs != nil { - return errs + if auditResults.ScansErr != nil { + return auditResults.ScansErr } // Only in case Xray's context was given (!auditCmd.IncludeVulnerabilities), and the user asked to fail the build accordingly, do so. @@ -182,7 +176,7 @@ func (auditCmd *AuditCommand) CommandName() string { // Runs an audit scan based on the provided auditParams. // Returns an audit Results object containing all the scan results. // If the current server is entitled for JAS, the advanced security results will be included in the scan results. -func RunAudit(auditParams *AuditParams, auditParallelRunner *xrayutils.SecurityParallelRunner) (results *xrayutils.Results, err error) { +func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) { // Initialize Results struct results = xrayutils.NewAuditResults() serverDetails, err := auditParams.ServerDetails() @@ -203,6 +197,7 @@ func RunAudit(auditParams *AuditParams, auditParallelRunner *xrayutils.SecurityP } results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId + auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) @@ -222,7 +217,6 @@ func RunAudit(auditParams *AuditParams, auditParallelRunner *xrayutils.SecurityP if scaScanErr := buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, results); scaScanErr != nil { auditParallelRunner.AddErrorToChan(scaScanErr) } - testWG := sync.WaitGroup{} go func() { auditParallelRunner.ScaScansWg.Wait() auditParallelRunner.JasWg.Wait() @@ -230,11 +224,13 @@ func RunAudit(auditParams *AuditParams, auditParallelRunner *xrayutils.SecurityP auditParallelRunner.JasScannersWg.Wait() cleanup := jasScanner.ScannerDirCleanupFunc auditParallelRunner.AddErrorToChan(cleanup()) + close(auditParallelRunner.ErrorsQueue) auditParallelRunner.Runner.Done() }() + // a new routine that collects errors from the err channel into results object go func() { - testWG.Add(1) - defer testWG.Done() + auditParallelRunner.ErrWg.Add(1) + defer auditParallelRunner.ErrWg.Done() for { select { case e, ok := <-auditParallelRunner.ErrorsQueue: @@ -251,7 +247,7 @@ func RunAudit(auditParams *AuditParams, auditParallelRunner *xrayutils.SecurityP auditParams.Progress().SetHeadlineMsg("Scanning for issues") } auditParallelRunner.Runner.Run() - testWG.Wait() + auditParallelRunner.ErrWg.Wait() return } diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index c26babbc..d7a998c9 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -35,14 +35,14 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s // Don't execute other scanners when scanning third party dependencies. if !thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { - if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults, module, secretsScanType), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module, secretsScanType), errHandlerFunc); err != nil { return } if runAllScanners { - if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults, module), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { return } - if err = addModuleJasScanTask(module, utils.Sast, securityParallelRunner, runSastScan(securityParallelRunner, scanner, scanResults, module), errHandlerFunc); err != nil { + if err = addModuleJasScanTask(module, utils.Sast, securityParallelRunner, runSastScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil { return } } @@ -68,7 +68,7 @@ func addModuleJasScanTask(module jfrogappsconfig.Module, scanType utils.JasScanT return } -func runSecretsScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, +func runSecretsScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module, secretsScanType secrets.SecretsScanType) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { @@ -79,13 +79,13 @@ func runSecretsScan(securityParallelRunner *utils.SecurityParallelRunner, scanne return fmt.Errorf("%s%s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() - scanResults.ExtendedScanResults.SecretsScanResults = append(scanResults.ExtendedScanResults.SecretsScanResults, results...) + extendedScanResults.SecretsScanResults = append(extendedScanResults.SecretsScanResults, results...) securityParallelRunner.ResultsMu.Unlock() return } } -func runIacScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, +func runIacScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { @@ -96,13 +96,13 @@ func runIacScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *j return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() - scanResults.ExtendedScanResults.IacScanResults = append(scanResults.ExtendedScanResults.IacScanResults, results...) + extendedScanResults.IacScanResults = append(extendedScanResults.IacScanResults, results...) securityParallelRunner.ResultsMu.Unlock() return } } -func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, +func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner *jas.JasScanner, extendedScanResults *utils.ExtendedScanResults, module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { defer func() { @@ -113,7 +113,7 @@ func runSastScan(securityParallelRunner *utils.SecurityParallelRunner, scanner * return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } securityParallelRunner.ResultsMu.Lock() - scanResults.ExtendedScanResults.SastScanResults = append(scanResults.ExtendedScanResults.SastScanResults, results...) + extendedScanResults.SastScanResults = append(extendedScanResults.SastScanResults, results...) securityParallelRunner.ResultsMu.Unlock() return } diff --git a/utils/analyticsmetrics.go b/utils/analyticsmetrics.go index f1814fb2..b9b57beb 100644 --- a/utils/analyticsmetrics.go +++ b/utils/analyticsmetrics.go @@ -154,14 +154,12 @@ func (ams *AnalyticsMetricsService) GetGeneralEvent(msi string) (*xscservices.Xs return event, err } -func (ams *AnalyticsMetricsService) CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(auditResults *Results, auditParallelRunner *SecurityParallelRunner) *xscservices.XscAnalyticsGeneralEventFinalize { +func (ams *AnalyticsMetricsService) CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(auditResults *Results) *xscservices.XscAnalyticsGeneralEventFinalize { totalDuration := time.Since(ams.GetStartTime()) eventStatus := xscservices.Completed - auditParallelRunner.ResultsMu.Lock() if auditResults.ScansErr != nil { eventStatus = xscservices.Failed } - auditParallelRunner.ResultsMu.Unlock() basicEvent := xscservices.XscAnalyticsBasicGeneralEvent{ EventStatus: eventStatus, diff --git a/utils/analyticsmetrics_test.go b/utils/analyticsmetrics_test.go index a0a9746d..7e8b8890 100644 --- a/utils/analyticsmetrics_test.go +++ b/utils/analyticsmetrics_test.go @@ -111,7 +111,7 @@ func TestAnalyticsMetricsService_createAuditResultsFromXscAnalyticsBasicGeneralE time.Sleep(time.Millisecond) for _, tt := range testStruct { t.Run(tt.name, func(t *testing.T) { - event := am.CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(tt.auditResults, &SecurityParallelRunner{}) + event := am.CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(tt.auditResults) assert.Equal(t, tt.want.TotalFindings, event.TotalFindings) assert.Equal(t, tt.want.EventStatus, event.EventStatus) totalDuration, err := time.ParseDuration(event.TotalScanDuration) diff --git a/utils/parallel_runner.go b/utils/parallel_runner.go index 875c6dbc..b4068091 100644 --- a/utils/parallel_runner.go +++ b/utils/parallel_runner.go @@ -12,6 +12,7 @@ type SecurityParallelRunner struct { ScaScansWg sync.WaitGroup // Verify that the sca scan routines are done before running contextual scan JasScannersWg sync.WaitGroup // Verify that all scanners routines are done before cleaning temp dir JasWg sync.WaitGroup // Verify that downloading analyzer manager and running all scanners are done + ErrWg sync.WaitGroup // Verify that all errors are handled before finishing the audit func } func NewSecurityParallelRunner(numOfParallelScans int) SecurityParallelRunner { From 699fc36e7780607f1c11d3a522e750f80bc97c2e Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 15:48:57 +0300 Subject: [PATCH 56/59] add errWg outside new go routine --- commands/audit/audit.go | 2 +- jas/runner/jasrunner.go | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 7c0f9ed8..d011f416 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -198,6 +198,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId auditParallelRunner := utils.CreateSecurityParallelRunner(auditParams.threads) + auditParallelRunner.ErrWg.Add(1) jfrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) @@ -229,7 +230,6 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) }() // a new routine that collects errors from the err channel into results object go func() { - auditParallelRunner.ErrWg.Add(1) defer auditParallelRunner.ErrWg.Done() for { select { diff --git a/jas/runner/jasrunner.go b/jas/runner/jasrunner.go index d7a998c9..1a476d60 100644 --- a/jas/runner/jasrunner.go +++ b/jas/runner/jasrunner.go @@ -127,12 +127,7 @@ func runContextualScan(securityParallelRunner *utils.SecurityParallelRunner, sca }() // Wait for sca scans to complete before running contextual scan securityParallelRunner.ScaScansWg.Wait() - - securityParallelRunner.ResultsMu.Lock() - xrayScanResults := scanResults.GetScaScansXrayResults() - securityParallelRunner.ResultsMu.Unlock() - - results, err := applicability.RunApplicabilityScan(xrayScanResults, *directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) + results, err := applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), *directDependencies, scanner, thirdPartyApplicabilityScan, scanType, module, threadId) if err != nil { return fmt.Errorf("%s %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error()) } From 4def17c1a1be99645158b419dd4f9b1105b90320 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 16:23:56 +0300 Subject: [PATCH 57/59] remove format parameter from audit jas tests --- audit_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/audit_test.go b/audit_test.go index 313c68a8..1e0a2bdb 100644 --- a/audit_test.go +++ b/audit_test.go @@ -435,27 +435,27 @@ func addDummyPackageDescriptor(t *testing.T, hasPackageJson bool) { // JAS func TestXrayAuditJasSimpleJson(t *testing.T) { - output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("jas", "jas"), "3") + output := testXrayAuditJas(t, filepath.Join("jas", "jas"), "3") securityTestUtils.VerifySimpleJsonJasResults(t, output, 1, 9, 7, 3, 0, 2, 2) } func TestXrayAuditJasSimpleJsonWithOneThread(t *testing.T) { - output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("jas", "jas"), "1") + output := testXrayAuditJas(t, filepath.Join("jas", "jas"), "1") securityTestUtils.VerifySimpleJsonJasResults(t, output, 1, 9, 7, 3, 0, 2, 2) } func TestXrayAuditJasSimpleJsonWithConfig(t *testing.T) { - output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("jas", "jas-config"), "3") + output := testXrayAuditJas(t, filepath.Join("jas", "jas-config"), "3") securityTestUtils.VerifySimpleJsonJasResults(t, output, 0, 0, 1, 3, 0, 2, 2) } func TestXrayAuditJasNoViolationsSimpleJson(t *testing.T) { - output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("package-managers", "npm", "npm"), "3") + output := testXrayAuditJas(t, filepath.Join("package-managers", "npm", "npm"), "3") securityTestUtils.VerifySimpleJsonScanResults(t, output, 0, 1, 0) securityTestUtils.VerifySimpleJsonJasResults(t, output, 0, 0, 0, 0, 0, 0, 1) } -func testXrayAuditJas(t *testing.T, format string, project string, threads string) string { +func testXrayAuditJas(t *testing.T, project string, threads string) string { securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion) tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t) defer createTempDirCallback() @@ -469,7 +469,7 @@ func testXrayAuditJas(t *testing.T, format string, project string, threads strin assert.NoError(t, err) chdirCallback := clientTests.ChangeDirWithCallback(t, baseWd, tempDirPath) defer chdirCallback() - return securityTests.PlatformCli.WithoutCredentials().RunCliCmdWithOutput(t, "audit", "--format="+format, "--threads="+threads) + return securityTests.PlatformCli.WithoutCredentials().RunCliCmdWithOutput(t, "audit", "--format="+string(format.SimpleJson), "--threads="+threads) } func TestXrayAuditDetectTech(t *testing.T) { From cad75b190938e11044f51533ff870e70abfe87cd Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 16:28:11 +0300 Subject: [PATCH 58/59] use for loop instead of select --- commands/audit/audit.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index d011f416..a5eacb24 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -231,16 +231,10 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) // a new routine that collects errors from the err channel into results object go func() { defer auditParallelRunner.ErrWg.Done() - for { - select { - case e, ok := <-auditParallelRunner.ErrorsQueue: - if !ok { - return - } - auditParallelRunner.ResultsMu.Lock() - results.ScansErr = errors.Join(results.ScansErr, e) - auditParallelRunner.ResultsMu.Unlock() - } + for e := range auditParallelRunner.ErrorsQueue { + auditParallelRunner.ResultsMu.Lock() + results.ScansErr = errors.Join(results.ScansErr, e) + auditParallelRunner.ResultsMu.Unlock() } }() if auditParams.Progress() != nil { From dbc7a8221efb6def47dc9f29ff4acd9f2ce7f724 Mon Sep 17 00:00:00 2001 From: Or Zinger Date: Sun, 16 Jun 2024 16:41:05 +0300 Subject: [PATCH 59/59] remove mutex --- commands/audit/audit.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index a5eacb24..8b776647 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -232,9 +232,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) go func() { defer auditParallelRunner.ErrWg.Done() for e := range auditParallelRunner.ErrorsQueue { - auditParallelRunner.ResultsMu.Lock() results.ScansErr = errors.Join(results.ScansErr, e) - auditParallelRunner.ResultsMu.Unlock() } }() if auditParams.Progress() != nil {