Merge remote-tracking branch 'origin/dev'

build/dotnet.go

@@ -84,7 +84,7 @@ func (dm *DotnetModule) CalcDependencies() error {
 	if err != nil {
 		return err
 	}
-	sol, err := solution.Load(dm.solutionPath, slnFile, dm.containingBuild.logger)
+	sol, err := solution.Load(dm.solutionPath, slnFile, "", dm.containingBuild.logger)
 	if err != nil {
 		return err
 	}

build/golang.go

@@ -164,12 +164,12 @@ func (gm *GoModule) getPackagePathIfExists(cachePath, encodedDependencyId string
 func populateZip(packageId, zipPath string) (zipDependency entities.Dependency, err error) {
 	// Zip file dependency for the build-info
 	zipDependency = entities.Dependency{Id: packageId}
-	md5, sha1, sha2, err := utils.GetFileChecksums(zipPath)
+	checksums, err := utils.GetFileChecksums(zipPath)
 	if err != nil {
 		return
 	}
 	zipDependency.Type = "zip"
-	zipDependency.Checksum = entities.Checksum{Sha1: sha1, Md5: md5, Sha256: sha2}
+	zipDependency.Checksum = entities.Checksum{Sha1: checksums[utils.SHA1], Md5: checksums[utils.MD5], Sha256: checksums[utils.SHA256]}
 	return
 }
 

build/testdata/python/pip/expected_pip_buildinfo_with_module_name.json

@@ -30,7 +30,7 @@
         },
         {
           "id": "six:1.16.0",
-          "type": "whl",
+          "type": "metadata",
           "requestedBy": [
             [
               "nltk:3.4.5",

build/testdata/python/pip/expected_pip_buildinfo_without_module_name.json

@@ -30,7 +30,7 @@
         },
         {
           "id": "six:1.16.0",
-          "type": "whl",
+          "type": "metadata",
           "requestedBy": [
             [
               "nltk:3.4.5",

build/utils/dotnet/solution/solution.go

@@ -25,10 +25,10 @@ type Solution interface {
 
 var projectRegExp *regexp.Regexp
 
-func Load(path, slnFile string, log utils.Log) (Solution, error) {
+func Load(path, slnFile, excludePattern string, log utils.Log) (Solution, error) {
 	solution := &solution{path: path, slnFile: slnFile}
 	// Reads all projects from '.sln' files.
-	slnProjects, err := solution.getProjectsListFromSlns(log)
+	slnProjects, err := solution.getProjectsListFromSlns(excludePattern, log)
 	if err != nil {
 		return solution, err
 	}
@@ -148,13 +148,16 @@ func (solution *solution) DependenciesSourcesAndProjectsPathExist() bool {
 	return len(solution.dependenciesSources) > 0 && len(solution.projects) > 0
 }
 
-func (solution *solution) getProjectsListFromSlns(log utils.Log) ([]project.Project, error) {
+func (solution *solution) getProjectsListFromSlns(excludePattern string, log utils.Log) ([]project.Project, error) {
 	slnProjects, err := solution.getProjectsFromSlns()
 	if err != nil {
 		return nil, err
 	}
 	if slnProjects != nil {
-		return solution.parseProjectsFromSolutionFile(slnProjects, log)
+		if len(excludePattern) > 0 {
+			log.Debug(fmt.Sprintf("Testing to exclude projects by pattern: %s", excludePattern))
+		}
+		return solution.parseProjectsFromSolutionFile(slnProjects, excludePattern, log)
 	}
 	return nil, nil
 }
@@ -174,14 +177,22 @@ func (solution *solution) loadProjects(slnProjects []project.Project, log utils.
 	return nil
 }
 
-func (solution *solution) parseProjectsFromSolutionFile(slnProjects []string, log utils.Log) ([]project.Project, error) {
+func (solution *solution) parseProjectsFromSolutionFile(slnProjects []string, excludePattern string, log utils.Log) ([]project.Project, error) {
 	var projects []project.Project
 	for _, projectLine := range slnProjects {
 		projectName, projFilePath, err := parseProjectLine(projectLine, solution.path)
 		if err != nil {
 			log.Error(err)
 			continue
 		}
+		// Exclude projects by pattern.
+		if exclude, err := isProjectExcluded(projFilePath, excludePattern); err != nil {
+			log.Error(err)
+			continue
+		} else if exclude {
+			log.Debug(fmt.Sprintf("Skipping a project \"%s\", since the path '%s' is excluded", projectName, projFilePath))
+			continue
+		}
 		// Looking for .*proj files.
 		if !strings.HasSuffix(filepath.Ext(projFilePath), "proj") {
 			log.Debug(fmt.Sprintf("Skipping a project \"%s\", since it doesn't have a '.*proj' file path.", projectName))
@@ -192,6 +203,13 @@ func (solution *solution) parseProjectsFromSolutionFile(slnProjects []string, lo
 	return projects, nil
 }
 
+func isProjectExcluded(projFilePath, excludePattern string) (exclude bool, err error) {
+	if len(excludePattern) == 0 {
+		return
+	}
+	return regexp.MatchString(excludePattern, projFilePath)
+}
+
 func (solution *solution) loadSingleProjectFromDir(log utils.Log) error {
 	// List files with .*proj extension.
 	projFiles, err := utils.ListFilesByFilterFunc(solution.path, func(filePath string) (bool, error) {

build/utils/dotnet/solution/solution_test.go

@@ -17,7 +17,7 @@ import (
 var logger = utils.NewDefaultLogger(utils.INFO)
 
 func TestEmptySolution(t *testing.T) {
-	solution, err := Load(".", "", logger)
+	solution, err := Load(".", "", "", logger)
 	if err != nil {
 		t.Error(err)
 	}
@@ -142,6 +142,7 @@ func replaceCarriageSign(results []string) {
 }
 
 func TestLoad(t *testing.T) {
+	// Prepare
 	log := utils.NewDefaultLogger(utils.INFO)
 	wd, err := os.Getwd()
 	if err != nil {
@@ -156,13 +157,26 @@ func TestLoad(t *testing.T) {
 	nugetCmd := exec.Command("nuget", "restore", filepath.Join(wd, "tmp", "nugetproj", "solutions", "nugetproj.sln"))
 	assert.NoError(t, nugetCmd.Run())
 
-	// 'nugetproj' contains 2 'packages.config' files for 2 projects -
-	// 1. located in the project's root directory.
-	// 2. located in solutions directory.
-	solution := solution{path: filepath.Join(wd, "testdata", "nugetproj", "solutions"), slnFile: "nugetproj.sln"}
-	solutions, err := Load(solution.path, solution.slnFile, log)
-	if err != nil {
-		t.Error(err)
+	testCases := []struct {
+		name                 string
+		excludePattern       string
+		expectedProjectCount int
+	}{
+		{"noExcludePattern", "", 2},
+		{"excludePattern", "proj1", 1},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			// 'nugetproj' contains 2 'packages.config' files for 2 projects -
+			// 1. located in the project's root directory.
+			// 2. located in solutions directory.
+			solution := solution{path: filepath.Join(wd, "testdata", "nugetproj", "solutions"), slnFile: "nugetproj.sln"}
+			solutions, err := Load(solution.path, solution.slnFile, testCase.excludePattern, log)
+			if err != nil {
+				t.Error(err)
+			}
+			assert.Equal(t, testCase.expectedProjectCount, len(solutions.GetProjects()))
+		})
 	}
-	assert.Equal(t, 2, len(solutions.GetProjects()))
 }

build/utils/npm.go

@@ -5,12 +5,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"golang.org/x/exp/slices"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 
+	"golang.org/x/exp/slices"
+
 	"github.com/buger/jsonparser"
 	"github.com/jfrog/build-info-go/entities"
 	"github.com/jfrog/build-info-go/utils"
@@ -380,7 +381,11 @@ func calculateChecksum(cacache *cacache, name, version, integrity string) (md5 s
 	if err != nil {
 		return
 	}
-	return utils.GetFileChecksums(path)
+	checksums, err := utils.GetFileChecksums(path)
+	if err != nil {
+		return
+	}
+	return checksums[utils.MD5], checksums[utils.SHA1], checksums[utils.SHA256], err
 }
 
 // Merge two scopes and remove duplicates.

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/BurntSushi/toml v1.3.2
 	github.com/CycloneDX/cyclonedx-go v0.8.0
 	github.com/buger/jsonparser v1.1.1
-	github.com/jfrog/gofrog v1.6.0
+	github.com/jfrog/gofrog v1.6.3
 	github.com/minio/sha256-simd v1.0.1
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli/v2 v2.27.1

go.sum

@@ -10,8 +10,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/jfrog/gofrog v1.6.0 h1:jOwb37nHY2PnxePNFJ6e6279Pgkr3di05SbQQw47Mq8=
-github.com/jfrog/gofrog v1.6.0/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg=
+github.com/jfrog/gofrog v1.6.3 h1:F7He0+75HcgCe6SGTSHLFCBDxiE2Ja0tekvvcktW6wc=
+github.com/jfrog/gofrog v1.6.3/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg=
 github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
 github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=

utils/checksum.go

@@ -2,6 +2,8 @@ package utils
 
 import (
 	"bufio"
+	"errors"
+
 	//#nosec G501 -- md5 is supported by Artifactory.
 	"crypto/md5"
 	//#nosec G505 -- sha1 is supported by Artifactory.
@@ -30,27 +32,38 @@ var algorithmFunc = map[Algorithm]func() hash.Hash{
 	SHA256: sha256.New,
 }
 
-func GetFileChecksums(filePath string) (md5, sha1, sha2 string, err error) {
+func GetFileChecksums(filePath string, checksumType ...Algorithm) (checksums map[Algorithm]string, err error) {
 	file, err := os.Open(filePath)
 	if err != nil {
 		return
 	}
 	defer func() {
-		e := file.Close()
-		if err == nil {
-			err = e
-		}
+		err = errors.Join(err, file.Close())
 	}()
-	checksumInfo, err := CalcChecksums(file)
-	if err != nil {
-		return
-	}
-	md5, sha1, sha2 = checksumInfo[MD5], checksumInfo[SHA1], checksumInfo[SHA256]
-	return
+	return CalcChecksums(file, checksumType...)
 }
 
 // CalcChecksums calculates all hashes at once using AsyncMultiWriter. The file is therefore read only once.
 func CalcChecksums(reader io.Reader, checksumType ...Algorithm) (map[Algorithm]string, error) {
+	hashes, err := calcChecksums(reader, checksumType...)
+	if err != nil {
+		return nil, err
+	}
+	results := sumResults(hashes)
+	return results, nil
+}
+
+// CalcChecksumsBytes calculates hashes like `CalcChecksums`, returns result as bytes
+func CalcChecksumsBytes(reader io.Reader, checksumType ...Algorithm) (map[Algorithm][]byte, error) {
+	hashes, err := calcChecksums(reader, checksumType...)
+	if err != nil {
+		return nil, err
+	}
+	results := sumResultsBytes(hashes)
+	return results, nil
+}
+
+func calcChecksums(reader io.Reader, checksumType ...Algorithm) (map[Algorithm]hash.Hash, error) {
 	hashes := getChecksumByAlgorithm(checksumType...)
 	var multiWriter io.Writer
 	pageSize := os.Getpagesize()
@@ -64,8 +77,7 @@ func CalcChecksums(reader io.Reader, checksumType ...Algorithm) (map[Algorithm]s
 	if err != nil {
 		return nil, err
 	}
-	results := sumResults(hashes)
-	return results, nil
+	return hashes, nil
 }
 
 func sumResults(hashes map[Algorithm]hash.Hash) map[Algorithm]string {
@@ -76,6 +88,14 @@ func sumResults(hashes map[Algorithm]hash.Hash) map[Algorithm]string {
 	return results
 }
 
+func sumResultsBytes(hashes map[Algorithm]hash.Hash) map[Algorithm][]byte {
+	results := map[Algorithm][]byte{}
+	for k, v := range hashes {
+		results[k] = v.Sum(nil)
+	}
+	return results
+}
+
 func getChecksumByAlgorithm(checksumType ...Algorithm) map[Algorithm]hash.Hash {
 	hashes := map[Algorithm]hash.Hash{}
 	if len(checksumType) == 0 {

utils/checksum_test.go

@@ -0,0 +1,42 @@
+package utils
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	fileContent    = "Why did the robot bring a ladder to the bar? It heard the drinks were on the house."
+	expectedMd5    = "70bd6370a86813f2504020281e4a2e2e"
+	expectedSha1   = "8c3578ac814c9f02803001a5d3e5d78a7fd0f9cc"
+	expectedSha256 = "093d901b28a59f7d95921f3f4fb97a03fe7a1cf8670507ffb1d6f9a01b3e890a"
+)
+
+func TestGetFileChecksums(t *testing.T) {
+	// Create a temporary file
+	tempFile, err := os.CreateTemp("", "TestGetFileChecksums")
+	assert.NoError(t, err)
+	defer func() {
+		assert.NoError(t, tempFile.Close())
+		assert.NoError(t, os.Remove(tempFile.Name()))
+	}()
+
+	// Write something to the file
+	_, err = tempFile.Write([]byte(fileContent))
+	assert.NoError(t, err)
+
+	// Calculate only sha1 and match
+	checksums, err := GetFileChecksums(tempFile.Name(), SHA1)
+	assert.NoError(t, err)
+	assert.Len(t, checksums, 1)
+	assert.Equal(t, expectedSha1, checksums[SHA1])
+
+	// Calculate md5, sha1 and sha256 checksums and match
+	checksums, err = GetFileChecksums(tempFile.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, expectedMd5, checksums[MD5])
+	assert.Equal(t, expectedSha1, checksums[SHA1])
+	assert.Equal(t, expectedSha256, checksums[SHA256])
+}

utils/fileutils.go

@@ -599,10 +599,10 @@ func calcChecksumDetails(filePath string) (checksum entities.Checksum, err error
 		err = errors.Join(err, file.Close())
 	}()
 
-	checksumInfo, err := CalcChecksums(file)
+	checksums, err := CalcChecksums(file)
 	if err != nil {
 		return entities.Checksum{}, err
 	}
-	checksum = entities.Checksum{Md5: checksumInfo[MD5], Sha1: checksumInfo[SHA1], Sha256: checksumInfo[SHA256]}
+	checksum = entities.Checksum{Md5: checksums[MD5], Sha1: checksums[SHA1], Sha256: checksums[SHA256]}
 	return
 }