Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: golang remote license search when error reading local mod dir #3549

Merged
merged 1 commit into from
Jan 6, 2025
Merged

fix: golang remote license search when error reading local mod dir #3549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions syft/pkg/cataloger/golang/licenses.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,29 +80,38 @@ func remotesForModule(proxies []string, noProxy []string, module string) []strin
return proxies
}

func (c *goLicenseResolver) getLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, moduleName, moduleVersion string) ([]pkg.License, error) {
func (c *goLicenseResolver) getLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, moduleName, moduleVersion string) []pkg.License {
// search the scan target first, ignoring local and remote sources
goLicenses, err := c.findLicensesInSource(ctx, scanner, resolver,
fmt.Sprintf(`**/go/pkg/mod/%s@%s/*`, processCaps(moduleName), moduleVersion),
)
if err != nil || len(goLicenses) > 0 {
return toPkgLicenses(goLicenses), err
if err != nil {
log.WithFields("error", err, "module", moduleName, "version", moduleVersion).Trace("unable to read golang licenses from source")
}
if len(goLicenses) > 0 {
return toPkgLicenses(goLicenses)
}

// look in the local host mod directory...
if c.opts.SearchLocalModCacheLicenses {
goLicenses, err = c.getLicensesFromLocal(ctx, scanner, moduleName, moduleVersion)
if err != nil || len(goLicenses) > 0 {
return toPkgLicenses(goLicenses), err
if err != nil {
log.WithFields("error", err, "module", moduleName, "version", moduleVersion).Trace("unable to read golang licenses local")
}
if len(goLicenses) > 0 {
return toPkgLicenses(goLicenses)
}
}

// download from remote sources
if c.opts.SearchRemoteLicenses {
goLicenses, err = c.getLicensesFromRemote(ctx, scanner, moduleName, moduleVersion)
if err != nil {
log.WithFields("error", err, "module", moduleName, "version", moduleVersion).Debug("unable to read golang licenses remote")
}
}

return toPkgLicenses(goLicenses), err
return toPkgLicenses(goLicenses)
}

func (c *goLicenseResolver) getLicensesFromLocal(ctx context.Context, scanner licenses.Scanner, moduleName, moduleVersion string) ([]goLicense, error) {
Expand Down
174 changes: 84 additions & 90 deletions syft/pkg/cataloger/golang/licenses_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,81 +23,11 @@ import (
"github.com/anchore/syft/syft/pkg"
)

func Test_LocalLicenseSearch(t *testing.T) {
func Test_LicenseSearch(t *testing.T) {
loc1 := file.NewLocation("github.com/someorg/somename@v0.3.2/LICENSE")
loc2 := file.NewLocation("github.com/!cap!o!r!g/!cap!project@v4.111.5/LICENSE.txt")
loc3 := file.NewLocation("github.com/someorg/strangelicense@v1.2.3/LiCeNsE.tXt")

licenseScanner := licenses.TestingOnlyScanner()

tests := []struct {
name string
version string
expected pkg.License
}{
{
name: "github.com/someorg/somename",
version: "v0.3.2",
expected: pkg.License{
Value: "Apache-2.0",
SPDXExpression: "Apache-2.0",
Type: license.Concluded,
URLs: []string{"file://$GOPATH/pkg/mod/" + loc1.RealPath},
Locations: file.NewLocationSet(),
},
},
{
name: "github.com/CapORG/CapProject",
version: "v4.111.5",
expected: pkg.License{
Value: "MIT",
SPDXExpression: "MIT",
Type: license.Concluded,
URLs: []string{"file://$GOPATH/pkg/mod/" + loc2.RealPath},
Locations: file.NewLocationSet(),
},
},
{
name: "github.com/someorg/strangelicense",
version: "v1.2.3",
expected: pkg.License{
Value: "Apache-2.0",
SPDXExpression: "Apache-2.0",
Type: license.Concluded,
URLs: []string{"file://$GOPATH/pkg/mod/" + loc3.RealPath},
Locations: file.NewLocationSet(),
},
},
}

wd, err := os.Getwd()
require.NoError(t, err)

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
l := newGoLicenseResolver(
"",
CatalogerConfig{
SearchLocalModCacheLicenses: true,
LocalModCacheDir: filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod"),
},
)
lics, err := l.getLicenses(context.Background(), licenseScanner, fileresolver.Empty{}, test.name, test.version)
require.NoError(t, err)

require.Len(t, lics, 1)

require.Equal(t, test.expected, lics[0])
})
}
}

func Test_RemoteProxyLicenseSearch(t *testing.T) {
loc1 := file.NewLocation("github.com/someorg/somename@v0.3.2/LICENSE")
loc2 := file.NewLocation("github.com/!cap!o!r!g/!cap!project@v4.111.5/LICENSE.txt")

licenseScanner := licenses.TestingOnlyScanner()

server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf := &bytes.Buffer{}
uri := strings.TrimPrefix(strings.TrimSuffix(r.RequestURI, ".zip"), "/")
Expand Down Expand Up @@ -135,52 +65,116 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
}))
defer server.Close()

wd, err := os.Getwd()
require.NoError(t, err)

licenseScanner := licenses.TestingOnlyScanner()

tests := []struct {
name string
version string
expected pkg.License
config CatalogerConfig
expected []pkg.License
}{
{
name: "github.com/someorg/somename",
version: "v0.3.2",
expected: pkg.License{
config: CatalogerConfig{
SearchLocalModCacheLicenses: true,
LocalModCacheDir: filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod"),
},
expected: []pkg.License{{
Value: "Apache-2.0",
SPDXExpression: "Apache-2.0",
Type: license.Concluded,
URLs: []string{server.URL + "/github.com/someorg/somename/@v/v0.3.2.zip#" + loc1.RealPath},
URLs: []string{"file://$GOPATH/pkg/mod/" + loc1.RealPath},
Locations: file.NewLocationSet(),
}},
},
{
name: "github.com/CapORG/CapProject",
version: "v4.111.5",
config: CatalogerConfig{
SearchLocalModCacheLicenses: true,
LocalModCacheDir: filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod"),
},
expected: []pkg.License{{
Value: "MIT",
SPDXExpression: "MIT",
Type: license.Concluded,
URLs: []string{"file://$GOPATH/pkg/mod/" + loc2.RealPath},
Locations: file.NewLocationSet(),
}},
},
{
name: "github.com/someorg/strangelicense",
version: "v1.2.3",
config: CatalogerConfig{
SearchLocalModCacheLicenses: true,
LocalModCacheDir: filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod"),
},
expected: []pkg.License{{
Value: "Apache-2.0",
SPDXExpression: "Apache-2.0",
Type: license.Concluded,
URLs: []string{"file://$GOPATH/pkg/mod/" + loc3.RealPath},
Locations: file.NewLocationSet(),
}},
},
{
name: "github.com/someorg/somename",
version: "v0.3.2",
config: CatalogerConfig{
SearchRemoteLicenses: true,
Proxies: []string{server.URL},
},
expected: []pkg.License{{
Value: "Apache-2.0",
SPDXExpression: "Apache-2.0",
Type: license.Concluded,
URLs: []string{server.URL + "/github.com/someorg/somename/@v/v0.3.2.zip#" + loc1.RealPath},
Locations: file.NewLocationSet(),
}},
},
{
name: "github.com/CapORG/CapProject",
version: "v4.111.5",
expected: pkg.License{
config: CatalogerConfig{
SearchRemoteLicenses: true,
Proxies: []string{server.URL},
},
expected: []pkg.License{{
Value: "MIT",
SPDXExpression: "MIT",
Type: license.Concluded,
URLs: []string{server.URL + "/github.com/CapORG/CapProject/@v/v4.111.5.zip#" + loc2.RealPath},
Locations: file.NewLocationSet(),
}},
},
{
name: "github.com/CapORG/CapProject",
version: "v4.111.5",
config: CatalogerConfig{
SearchLocalModCacheLicenses: true,
LocalModCacheDir: filepath.Join(wd, "test-fixtures"), // valid dir but does not find modules
SearchRemoteLicenses: true,
Proxies: []string{server.URL},
},
expected: []pkg.License{{
Value: "MIT",
SPDXExpression: "MIT",
Type: license.Concluded,
URLs: []string{server.URL + "/github.com/CapORG/CapProject/@v/v4.111.5.zip#" + loc2.RealPath},
Locations: file.NewLocationSet(),
}},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {

l := newGoLicenseResolver(
"",
CatalogerConfig{
SearchRemoteLicenses: true,
Proxies: []string{server.URL},
},
)

lics, err := l.getLicenses(context.Background(), licenseScanner, fileresolver.Empty{}, test.name, test.version)
require.NoError(t, err)

require.Len(t, lics, 1)

require.Equal(t, test.expected, lics[0])
l := newGoLicenseResolver("", test.config)
lics := l.getLicenses(context.Background(), licenseScanner, fileresolver.Empty{}, test.name, test.version)
require.EqualValues(t, test.expected, lics)
})
}
}
Expand Down
13 changes: 2 additions & 11 deletions syft/pkg/cataloger/golang/parse_go_binary.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,7 @@ func (c *goBinaryCataloger) buildGoPkgInfo(ctx context.Context, licenseScanner l
continue
}

lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, dep.Path, dep.Version)
if err != nil {
log.Tracef("error getting licenses for golang package: %s %v", dep.Path, err)
}

lics := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, dep.Path, dep.Version)
gover, experiments := getExperimentsFromVersion(mod.GoVersion)
p := c.newGoBinaryPackage(
dep,
Expand Down Expand Up @@ -162,12 +158,7 @@ func missingMainModule(mod *extendedBuildInfo) bool {

func (c *goBinaryCataloger) makeGoMainPackage(ctx context.Context, licenseScanner licenses.Scanner, resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location, reader io.ReadSeekCloser) pkg.Package {
gbs := getBuildSettings(mod.Settings)

lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, mod.Main.Path, mod.Main.Version)
if err != nil {
log.Tracef("error getting licenses for golang package: %s %v", mod.Main.Path, err)
}

lics := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, mod.Main.Path, mod.Main.Version)
gover, experiments := getExperimentsFromVersion(mod.GoVersion)
main := c.newGoBinaryPackage(
&mod.Main,
Expand Down
11 changes: 2 additions & 9 deletions syft/pkg/cataloger/golang/parse_go_mod.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,7 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol
}

for _, m := range f.Require {
lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, m.Mod.Path, m.Mod.Version)
if err != nil {
log.Tracef("error getting licenses for package: %s %v", m.Mod.Path, err)
}

lics := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, m.Mod.Path, m.Mod.Version)
packages[m.Mod.Path] = pkg.Package{
Name: m.Mod.Path,
Version: m.Mod.Version,
Expand All @@ -74,10 +70,7 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol

// remove any old packages and replace with new ones...
for _, m := range f.Replace {
lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, m.New.Path, m.New.Version)
if err != nil {
log.Tracef("error getting licenses for package: %s %v", m.New.Path, err)
}
lics := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, m.New.Path, m.New.Version)

// the old path and new path may be the same, in which case this is a noop,
// but if they're different we need to remove the old package.
Expand Down
Loading