Skip to content

Commit

Permalink
feat(go): construct dependencies of go.mod main module in the parser (
Browse files Browse the repository at this point in the history 
#7977)

Signed-off-by: knqyf263 <knqyf263@gmail.com>
Co-authored-by: knqyf263 <knqyf263@gmail.com>
  • Loading branch information
@DmitriyLewen @knqyf263
DmitriyLewen and knqyf263 authored Nov 22, 2024
1 parent bcdc0bb commit 5448ba2
Show file tree
Hide file tree
Showing 7 changed files with 278 additions and 68 deletions.
43 changes: 31 additions & 12 deletions pkg/dependency/parser/golang/mod/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"io"
"regexp"
"sort"
"strconv"
"strings"

Expand Down Expand Up @@ -101,17 +102,6 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc
}
}

// Main module
if m := modFileParsed.Module; m != nil {
pkgs[m.Mod.Path] = ftypes.Package{
ID: packageID(m.Mod.Path, m.Mod.Version),
Name: m.Mod.Path,
Version: m.Mod.Version,
ExternalReferences: p.GetExternalRefs(m.Mod.Path),
Relationship: ftypes.RelationshipRoot,
}
}

// Required modules
for _, require := range modFileParsed.Require {
// Skip indirect dependencies less than Go 1.17
Expand Down Expand Up @@ -163,7 +153,36 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc
}
}

return lo.Values(pkgs), nil, nil
var deps ftypes.Dependencies
// Main module
if m := modFileParsed.Module; m != nil {
root := ftypes.Package{
ID: packageID(m.Mod.Path, m.Mod.Version),
Name: m.Mod.Path,
Version: m.Mod.Version,
ExternalReferences: p.GetExternalRefs(m.Mod.Path),
Relationship: ftypes.RelationshipRoot,
}

// Store child dependencies for the root package (main module).
// We will build a dependency graph for Direct/Indirect in `fanal` using additional files.
dependsOn := lo.FilterMap(lo.Values(pkgs), func(pkg ftypes.Package, _ int) (string, bool) {
return pkg.ID, pkg.Relationship == ftypes.RelationshipDirect
})

sort.Strings(dependsOn)
deps = append(deps, ftypes.Dependency{
ID: root.ID,
DependsOn: dependsOn,
})

pkgs[root.Name] = root
}

pkgSlice := lo.Values(pkgs)
sort.Sort(ftypes.Packages(pkgSlice))

return pkgSlice, deps, nil
}

// lessThan checks if the Go version is less than `<majorVer>.<minorVer>`
Expand Down
105 changes: 57 additions & 48 deletions pkg/dependency/parser/golang/mod/parse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package mod

import (
"os"
"sort"
"testing"

"github.com/stretchr/testify/assert"
Expand All @@ -18,74 +17,86 @@ func TestParse(t *testing.T) {
file string
replace bool
useMinVersion bool
want []ftypes.Package
wantPkgs []ftypes.Package
wantDeps []ftypes.Dependency
}{
{
name: "normal with stdlib",
file: "testdata/normal/go.mod",
replace: true,
useMinVersion: true,
want: GoModNormal,
wantPkgs: GoModNormal,
wantDeps: GoModNormalDeps,
},
{
name: "normal",
file: "testdata/normal/go.mod",
replace: true,
want: GoModNormalWithoutStdlib,
name: "normal",
file: "testdata/normal/go.mod",
replace: true,
wantPkgs: GoModNormalWithoutStdlib,
wantDeps: GoModNormalWithoutStdlibDeps,
},
{
name: "without go version",
file: "testdata/no-go-version/gomod",
replace: true,
want: GoModNoGoVersion,
name: "without go version",
file: "testdata/no-go-version/gomod",
replace: true,
wantPkgs: GoModNoGoVersion,
wantDeps: defaultGoDepParserDeps,
},
{
name: "replace",
file: "testdata/replaced/go.mod",
replace: true,
want: GoModReplaced,
name: "replace",
file: "testdata/replaced/go.mod",
replace: true,
wantPkgs: GoModReplaced,
wantDeps: GoModReplacedDeps,
},
{
name: "no replace",
file: "testdata/replaced/go.mod",
replace: false,
want: GoModUnreplaced,
name: "no replace",
file: "testdata/replaced/go.mod",
replace: false,
wantPkgs: GoModUnreplaced,
wantDeps: GoModUnreplacedDeps,
},
{
name: "replace with version",
file: "testdata/replaced-with-version/go.mod",
replace: true,
want: GoModReplacedWithVersion,
name: "replace with version",
file: "testdata/replaced-with-version/go.mod",
replace: true,
wantPkgs: GoModReplacedWithVersion,
wantDeps: GoModReplacedWithVersionDeps,
},
{
name: "replaced with version mismatch",
file: "testdata/replaced-with-version-mismatch/go.mod",
replace: true,
want: GoModReplacedWithVersionMismatch,
name: "replaced with version mismatch",
file: "testdata/replaced-with-version-mismatch/go.mod",
replace: true,
wantPkgs: GoModReplacedWithVersionMismatch,
wantDeps: defaultGoDepParserDeps,
},
{
name: "replaced with local path",
file: "testdata/replaced-with-local-path/go.mod",
replace: true,
want: GoModReplacedWithLocalPath,
name: "replaced with local path",
file: "testdata/replaced-with-local-path/go.mod",
replace: true,
wantPkgs: GoModReplacedWithLocalPath,
wantDeps: defaultGoDepParserDeps,
},
{
name: "replaced with local path and version",
file: "testdata/replaced-with-local-path-and-version/go.mod",
replace: true,
want: GoModReplacedWithLocalPathAndVersion,
name: "replaced with local path and version",
file: "testdata/replaced-with-local-path-and-version/go.mod",
replace: true,
wantPkgs: GoModReplacedWithLocalPathAndVersion,
wantDeps: defaultGoDepParserDeps,
},
{
name: "replaced with local path and version, mismatch",
file: "testdata/replaced-with-local-path-and-version-mismatch/go.mod",
replace: true,
want: GoModReplacedWithLocalPathAndVersionMismatch,
name: "replaced with local path and version, mismatch",
file: "testdata/replaced-with-local-path-and-version-mismatch/go.mod",
replace: true,
wantPkgs: GoModReplacedWithLocalPathAndVersionMismatch,
wantDeps: defaultGoDepParserDeps,
},
{
name: "go 1.16",
file: "testdata/go116/go.mod",
replace: true,
want: GoMod116,
name: "go 1.16",
file: "testdata/go116/go.mod",
replace: true,
wantPkgs: GoMod116,
wantDeps: defaultGoDepParserDeps,
},
}

Expand All @@ -94,13 +105,11 @@ func TestParse(t *testing.T) {
f, err := os.Open(tt.file)
require.NoError(t, err)

got, _, err := NewParser(tt.replace, tt.useMinVersion).Parse(f)
gotPkgs, gotDeps, err := NewParser(tt.replace, tt.useMinVersion).Parse(f)
require.NoError(t, err)

sort.Sort(ftypes.Packages(got))
sort.Sort(ftypes.Packages(tt.want))

assert.Equal(t, tt.want, got)
assert.Equal(t, tt.wantPkgs, gotPkgs)
assert.Equal(t, tt.wantDeps, gotDeps)
})
}
}
Expand Down
66 changes: 60 additions & 6 deletions pkg/dependency/parser/golang/mod/parse_testcase.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,6 @@ var (
},
},
},
{
ID: "stdlib@v1.22.5",
Name: "stdlib",
Version: "v1.22.5",
Relationship: ftypes.RelationshipDirect,
},
{
ID: "github.com/aquasecurity/go-version@v0.0.0-20240603093900-cf8a8d29271d",
Name: "github.com/aquasecurity/go-version",
Expand All @@ -38,6 +32,12 @@ var (
},
},
},
{
ID: "stdlib@v1.22.5",
Name: "stdlib",
Version: "v1.22.5",
Relationship: ftypes.RelationshipDirect,
},
{
ID: "github.com/davecgh/go-spew@v1.1.2-0.20180830191138-d8f796af33cc",
Name: "github.com/davecgh/go-spew",
Expand Down Expand Up @@ -82,10 +82,29 @@ var (
},
}

GoModNormalDeps = ftypes.Dependencies{
{
ID: "github.com/org/repo",
DependsOn: []string{
"github.com/aquasecurity/go-version@v0.0.0-20240603093900-cf8a8d29271d",
"stdlib@v1.22.5",
},
},
}

GoModNormalWithoutStdlib = slices.DeleteFunc(slices.Clone(GoModNormal), func(f ftypes.Package) bool {
return f.Name == "stdlib"
})

GoModNormalWithoutStdlibDeps = ftypes.Dependencies{
{
ID: "github.com/org/repo",
DependsOn: []string{
"github.com/aquasecurity/go-version@v0.0.0-20240603093900-cf8a8d29271d",
},
},
}

// execute go mod tidy in replaced folder
GoModReplaced = []ftypes.Package{
{
Expand Down Expand Up @@ -118,6 +137,14 @@ var (
Relationship: ftypes.RelationshipIndirect,
},
}
GoModReplacedDeps = ftypes.Dependencies{
{
ID: "github.com/org/repo",
DependsOn: []string{
"github.com/aquasecurity/go-dep-parser@v0.0.0-20220406074731-71021a481237",
},
},
}

// execute go mod tidy in replaced folder
GoModUnreplaced = []ftypes.Package{
Expand Down Expand Up @@ -152,6 +179,15 @@ var (
},
}

GoModUnreplacedDeps = ftypes.Dependencies{
{
ID: "github.com/org/repo",
DependsOn: []string{
"github.com/aquasecurity/go-dep-parser@v0.0.0-20211110174639-8257534ffed3",
},
},
}

// execute go mod tidy in replaced-with-version folder
GoModReplacedWithVersion = []ftypes.Package{
{
Expand Down Expand Up @@ -185,6 +221,15 @@ var (
},
}

GoModReplacedWithVersionDeps = ftypes.Dependencies{
{
ID: "github.com/org/repo",
DependsOn: []string{
"github.com/aquasecurity/go-dep-parser@v0.0.0-20220406074731-71021a481237",
},
},
}

// execute go mod tidy in replaced-with-version-mismatch folder
GoModReplacedWithVersionMismatch = []ftypes.Package{
{
Expand Down Expand Up @@ -230,6 +275,15 @@ var (
},
}

defaultGoDepParserDeps = ftypes.Dependencies{
{
ID: "github.com/org/repo",
DependsOn: []string{
"github.com/aquasecurity/go-dep-parser@v0.0.0-20211224170007-df43bca6b6ff",
},
},
}

// execute go mod tidy in replaced-with-local-path folder
GoModReplacedWithLocalPath = []ftypes.Package{
{
Expand Down
37 changes: 37 additions & 0 deletions pkg/fanal/analyzer/language/golang/mod/mod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ func (a *gomodAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys
a.logger.Warn("Unable to collect additional info", log.Err(err))
}

// Add orphan indirect dependencies under the main module
a.addOrphanIndirectDepsUnderRoot(apps)

return &analyzer.AnalysisResult{
Applications: apps,
}, nil
Expand Down Expand Up @@ -212,6 +215,40 @@ func (a *gomodAnalyzer) collectDeps(modDir, pkgID string) (types.Dependency, err
}, nil
}

// addOrphanIndirectDepsUnderRoot handles indirect dependencies that have no identifiable parent packages in the dependency tree.
// This situation can occur when:
// - $GOPATH/pkg directory doesn't exist
// - Module cache is incomplete
// - etc.
//
// In such cases, indirect packages become "orphaned" - they exist in the dependency list
// but have no connection to the dependency tree. This function resolves this issue by:
// 1. Finding the root (main) module
// 2. Identifying all indirect dependencies that have no parent packages
// 3. Adding these orphaned indirect dependencies under the main module
//
// This ensures that all packages remain visible in the dependency tree, even when the complete
// dependency chain cannot be determined.
func (a *gomodAnalyzer) addOrphanIndirectDepsUnderRoot(apps []types.Application) {
for _, app := range apps {
// Find the main module
_, rootIdx, found := lo.FindIndexOf(app.Packages, func(pkg types.Package) bool {
return pkg.Relationship == types.RelationshipRoot
})
if !found {
continue
}

// Collect all orphan indirect dependencies that are unable to identify parents
parents := app.Packages.ParentDeps()
orphanDeps := lo.FilterMap(app.Packages, func(pkg types.Package, _ int) (string, bool) {
return pkg.ID, pkg.Relationship == types.RelationshipIndirect && len(parents[pkg.ID]) == 0
})
// Add orphan indirect dependencies under the main module
app.Packages[rootIdx].DependsOn = append(app.Packages[rootIdx].DependsOn, orphanDeps...)
}
}

func parse(fsys fs.FS, path string, parser language.Parser) (*types.Application, error) {
f, err := fsys.Open(path)
if err != nil {
Expand Down
Loading

0 comments on commit 5448ba2

Please sign in to comment.