v2: list module deps from package or binary

v2/.gitignore

@@ -0,0 +1,11 @@
+# built binary
+go-licenses
+
+# distribution folder
+dist
+
+# MacOS common ignores
+.DS_Store
+
+# Editor
+.vscode

v2/README.md

@@ -0,0 +1,16 @@
+# go-licenses v2
+
+A tool to automate license management workflow for go module project's dependencies and transitive dependencies.
+
+## **THIS IS STILL UNDER DEVELOPMENT**
+
+The v2 package is being developed and currently incomplete, @Bobgy is
+upstreaming changes from his fork in <https://github.com/Bobgy/go-licenses/blob/main/v2>.
+
+Tracking issue where you can find the roadmap and progress:
+<https://github.com/google/go-licenses/issues/70>.
+
+The major changes from v1 are:
+
+* V2 only supports go modules, it can get license URL for modules without a need for you to vendor your dependencies.
+* V2 does not assume each module has a single license, v2 will scan all the files for each module to find licenses.

v2/go.mod

@@ -0,0 +1,13 @@
+module github.com/google/go-licenses/v2
+
+go 1.15
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/hashicorp/go-multierror v1.1.1
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/stretchr/testify v1.4.0
+	golang.org/x/tools v0.1.5
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)

v2/go.sum

@@ -0,0 +1,52 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.4 h1:cVngSRcfgyZCzys3KYOpCFa+4dqX/Oub9tAq00ttGVs=
+golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

v2/gocli/deps_test.go

@@ -0,0 +1,123 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gocli_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/google/go-licenses/v2/gocli"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestListDeps(t *testing.T) {
+	var tests = []struct {
+		workdir    string
+		mainModule string
+		modules    []string
+	}{
+		{
+			workdir:    "../tests/modules/hello01",
+			mainModule: "github.com/google/go-licenses/v2/tests/modules/hello01",
+			modules: []string{
+				"github.com/google/go-licenses/v2/tests/modules/hello01@(devel)",
+			},
+		},
+		{
+			workdir:    "../tests/modules/cli02",
+			mainModule: "github.com/google/go-licenses/v2/tests/modules/cli02",
+			modules: []string{
+				"github.com/google/go-licenses/v2/tests/modules/cli02@(devel)",
+				"github.com/fsnotify/fsnotify@v1.4.9",
+				"github.com/hashicorp/hcl@v1.0.0",
+				"github.com/magiconair/properties@v1.8.5",
+				"github.com/mitchellh/go-homedir@v1.1.0",
+				"github.com/mitchellh/mapstructure@v1.4.1",
+				"github.com/pelletier/go-toml@v1.9.3",
+				"github.com/spf13/afero@v1.6.0",
+				"github.com/spf13/cast@v1.3.1",
+				"github.com/spf13/cobra@v1.1.3",
+				"github.com/spf13/jwalterweatherman@v1.1.0",
+				"github.com/spf13/pflag@v1.0.5",
+				"github.com/spf13/viper@v1.8.0",
+				"github.com/subosito/gotenv@v1.2.0",
+				"golang.org/x/sys@v0.0.0-20210510120138-977fb7262007",
+				"golang.org/x/text@v0.3.5",
+				"gopkg.in/ini.v1@v1.62.0",
+				"gopkg.in/yaml.v2@v2.4.0",
+			},
+		},
+	}
+	originalWorkDir, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, tc := range tests {
+		os.Chdir(filepath.Join(originalWorkDir, tc.workdir))
+		sort.Strings(tc.modules)
+		normalize := func(mods []gocli.Module) []string {
+			res := make([]string, 0, len(mods))
+			for _, module := range mods {
+				ver := module.Version
+				if module.Main && ver == "" {
+					// Main module may not have the version, normalize as develop version.
+					ver = "(devel)"
+				}
+				assert.NotEmpty(t, module.Path)
+				assert.NotEmpty(t, ver)
+				res = append(res, fmt.Sprintf("%s@%s", module.Path, ver))
+			}
+			sort.Strings(res)
+			return res
+		}
+
+		t.Run(fmt.Sprintf("gocli.ExtractBinaryMetadata(%s)", tc.workdir), func(t *testing.T) {
+			tempDir, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer os.Remove(tempDir)
+			// This outputs the built binary as name "main".
+			binaryName := path.Join(tempDir, "main")
+			cmd := exec.Command("go", "build", "-o", binaryName)
+			_, err = cmd.Output()
+			// defer remove before checking error, because the file
+			// may be created even when there's an error.
+			defer os.Remove(binaryName)
+			if err != nil {
+				t.Fatalf("go build: %v", err)
+			}
+			metadata, err := gocli.ExtractBinaryMetadata(binaryName)
+			if err != nil {
+				t.Fatal(err)
+			}
+			assert.Equal(t, tc.modules, normalize(append(metadata.Deps, metadata.Main)))
+		})
+
+		t.Run(fmt.Sprintf("gocli.ListDeps(%s)", tc.workdir), func(t *testing.T) {
+			mods, err := gocli.ListDeps(tc.mainModule)
+			if err != nil {
+				t.Fatalf("gocli.ListDeps: %v", err)
+			}
+			assert.Equal(t, tc.modules, normalize(mods), "gocli.Modules")
+		})
+	}
+}

v2/gocli/go_binary.go

@@ -0,0 +1,133 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gocli
+
+import (
+	"fmt"
+
+	"github.com/google/go-licenses/v2/third_party/go/runtime/debug"
+)
+
+// Module metadata extracted from binary and local go module workspace.
+type BinaryMetadata struct {
+	// The main module used to build the binary.
+	// e.g. github.com//google/go-licenses/v2/tests/modules/cli02
+	Main Module
+	// Detailed metadata of all the module dependencies.
+	// Does not include the main module.
+	Deps []Module
+}
+
+// List dependencies from module metadata in a go binary.
+// Modules with replace directives are returned as the replaced module instead.
+//
+// Prerequisites:
+// * The go binary must be built with go modules without any further modifications.
+// * The command must run with working directory same as to build the analyzed
+// go binary, because we need the exact go modules info used to build it.
+//
+// Here, I am using [1] as a short term solution. It runs [4] go version -m and parses
+// output. This is preferred over [2], because [2] is an alternative implemention
+// for go version -m, and I expect better long term compatibility for go version -m.
+//
+// The parsing command output hack is still unfavorable in the long term. As
+// dicussed in [3], golang community will move go version parsing into an individual
+// module in golang.org/x. We can use that module instead after it is built.
+//
+// References of similar implementations or dicussions:
+// 1. https://github.com/uw-labs/lichen/blob/be9752894a5958f6ba7be9e05dc370b7a73b58db/internal/module/extract.go#L16
+// 2. https://github.com/mitchellh/golicense/blob/8c09a94a11ac73299a72a68a7b41e3a737119f91/module/module.go#L27
+// 3. https://github.com/golang/go/issues/39301
+// 4. https://golang.org/pkg/cmd/go/internal/version/
+func ExtractBinaryMetadata(path string) (*BinaryMetadata, error) {
+	buildInfo, err := listModulesInBinary(path)
+	if err != nil {
+		return nil, err
+	}
+	main, deps, err := joinModulesMetadata(&buildInfo.Main, buildInfo.Deps)
+	if err != nil {
+		return nil, err
+	}
+	return &BinaryMetadata{
+		Main: main,
+		Deps: deps,
+	}, nil
+}
+
+func listModulesInBinary(path string) (buildinfo *debug.BuildInfo, err error) {
+	// TODO(Bobgy): replace with x/mod equivalent from https://github.com/golang/go/issues/39301
+	// when it is available.
+	buildinfo, err = version(path)
+	if err != nil {
+		err = fmt.Errorf("listModulesInGoBinary(path=%q): %w", path, err)
+	}
+	return buildinfo, nil
+}
+
+// joinModulesMetadata inner joins local go modules metadata with module ref
+// extracted from the binary.
+// The local go modules metadata is taken from calling `go list -m -json all`.
+// Only those appeared in refs will be returned.
+// An error is reported when we cannot find go module metadata for some refs,
+// or when there's a version mismatch. These errors usually indicate your current
+// working directory does not match exactly where the go binary is built.
+func joinModulesMetadata(mainRef *debug.Module, refs []*debug.Module) (main Module, deps []Module, err error) {
+	// Note, there was an attempt to use golang.org/x/tools/go/packages for
+	// loading modules instead, but it fails for modules like golang.org/x/sys.
+	// These modules only contains sub-packages, but no source code, so it
+	// throws an error when using packages.Load.
+	// More context: https://github.com/google/go-licenses/pull/71#issuecomment-890342154
+	localModulesDict, err := ListModules()
+	if err != nil {
+		return main, nil, err
+	}
+	find := func(ref *debug.Module) (*Module, error) {
+		if ref == nil {
+			return nil, fmt.Errorf("ref is nil")
+		}
+		mod, ok := localModulesDict[ref.Path]
+		if !ok {
+			return nil, fmt.Errorf("Cannot find %v in current dir's go modules. Are you running this tool from the working dir to build the binary you are analyzing?", ref.Path)
+		}
+		if mod.Dir == "" {
+			return nil, fmt.Errorf("Module %v's local directory is empty. Did you run `go mod download`?", ref.Path)
+		}
+		ver := ref.Version
+		if ver == "(devel)" {
+			// Main module's version will be (devel). We should expect an empty version when listing the module info.
+			ver = ""
+		}
+		if ver != mod.Version {
+			return nil, fmt.Errorf("Found %v@%v in go binary, but %v is downloaded in go modules. Are you running this tool from the working dir to build the binary you are analyzing?", ref.Path, ref.Version, mod.Version)
+		}
+		return &mod, nil
+	}
+	if mainRef != nil {
+		found, err := find(mainRef)
+		if err != nil {
+			return main, nil, err
+		}
+		main = *found
+	}
+
+	for _, ref := range refs {
+		found, err := find(ref)
+		if err != nil {
+			return main, nil, err
+		}
+		deps = append(deps, *found)
+	}
+	return main, deps, nil
+}