Cross-module support

This patch updates the controller-gen package loader to support
nested Go modules. If the syntax for the paths= argument ends with
four dot-characters "/....", then this now indicates to the loader
that any nested Go modules in that path should also be considered
when loading packages.

pkg/loader/loader.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Kubernetes Authors.
+Copyright 2019-2022 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -25,9 +25,12 @@ import (
 	"go/types"
 	"io/ioutil"
 	"os"
+	"path"
+	"path/filepath"
 	"sync"
 
 	"golang.org/x/tools/go/packages"
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 // Much of this is strongly inspired by the contents of go/packages,
@@ -329,7 +332,7 @@ func LoadRoots(roots ...string) ([]*Package, error) {
 //
 // This is generally only useful for use in testing when you need to modify
 // loading settings to load from a fake location.
-func LoadRootsWithConfig(cfg *packages.Config, roots ...string) ([]*Package, error) {
+func LoadRootsWithConfig(cfg *packages.Config, roots ...string) (pkgs []*Package, retErr error) {
 	l := &loader{
 		cfg:      cfg,
 		packages: make(map[*packages.Package]*Package),
@@ -341,13 +344,90 @@ func LoadRootsWithConfig(cfg *packages.Config, roots ...string) ([]*Package, err
 	// put our build flags first so that callers can override them
 	l.cfg.BuildFlags = append([]string{"-tags", "ignore_autogenerated"}, l.cfg.BuildFlags...)
 
-	rawPkgs, err := packages.Load(l.cfg, roots...)
-	if err != nil {
+	// check each root to see if it should be expanded to include nested modules
+	var goModDirs []string
+	findGoModules := func(p string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() && path.Base(p) == "go.mod" {
+			absPathDirOfGoMod, err := filepath.Abs(filepath.Dir(p))
+			if err != nil {
+				return err
+			}
+			goModDirs = append(goModDirs, absPathDirOfGoMod)
+		}
+		return nil
+	}
+	for i, r := range roots {
+		// skip roots whose last path element is not four dot characters
+		if filepath.Base(r) != "...." {
+			continue
+		}
+		// update the root to no longer descend into nested modules
+		roots[i] = r[:len(r)-1]
+		// add any nested modules to the list of Go module directories to
+		// process later
+		if err := filepath.WalkDir(r[:len(r)-4], findGoModules); err != nil {
+			return nil, err
+		}
+	}
+
+	// loadRoots parses the provided file paths and returns their load packages
+	//
+	// uniqueRoots is used to keep track of the discovered packages to be nice
+	// and try and prevent packages from showing up twice when nested module
+	// support is enabled. there is not harm that comes from this per se, but
+	// it makes testing easier when a known number of modules can be asserted
+	uniqueRoots := sets.String{}
+	loadRoots := func(roots ...string) error {
+		rawPkgs, err := packages.Load(l.cfg, roots...)
+		if err != nil {
+			return err
+		}
+		for _, rawPkg := range rawPkgs {
+			pkg := l.packageFor(rawPkg)
+			if !uniqueRoots.Has(pkg.ID) {
+				l.Roots = append(l.Roots, pkg)
+				uniqueRoots.Insert(pkg.ID)
+			}
+		}
+		return nil
+	}
+
+	// load the packages from the main module
+	if err := loadRoots(roots...); err != nil {
 		return nil, err
 	}
 
-	for _, rawPkg := range rawPkgs {
-		l.Roots = append(l.Roots, l.packageFor(rawPkg))
+	// return early when no nested Go modules were detected
+	if len(goModDirs) == 0 {
+		return l.Roots, nil
+	}
+
+	// ensure the working directory is updated back to its original location
+	// as we switch into the directory of each Go module to process them to
+	// accommodate the package loader
+	workingDir, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		retErr = os.Chdir(workingDir)
+	}()
+
+	// load the packages from the nested modules
+	for _, p := range goModDirs {
+		// change the working directory to the root of the nested module
+		// so the package loader does not complain about processing a
+		// directory that is not a member of the root module
+		if err := os.Chdir(p); err != nil {
+			return nil, err
+		}
+		// load the packages from the nested module
+		if err := loadRoots("./..."); err != nil {
+			return nil, err
+		}
 	}
 
 	return l.Roots, nil

pkg/loader/loader_suite_test.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package loader_test
+
+import (
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	originalWorkingDir string
+)
+
+func TestLoader(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Loader Patching Suite")
+}
+
+var _ = BeforeSuite(func() {
+	cwd, err := os.Getwd()
+	Expect(err).ToNot(HaveOccurred())
+	originalWorkingDir = cwd
+	Expect(os.Chdir("./testmods")).To(Succeed())
+})
+
+var _ = AfterSuite(func() {
+	Expect(os.Chdir(originalWorkingDir)).To(Succeed())
+})

pkg/loader/loader_test.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package loader_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"sigs.k8s.io/controller-tools/pkg/loader"
+)
+
+var _ = Describe("Loader parsing root module", func() {
+	const testmodsPkg = "sigs.k8s.io/controller-tools/pkg/loader/testmods"
+
+	var indexOfPackage = func(pkgID string, pkgs []*loader.Package) int {
+		for i := range pkgs {
+			if pkgs[i].ID == pkgID {
+				return i
+			}
+		}
+		return -1
+	}
+
+	var assertPkgExists = func(pkgID string, pkgs []*loader.Package) {
+		ExpectWithOffset(1, indexOfPackage(pkgID, pkgs)).Should(BeNumerically(">", -1))
+	}
+
+	Context("with ./... syntax", func() {
+		It("should fail to load nested modules", func() {
+			pkgs, err := loader.LoadRoots("./...")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pkgs).To(HaveLen(4))
+			assertPkgExists(testmodsPkg, pkgs)
+			assertPkgExists(testmodsPkg+"/childDirA", pkgs)
+			assertPkgExists(testmodsPkg+"/childDirA/grandchildDirA", pkgs)
+			assertPkgExists(testmodsPkg+"/childDirA/grandchildDirB", pkgs)
+		})
+	})
+	Context("with ./.... syntax that detects nested modules", func() {
+		It("should load nested modules", func() {
+			pkgs, err := loader.LoadRoots("./....")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pkgs).To(HaveLen(6))
+			assertPkgExists(testmodsPkg, pkgs)
+			assertPkgExists(testmodsPkg+"/childDirA", pkgs)
+			assertPkgExists(testmodsPkg+"/childDirA/grandchildDirA", pkgs)
+			assertPkgExists(testmodsPkg+"/childDirA/grandchildDirB", pkgs)
+			assertPkgExists(testmodsPkg+"/childDirA/grandchildDirC", pkgs)
+			assertPkgExists(testmodsPkg+"/childDirB", pkgs)
+		})
+	})
+})

pkg/loader/testmods/childDirA/dummy.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dummy

pkg/loader/testmods/childDirA/grandchildDirA/dummy.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dummy

pkg/loader/testmods/childDirA/grandchildDirB/dummy.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dummy

pkg/loader/testmods/childDirA/grandchildDirC/dummy.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dummy

pkg/loader/testmods/childDirA/grandchildDirC/go.mod

@@ -0,0 +1,3 @@
+module sigs.k8s.io/controller-tools/pkg/loader/testmods/childDirA/grandchildDirC
+
+go 1.17

pkg/loader/testmods/childDirB/dummy.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dummy

pkg/loader/testmods/childDirB/go.mod

@@ -0,0 +1,3 @@
+module sigs.k8s.io/controller-tools/pkg/loader/testmods/childDirB
+
+go 1.17

pkg/loader/testmods/dummy.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dummy

pkg/loader/testmods/go.mod

@@ -0,0 +1,3 @@
+module sigs.k8s.io/controller-tools/pkg/loader/testmods
+
+go 1.17