add support for external types

paramgen/paramgen/integration_test.go

@@ -39,6 +39,10 @@ func TestIntegration(t *testing.T) {
 		havePath:   "./testdata/tags",
 		structName: "Config",
 		wantPath:   "./testdata/tags/want.go",
+	}, {
+		havePath:   "./testdata/dependencies",
+		structName: "Config",
+		wantPath:   "./testdata/dependencies/want.go",
 	}}
 
 	for _, tc := range testCases {

paramgen/paramgen/paramgen.go

@@ -21,6 +21,7 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/token"
+	"io"
 	"io/fs"
 	"os/exec"
 	"reflect"
@@ -115,31 +116,34 @@ func parsePackage(path string) (*ast.Package, error) {
 	filterTests := func(info fs.FileInfo) bool {
 		return !strings.HasSuffix(info.Name(), "_test.go")
 	}
-	pkgs, err := parser.ParseDir(fset, path, filterTests, parser.ParseComments)
+	pkgs, err := parser.ParseDir(fset, path, filterTests, parser.ParseComments|parser.SkipObjectResolution)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't parse directory %s: %w", path, err)
 	}
-	// Make sure they are all in one package.
-	if len(pkgs) == 0 {
-		return nil, fmt.Errorf("no source-code package in directory %s", path)
-	}
-	// Ignore files with go:build constraint set to "tools" (common pattern in
-	// Conduit connectors).
 	for pkgName, pkg := range pkgs {
+		// Ignore files with go:build constraint set to "tools" (common pattern in
+		// Conduit connectors).
 		maps.DeleteFunc(pkg.Files, func(_ string, f *ast.File) bool {
 			return hasBuildConstraint(f, "tools")
 		})
-		if len(pkg.Files) == 0 {
+		// Remove empty packages or the main package (can't be imported).
+		if len(pkg.Files) == 0 || pkgName == "main" {
 			delete(pkgs, pkgName)
 		}
 	}
-	if len(pkgs) > 1 {
+
+	// Make sure there is only 1 package.
+	switch len(pkgs) {
+	case 0:
+		return nil, fmt.Errorf("no source-code package in directory %s", path)
+	case 1:
+		for _, pkg := range pkgs {
+			return pkg, nil
+		}
+		panic("unreachable")
+	default:
 		return nil, fmt.Errorf("multiple packages %v in directory %s", maps.Keys(pkgs), path)
 	}
-	for _, v := range pkgs {
-		return v, nil // return first package
-	}
-	panic("unreachable")
 }
 
 // hasBuildConstraint is a very naive way to check if a file has a build
@@ -428,17 +432,21 @@ func (p *parameterParser) findPackage(importPath string) (*ast.Package, error) {
 	// first cleanup string
 	importPath = strings.Trim(importPath, `"`)
 
-	if !strings.HasPrefix(importPath, p.mod.Path) {
-		// we only allow types declared in the same module
-		return nil, fmt.Errorf("we do not support parameters from package %v (please use builtin types or time.Duration)", importPath)
-	}
-
 	if pkg, ok := p.imports[importPath]; ok {
 		// it's cached already
 		return pkg, nil
 	}
 
 	pkgDir := p.mod.Dir + strings.TrimPrefix(importPath, p.mod.Path)
+	if !strings.HasPrefix(importPath, p.mod.Path) {
+		// Import path is not part of the module, we need to find the package path
+		var err error
+		pkgDir, err = p.packageToPath(importPath)
+		if err != nil {
+			return nil, fmt.Errorf("could not get package path for %q: %w", importPath, err)
+		}
+	}
+
 	pkg, err := parsePackage(pkgDir)
 	if err != nil {
 		return nil, fmt.Errorf("could not parse package dir %q: %w", pkgDir, err)
@@ -715,3 +723,33 @@ func (p *parameterParser) parseValidation(str string) (config.Validation, error)
 		return nil, fmt.Errorf("invalid value for tag validate: %s", str)
 	}
 }
+
+// packageToPath takes a package import path and returns the path to the directory
+// of that package.
+func (p *parameterParser) packageToPath(pkg string) (string, error) {
+	cmd := exec.Command("go", "list", "-f", "{{.Dir}}", pkg)
+	cmd.Dir = p.mod.Dir
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", fmt.Errorf("error piping stdout of go list command: %w", err)
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return "", fmt.Errorf("error piping stderr of go list command: %w", err)
+	}
+	if err := cmd.Start(); err != nil {
+		return "", fmt.Errorf("error starting go list command: %w", err)
+	}
+	path, err := io.ReadAll(stdout)
+	if err != nil {
+		return "", fmt.Errorf("error reading stdout of go list command: %w", err)
+	}
+	errMsg, err := io.ReadAll(stderr)
+	if err != nil {
+		return "", fmt.Errorf("error reading stderr of go list command: %w", err)
+	}
+	if err := cmd.Wait(); err != nil {
+		return "", fmt.Errorf("error running command %q (error message: %q): %w", cmd.String(), errMsg, err)
+	}
+	return strings.TrimRight(string(path), "\n"), nil
+}

paramgen/paramgen/paramgen_test.go

@@ -23,7 +23,7 @@ import (
 	"github.com/matryer/is"
 )
 
-func TestParseSpecificationSuccess(t *testing.T) {
+func TestParseParametersSuccess(t *testing.T) {
 	testCases := []struct {
 		path string
 		name string
@@ -170,15 +170,15 @@ func TestParseSpecificationSuccess(t *testing.T) {
 	}
 }
 
-func TestParseSpecificationFail(t *testing.T) {
+func TestParseParametersFail(t *testing.T) {
 	testCases := []struct {
 		path    string
 		name    string
 		wantErr error
 	}{{
 		path:    "./testdata/invalid1",
 		name:    "SourceConfig",
-		wantErr: errors.New("we do not support parameters from package net/http (please use builtin types or time.Duration)"),
+		wantErr: errors.New("unexpected type: *ast.InterfaceType"),
 	}, {
 		path:    "./testdata/invalid2",
 		name:    "SourceConfig",

paramgen/paramgen/testdata/complex/global.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// go:build ignoreBuildTags
+//go:build ignoreBuildTags
 
 package example
 

paramgen/paramgen/testdata/complex/specs.go

@@ -16,6 +16,13 @@ package example
 
 import "time"
 
+// DestinationConfig is ignored in tests, they only operate on SourceConfig.
+type DestinationConfig struct {
+	// GlobalConfig parameters should be nested under "global". This comment
+	// should be ignored.
+	Global GlobalConfig `json:"global"`
+}
+
 type SourceConfig struct {
 	// GlobalConfig parameters should be nested under "global". This comment
 	// should be ignored.

paramgen/paramgen/testdata/dependencies/go.mod

@@ -0,0 +1,16 @@
+module example.com/test2
+
+go 1.22.4
+
+require (
+	github.com/conduitio/conduit-commons v0.3.0
+	example.com/test v0.0.0
+)
+
+require (
+	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+)
+
+replace example.com/test => ../basic

paramgen/paramgen/testdata/dependencies/go.sum

@@ -0,0 +1,4 @@
+github.com/conduitio/conduit-commons v0.3.0/go.mod h1:roxZ88dv+fpbEjjTzkdGwwbmcpunSuiD8he43y0lAoo=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=

paramgen/paramgen/testdata/dependencies/specs.go

@@ -0,0 +1,24 @@
+// Copyright © 2023 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dependencies
+
+import (
+	example "example.com/test"
+)
+
+// Config is a reusable config struct used in the source and destination
+type Config struct {
+	example.SourceConfig
+}