From 3c2d1336f6a1d3aa2e295ac0d4d98ba099bca279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lovro=20Ma=C5=BEgon?= Date: Wed, 30 Oct 2024 16:38:36 +0100 Subject: [PATCH] add support for external types --- paramgen/paramgen/integration_test.go | 4 + paramgen/paramgen/paramgen.go | 74 +++++-- paramgen/paramgen/paramgen_test.go | 6 +- paramgen/paramgen/testdata/complex/global.go | 2 +- paramgen/paramgen/testdata/complex/specs.go | 7 + .../paramgen/testdata/dependencies/go.mod | 16 ++ .../paramgen/testdata/dependencies/go.sum | 4 + .../paramgen/testdata/dependencies/specs.go | 24 +++ .../paramgen/testdata/dependencies/want.go | 189 ++++++++++++++++++ 9 files changed, 304 insertions(+), 22 deletions(-) create mode 100644 paramgen/paramgen/testdata/dependencies/go.mod create mode 100644 paramgen/paramgen/testdata/dependencies/go.sum create mode 100644 paramgen/paramgen/testdata/dependencies/specs.go create mode 100644 paramgen/paramgen/testdata/dependencies/want.go diff --git a/paramgen/paramgen/integration_test.go b/paramgen/paramgen/integration_test.go index 86e3d16..443128b 100644 --- a/paramgen/paramgen/integration_test.go +++ b/paramgen/paramgen/integration_test.go @@ -39,6 +39,10 @@ func TestIntegration(t *testing.T) { havePath: "./testdata/tags", structName: "Config", wantPath: "./testdata/tags/want.go", + }, { + havePath: "./testdata/dependencies", + structName: "Config", + wantPath: "./testdata/dependencies/want.go", }} for _, tc := range testCases { diff --git a/paramgen/paramgen/paramgen.go b/paramgen/paramgen/paramgen.go index 087ffdd..605a32c 100644 --- a/paramgen/paramgen/paramgen.go +++ b/paramgen/paramgen/paramgen.go @@ -21,6 +21,7 @@ import ( "go/ast" "go/parser" "go/token" + "io" "io/fs" "os/exec" "reflect" @@ -115,31 +116,34 @@ func parsePackage(path string) (*ast.Package, error) { filterTests := func(info fs.FileInfo) bool { return !strings.HasSuffix(info.Name(), "_test.go") } - pkgs, err := parser.ParseDir(fset, path, filterTests, parser.ParseComments) + pkgs, err := parser.ParseDir(fset, path, filterTests, parser.ParseComments|parser.SkipObjectResolution) if err != nil { return nil, fmt.Errorf("couldn't parse directory %s: %w", path, err) } - // Make sure they are all in one package. - if len(pkgs) == 0 { - return nil, fmt.Errorf("no source-code package in directory %s", path) - } - // Ignore files with go:build constraint set to "tools" (common pattern in - // Conduit connectors). for pkgName, pkg := range pkgs { + // Ignore files with go:build constraint set to "tools" (common pattern in + // Conduit connectors). maps.DeleteFunc(pkg.Files, func(_ string, f *ast.File) bool { return hasBuildConstraint(f, "tools") }) - if len(pkg.Files) == 0 { + // Remove empty packages or the main package (can't be imported). + if len(pkg.Files) == 0 || pkgName == "main" { delete(pkgs, pkgName) } } - if len(pkgs) > 1 { + + // Make sure there is only 1 package. + switch len(pkgs) { + case 0: + return nil, fmt.Errorf("no source-code package in directory %s", path) + case 1: + for _, pkg := range pkgs { + return pkg, nil + } + panic("unreachable") + default: return nil, fmt.Errorf("multiple packages %v in directory %s", maps.Keys(pkgs), path) } - for _, v := range pkgs { - return v, nil // return first package - } - panic("unreachable") } // hasBuildConstraint is a very naive way to check if a file has a build @@ -428,17 +432,21 @@ func (p *parameterParser) findPackage(importPath string) (*ast.Package, error) { // first cleanup string importPath = strings.Trim(importPath, `"`) - if !strings.HasPrefix(importPath, p.mod.Path) { - // we only allow types declared in the same module - return nil, fmt.Errorf("we do not support parameters from package %v (please use builtin types or time.Duration)", importPath) - } - if pkg, ok := p.imports[importPath]; ok { // it's cached already return pkg, nil } pkgDir := p.mod.Dir + strings.TrimPrefix(importPath, p.mod.Path) + if !strings.HasPrefix(importPath, p.mod.Path) { + // Import path is not part of the module, we need to find the package path + var err error + pkgDir, err = p.packageToPath(importPath) + if err != nil { + return nil, fmt.Errorf("could not get package path for %q: %w", importPath, err) + } + } + pkg, err := parsePackage(pkgDir) if err != nil { return nil, fmt.Errorf("could not parse package dir %q: %w", pkgDir, err) @@ -715,3 +723,33 @@ func (p *parameterParser) parseValidation(str string) (config.Validation, error) return nil, fmt.Errorf("invalid value for tag validate: %s", str) } } + +// packageToPath takes a package import path and returns the path to the directory +// of that package. +func (p *parameterParser) packageToPath(pkg string) (string, error) { + cmd := exec.Command("go", "list", "-f", "{{.Dir}}", pkg) + cmd.Dir = p.mod.Dir + stdout, err := cmd.StdoutPipe() + if err != nil { + return "", fmt.Errorf("error piping stdout of go list command: %w", err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + return "", fmt.Errorf("error piping stderr of go list command: %w", err) + } + if err := cmd.Start(); err != nil { + return "", fmt.Errorf("error starting go list command: %w", err) + } + path, err := io.ReadAll(stdout) + if err != nil { + return "", fmt.Errorf("error reading stdout of go list command: %w", err) + } + errMsg, err := io.ReadAll(stderr) + if err != nil { + return "", fmt.Errorf("error reading stderr of go list command: %w", err) + } + if err := cmd.Wait(); err != nil { + return "", fmt.Errorf("error running command %q (error message: %q): %w", cmd.String(), errMsg, err) + } + return strings.TrimRight(string(path), "\n"), nil +} diff --git a/paramgen/paramgen/paramgen_test.go b/paramgen/paramgen/paramgen_test.go index 233ab14..06d04c6 100644 --- a/paramgen/paramgen/paramgen_test.go +++ b/paramgen/paramgen/paramgen_test.go @@ -23,7 +23,7 @@ import ( "github.com/matryer/is" ) -func TestParseSpecificationSuccess(t *testing.T) { +func TestParseParametersSuccess(t *testing.T) { testCases := []struct { path string name string @@ -170,7 +170,7 @@ func TestParseSpecificationSuccess(t *testing.T) { } } -func TestParseSpecificationFail(t *testing.T) { +func TestParseParametersFail(t *testing.T) { testCases := []struct { path string name string @@ -178,7 +178,7 @@ func TestParseSpecificationFail(t *testing.T) { }{{ path: "./testdata/invalid1", name: "SourceConfig", - wantErr: errors.New("we do not support parameters from package net/http (please use builtin types or time.Duration)"), + wantErr: errors.New("unexpected type: *ast.InterfaceType"), }, { path: "./testdata/invalid2", name: "SourceConfig", diff --git a/paramgen/paramgen/testdata/complex/global.go b/paramgen/paramgen/testdata/complex/global.go index e0c6b9e..f4f6c85 100644 --- a/paramgen/paramgen/testdata/complex/global.go +++ b/paramgen/paramgen/testdata/complex/global.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// go:build ignoreBuildTags +//go:build ignoreBuildTags package example diff --git a/paramgen/paramgen/testdata/complex/specs.go b/paramgen/paramgen/testdata/complex/specs.go index a6aea79..67dfa31 100644 --- a/paramgen/paramgen/testdata/complex/specs.go +++ b/paramgen/paramgen/testdata/complex/specs.go @@ -16,6 +16,13 @@ package example import "time" +// DestinationConfig is ignored in tests, they only operate on SourceConfig. +type DestinationConfig struct { + // GlobalConfig parameters should be nested under "global". This comment + // should be ignored. + Global GlobalConfig `json:"global"` +} + type SourceConfig struct { // GlobalConfig parameters should be nested under "global". This comment // should be ignored. diff --git a/paramgen/paramgen/testdata/dependencies/go.mod b/paramgen/paramgen/testdata/dependencies/go.mod new file mode 100644 index 0000000..a88abea --- /dev/null +++ b/paramgen/paramgen/testdata/dependencies/go.mod @@ -0,0 +1,16 @@ +module example.com/test2 + +go 1.22.4 + +require ( + github.com/conduitio/conduit-commons v0.3.0 + example.com/test v0.0.0 +) + +require ( + github.com/goccy/go-json v0.10.3 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect +) + +replace example.com/test => ../basic \ No newline at end of file diff --git a/paramgen/paramgen/testdata/dependencies/go.sum b/paramgen/paramgen/testdata/dependencies/go.sum new file mode 100644 index 0000000..6342c6f --- /dev/null +++ b/paramgen/paramgen/testdata/dependencies/go.sum @@ -0,0 +1,4 @@ +github.com/conduitio/conduit-commons v0.3.0/go.mod h1:roxZ88dv+fpbEjjTzkdGwwbmcpunSuiD8he43y0lAoo= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/paramgen/paramgen/testdata/dependencies/specs.go b/paramgen/paramgen/testdata/dependencies/specs.go new file mode 100644 index 0000000..c3c16a6 --- /dev/null +++ b/paramgen/paramgen/testdata/dependencies/specs.go @@ -0,0 +1,24 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dependencies + +import ( + example "example.com/test" +) + +// Config is a reusable config struct used in the source and destination +type Config struct { + example.SourceConfig +} diff --git a/paramgen/paramgen/testdata/dependencies/want.go b/paramgen/paramgen/testdata/dependencies/want.go new file mode 100644 index 0000000..f361e37 --- /dev/null +++ b/paramgen/paramgen/testdata/dependencies/want.go @@ -0,0 +1,189 @@ +// Code generated by paramgen. DO NOT EDIT. +// Source: github.com/ConduitIO/conduit-commons/tree/main/paramgen + +package dependencies + +import ( + "github.com/conduitio/conduit-commons/config" +) + +const ( + ConfigFoo = "foo" + ConfigMyBool = "myBool" + ConfigMyByte = "myByte" + ConfigMyDurSlice = "myDurSlice" + ConfigMyDuration = "myDuration" + ConfigMyFloat32 = "myFloat32" + ConfigMyFloat64 = "myFloat64" + ConfigMyFloatSlice = "myFloatSlice" + ConfigMyInt = "myInt" + ConfigMyInt16 = "myInt16" + ConfigMyInt32 = "myInt32" + ConfigMyInt64 = "myInt64" + ConfigMyInt8 = "myInt8" + ConfigMyIntSlice = "myIntSlice" + ConfigMyRune = "myRune" + ConfigMyString = "myString" + ConfigMyStringMap = "myStringMap.*" + ConfigMyStructMapMyInt = "myStructMap.*.myInt" + ConfigMyStructMapMyString = "myStructMap.*.myString" + ConfigMyUint = "myUint" + ConfigMyUint16 = "myUint16" + ConfigMyUint32 = "myUint32" + ConfigMyUint64 = "myUint64" + ConfigMyUint8 = "myUint8" +) + +func (Config) Parameters() map[string]config.Parameter { + return map[string]config.Parameter{ + ConfigFoo: { + Default: "bar", + Description: "MyGlobalString is a required field in the global config with the name\n\"foo\" and default value \"bar\".", + Type: config.ParameterTypeString, + Validations: []config.Validation{ + config.ValidationRequired{}, + }, + }, + ConfigMyBool: { + Default: "", + Description: "", + Type: config.ParameterTypeBool, + Validations: []config.Validation{}, + }, + ConfigMyByte: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + ConfigMyDurSlice: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + ConfigMyDuration: { + Default: "", + Description: "", + Type: config.ParameterTypeDuration, + Validations: []config.Validation{}, + }, + ConfigMyFloat32: { + Default: "", + Description: "", + Type: config.ParameterTypeFloat, + Validations: []config.Validation{}, + }, + ConfigMyFloat64: { + Default: "", + Description: "", + Type: config.ParameterTypeFloat, + Validations: []config.Validation{}, + }, + ConfigMyFloatSlice: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + ConfigMyInt: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{ + config.ValidationLessThan{V: 100}, + config.ValidationGreaterThan{V: 0}, + }, + }, + ConfigMyInt16: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + ConfigMyInt32: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + ConfigMyInt64: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + ConfigMyInt8: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + ConfigMyIntSlice: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + ConfigMyRune: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + ConfigMyString: { + Default: "", + Description: "MyString my string description", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + ConfigMyStringMap: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + ConfigMyStructMapMyInt: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + ConfigMyStructMapMyString: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + ConfigMyUint: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + ConfigMyUint16: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + ConfigMyUint32: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + ConfigMyUint64: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + ConfigMyUint8: { + Default: "", + Description: "", + Type: config.ParameterTypeInt, + Validations: []config.Validation{}, + }, + } +}