From 89516169d7fc37474d8448d3293858107453a4f9 Mon Sep 17 00:00:00 2001 From: Michael Pleshakov Date: Fri, 23 Feb 2024 18:00:58 -0500 Subject: [PATCH] Add code and scheme generation Problem: - Ensure the Attributes method of Exprotable interface are generated automatically for the telemetry data types. - Ensure the avro scheme (.avdl) is generated automatically for the telemetry data types. Solution: - Add generator tool in cmd/generator that generates code (Attributes method) and the scheme. - Generator can be used in //go:generate annotations in go source files. - Generator has "generator" build tag so that it is not included into the telemetry library by default. - Generator uses golang.org/x/tools/go/packages to parse source files. Testing: - Unit tests of parsing. - Unit tests of code and scheme generation - ensuring non-zero output. - Unit tests of generated code in cmd/generator/tests package - Manual validation of the generated scheme (cmd/generator/tests/data.avdl) using Apache Avro IDL Scheme Support IntelliJ plugin. CLOSES - https://github.com/nginxinc/telemetry-exporter/issues/18 --- .gitignore | 4 +- .golangci.yml | 2 + Makefile | 8 + cmd/generator/code.go | 110 +++++ cmd/generator/code_test.go | 42 ++ cmd/generator/main.go | 129 +++++ cmd/generator/parser.go | 452 ++++++++++++++++++ cmd/generator/parser_test.go | 369 ++++++++++++++ cmd/generator/scheme.go | 116 +++++ cmd/generator/scheme_test.go | 44 ++ cmd/generator/tests/data.avdl | 61 +++ cmd/generator/tests/data.go | 35 ++ .../tests/data_attributes_generated.go | 29 ++ cmd/generator/tests/data_test.go | 89 ++++ cmd/generator/tests/subtests/data2.go | 28 ++ .../subtests/data2_attributes_generated.go | 28 ++ go.mod | 2 +- 17 files changed, 1545 insertions(+), 3 deletions(-) create mode 100644 cmd/generator/code.go create mode 100644 cmd/generator/code_test.go create mode 100644 cmd/generator/main.go create mode 100644 cmd/generator/parser.go create mode 100644 cmd/generator/parser_test.go create mode 100644 cmd/generator/scheme.go create mode 100644 cmd/generator/scheme_test.go create mode 100644 cmd/generator/tests/data.avdl create mode 100644 cmd/generator/tests/data.go create mode 100644 cmd/generator/tests/data_attributes_generated.go create mode 100644 cmd/generator/tests/data_test.go create mode 100644 cmd/generator/tests/subtests/data2.go create mode 100644 cmd/generator/tests/subtests/data2_attributes_generated.go diff --git a/.gitignore b/.gitignore index 5da5d5c..944cb89 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,8 @@ *.dylib # Output of the go coverage tool -cmd-cover.out -cmd-cover.html +*cover.out +*cover.html # Vim *.swp diff --git a/.golangci.yml b/.golangci.yml index c09f67d..0433da7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -66,3 +66,5 @@ issues: max-same-issues: 0 run: timeout: 3m + build-tags: + - generator diff --git a/Makefile b/Makefile index 66bae1b..c98a53f 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ help: Makefile ## Display this help unit-test: go test ./... -race -coverprofile cmd-cover.out go tool cover -html=cmd-cover.out -o cmd-cover.html + go test -tags generator ./cmd/generator/... -race -coverprofile generator-cmd-cover.out + go tool cover -html=generator-cmd-cover.out -o generator-cmd-cover.html .PHONY: clean-go-cache clean-go-cache: ## Clean go cache @@ -36,3 +38,9 @@ dev-all: deps fmt vet lint unit-test ## Run all the development checks .PHONY: generate generate: ## Run go generate go generate ./... + go generate -tags generator ./cmd/generator/... + +.PHONY: generator-tests +generator-tests: ## Regenerate the generator generated files and run generator unit tests + go generate -tags generator ./cmd/generator/... # ensure the generated files generated by the generator are up to date + go test -tags generator ./cmd/generator/... -race -coverprofile cmd-cover.out diff --git a/cmd/generator/code.go b/cmd/generator/code.go new file mode 100644 index 0000000..d55a47a --- /dev/null +++ b/cmd/generator/code.go @@ -0,0 +1,110 @@ +//go:build generator + +package main + +import ( + "fmt" + "go/types" + "io" + "text/template" +) + +const codeTemplate = `{{ if .BuildTags }}//go:build {{ .BuildTags }}{{ end }} +package {{ .PackageName }} +/* +This is a generated file. DO NOT EDIT. +*/ + +import ( + "go.opentelemetry.io/otel/attribute" + "github.com/nginxinc/telemetry-exporter/pkg/telemetry" +) + +func (d *{{ .StructName }}) Attributes() []attribute.KeyValue { + var attrs []attribute.KeyValue + + {{ range .Fields -}} + attrs = append(attrs, {{ .AttributesSource }}) + {{ end }} + + return attrs +} + +var _ telemetry.Exportable = (*{{ .StructName }})(nil) +` + +type codeGen struct { + PackageName string + StructName string + BuildTags string + Fields []codeField +} + +type codeField struct { + AttributesSource string +} + +func getAttributeType(kind types.BasicKind) string { + switch kind { + case types.Int64: + return "Int64" + case types.Float64: + return "Float64" + case types.String: + return "String" + case types.Bool: + return "Bool" + default: + panic(fmt.Sprintf("unexpected kind %v", kind)) + } +} + +type codeGenConfig struct { + packageName string + typeName string + buildTags string + fields []field +} + +func generateCode(writer io.Writer, cfg codeGenConfig) error { + codeFields := make([]codeField, 0, len(cfg.fields)) + + for _, f := range cfg.fields { + var cf codeField + + if f.embeddedStruct { + cf = codeField{ + AttributesSource: fmt.Sprintf(`d.%s.Attributes()...`, f.name), + } + } else if f.slice { + cf = codeField{ + AttributesSource: fmt.Sprintf(`attribute.%sSlice("%s", d.%s)`, getAttributeType(f.fieldType), f.name, f.name), + } + } else { + cf = codeField{ + AttributesSource: fmt.Sprintf(`attribute.%s("%s", d.%s)`, getAttributeType(f.fieldType), f.name, f.name), + } + } + + codeFields = append(codeFields, cf) + } + + cg := codeGen{ + PackageName: cfg.packageName, + StructName: cfg.typeName, + Fields: codeFields, + BuildTags: cfg.buildTags, + } + + funcMap := template.FuncMap{ + "getAttributeType": getAttributeType, + } + + tmpl := template.Must(template.New("scheme").Funcs(funcMap).Parse(codeTemplate)) + + if err := tmpl.Execute(writer, cg); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + + return nil +} diff --git a/cmd/generator/code_test.go b/cmd/generator/code_test.go new file mode 100644 index 0000000..f917ae4 --- /dev/null +++ b/cmd/generator/code_test.go @@ -0,0 +1,42 @@ +//go:build generator + +package main + +import ( + "bytes" + "testing" + + . "github.com/onsi/gomega" + + "github.com/nginxinc/telemetry-exporter/cmd/generator/tests" +) + +func TestGenerateCode(t *testing.T) { + g := NewGomegaWithT(t) + + cfg := parsingConfig{ + pkgName: "tests", + typeName: "Data", + loadPattern: "github.com/nginxinc/telemetry-exporter/cmd/generator/tests", + buildFlags: []string{"-tags=generator"}, + } + + _ = tests.Data{} // depends on the type being defined + + fields, err := parse(cfg) + + g.Expect(err).ToNot(HaveOccurred()) + + var buf bytes.Buffer + + codeCfg := codeGenConfig{ + packageName: "tests", + typeName: "Data", + fields: fields, + } + + err = generateCode(&buf, codeCfg) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(buf.Bytes()).ToNot(BeEmpty()) +} diff --git a/cmd/generator/main.go b/cmd/generator/main.go new file mode 100644 index 0000000..ab60359 --- /dev/null +++ b/cmd/generator/main.go @@ -0,0 +1,129 @@ +//go:build generator + +package main + +import ( + "flag" + "fmt" + "os" + "strings" +) + +var ( + code = flag.Bool("code", true, "Generate code") + buildTags = flag.String("build-tags", "", "Comma separated list of build tags expected in the source files and that will be added to the generated code") //nolint:lll + scheme = flag.Bool("scheme", false, "Generate Avro scheme") + schemeNamespace = flag.String("scheme-namespace", "gateway.nginx.org", "Scheme namespace; required when -scheme is set") //nolint:lll + schemeProtocol = flag.String("scheme-protocol", "", "Scheme protocol; required when -scheme is set") + schemeDataFabricDataType = flag.String("scheme-df-datatype", "", "Scheme data fabric data type; required when -scheme is set") //nolint:lll + typeName = flag.String("type", "", "Type to generate; required") +) + +func exitWithError(err error) { + fmt.Fprintln(os.Stderr, "error: "+err.Error()) + os.Exit(1) +} + +func existWithUsage() { + flag.Usage() + os.Exit(1) +} + +func validateFlags() { + if *typeName == "" { + existWithUsage() + } + + if *scheme { + if *schemeNamespace == "" { + existWithUsage() + } + if *schemeProtocol == "" { + existWithUsage() + } + if *schemeDataFabricDataType == "" { + existWithUsage() + } + } +} + +func main() { + flag.Parse() + + validateFlags() + + pkgName := os.Getenv("GOPACKAGE") + if pkgName == "" { + exitWithError(fmt.Errorf("GOPACKAGE is not set")) + } + + var buildFlags []string + if *buildTags != "" { + buildFlags = []string{"-tags=" + *buildTags} + } + + cfg := parsingConfig{ + pkgName: pkgName, + typeName: *typeName, + buildFlags: buildFlags, + } + + fields, err := parse(cfg) + if err != nil { + exitWithError(fmt.Errorf("failed to parse struct: %w", err)) + } + + fmt.Printf("Successfully parsed struct %s\n", *typeName) + + if *code { + fmt.Println("Generating code") + + fileName := fmt.Sprintf("%s_attributes_generated.go", strings.ToLower(*typeName)) + + file, err := os.Create(fileName) + if err != nil { + exitWithError(fmt.Errorf("failed to create file: %w", err)) + } + defer file.Close() + + var codeGenBuildTags string + if *buildTags != "" { + codeGenBuildTags = strings.ReplaceAll(*buildTags, ",", " && ") + } + + codeCfg := codeGenConfig{ + packageName: pkgName, + typeName: *typeName, + fields: fields, + buildTags: codeGenBuildTags, + } + + if err := generateCode(file, codeCfg); err != nil { + exitWithError(fmt.Errorf("failed to generate code: %w", err)) + } + } + + if *scheme { + fmt.Println("Generating scheme") + + fileName := fmt.Sprintf("%s.avdl", strings.ToLower(*typeName)) + + file, err := os.Create(fileName) + if err != nil { + exitWithError(fmt.Errorf("failed to create file: %w", err)) + } + defer file.Close() + + schemeCfg := schemeGenConfig{ + namespace: *schemeNamespace, + protocol: *schemeProtocol, + dataFabricDataType: *schemeDataFabricDataType, + record: *typeName, + fields: fields, + } + + if err := generateScheme(file, schemeCfg); err != nil { + exitWithError(fmt.Errorf("failed to generate scheme: %w", err)) + } + } +} diff --git a/cmd/generator/parser.go b/cmd/generator/parser.go new file mode 100644 index 0000000..7424bc2 --- /dev/null +++ b/cmd/generator/parser.go @@ -0,0 +1,452 @@ +//go:build generator + +package main + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "sort" + "strings" + + "golang.org/x/tools/go/packages" +) + +// docStringFieldsProvider parses the code and finds doc string comments for fields of the structs. +type docStringFieldsProvider struct { + packages map[string]struct{} + docStrings map[string]string + buildFlags []string + loadTests bool +} + +// newDocStringFieldsProvider creates a new docStringFieldsProvider. +// loadTests specifies whether the parser will load test files (e.g. *_test.go). +// buildFlags are go build flags (e.g. -tags=foo). +func newDocStringFieldsProvider(loadTests bool, buildFlags []string) *docStringFieldsProvider { + return &docStringFieldsProvider{ + loadTests: loadTests, + buildFlags: buildFlags, + packages: make(map[string]struct{}), + docStrings: make(map[string]string), + } +} + +func parseFullTypeName(fullTypeName string) (pkgName string, typeName string) { + idx := strings.LastIndex(fullTypeName, ".") + if idx == -1 { + panic(fmt.Sprintf("invalid full type name: %s", fullTypeName)) + } + + return fullTypeName[:idx], fullTypeName[idx+1:] +} + +func getDocStringKey(pkgName string, typeName string, fieldName string) string { + return fmt.Sprintf("%s.%s.%s", pkgName, typeName, fieldName) +} + +// getDocString returns the doc string comment for the field of the struct. +// fullTypeName is the full type name of the struct +// (e.g. "github.com/nginxinc/nginx-gateway-fabric/pkg/mypackage.MyStruct"). +func (p *docStringFieldsProvider) getDocString(fullTypeName string, fieldName string) (string, error) { + pkgName, typeName := parseFullTypeName(fullTypeName) + + _, exists := p.packages[pkgName] + if !exists { + if err := p.parseDocStringsFromPackage(pkgName); err != nil { + return "", fmt.Errorf("failed to load struct comments from package %s: %w", pkgName, err) + } + } + + doc, exists := p.docStrings[getDocStringKey(pkgName, typeName, fieldName)] + if !exists { + return "", fmt.Errorf("doc string not found") + } + + trimmedComment := strings.TrimSpace(doc) + if trimmedComment == "" { + return "", fmt.Errorf("trimmed doc string is empty") + } + + return trimmedComment, nil +} + +func (p *docStringFieldsProvider) parseDocStringsFromPackage(pkgName string) error { + mode := packages.NeedName | packages.NeedSyntax | packages.NeedTypes + + cfg := packages.Config{ + Mode: mode, + Fset: token.NewFileSet(), + Tests: p.loadTests, + BuildFlags: p.buildFlags, + } + + pkgs, err := packages.Load(&cfg, pkgName) + if err != nil { + return fmt.Errorf("failed to load package: %w", err) + } + + var loadedPkg *packages.Package + + for _, pkg := range pkgs { + if p.loadTests && !strings.HasSuffix(pkg.ID, ".test]") { + continue + } + + if pkgName == pkg.PkgPath { + loadedPkg = pkg + break + } + } + + if loadedPkg == nil { + return fmt.Errorf("package %s not found", pkgName) + } + + p.packages[pkgName] = struct{}{} + + // for each struct in the package, + // save the doc string comments for the fields of the struct + for _, fileAst := range loadedPkg.Syntax { + ast.Inspect(fileAst, func(n ast.Node) bool { + structTypeSpec, ok := n.(*ast.TypeSpec) + if !ok { + return true + } + + structType, ok := structTypeSpec.Type.(*ast.StructType) + if !ok { + return true + } + + for _, f := range structType.Fields.List { + for _, name := range f.Names { + comment := f.Doc.Text() + if comment == "" { + continue + } + + p.docStrings[getDocStringKey(loadedPkg.PkgPath, structTypeSpec.Name.String(), name.Name)] = comment + } + } + return true + }) + } + + return nil +} + +type parsingError struct { + typeName string + fieldName string + msg string +} + +func (e parsingError) Error() string { + return fmt.Sprintf("type %s: field %s: %s", e.typeName, e.fieldName, e.msg) +} + +// parsingConfig is a configuration for the parser. +type parsingConfig struct { + // pkgName is the name of the package where the struct is located. + pkgName string + // typeName is the name of the struct. + typeName string + // loadPattern is the pattern to load the package. + // For example, "github.com/nginxinc/nginx-gateway-fabric/pkg/mypackage" or "." + // That path in the pattern are relative to the current working directory. + loadPattern string + // buildFlags are go build flags (e.g. -tags=foo). + buildFlags []string + // loadTests specifies whether the parser will load test files (e.g. *_test.go). + loadTests bool +} + +// field represents a field of a struct. +// the field is either a basic type, a slice of basic type or an embedded struct. +type field struct { + docString string + name string + embeddedStructFields []field + fieldType types.BasicKind + slice bool + embeddedStruct bool +} + +// parse parses the struct defined by the config. +// The fields of the struct must satisfy the following rules: +// - Must be exported. +// - Must be of basic type, slice of basic type or embedded struct, where the embedded struct must satisfy the same +// rules. +// - Must have unique names across all embedded structs. +// - Must have a doc string comment for each field. +func parse(parsingCfg parsingConfig) ([]field, error) { + mode := packages.NeedName | packages.NeedTypes | packages.NeedTypesInfo + + cfg := packages.Config{ + Mode: mode, + Fset: token.NewFileSet(), + Tests: parsingCfg.loadTests, + BuildFlags: parsingCfg.buildFlags, + } + + pattern := "." + if parsingCfg.loadPattern != "" { + pattern = parsingCfg.loadPattern + } + + loadedPackages, err := packages.Load(&cfg, pattern) + if err != nil { + return nil, fmt.Errorf("failed to load package: %w", err) + } + + var pkg *packages.Package + + for _, p := range loadedPackages { + if cfg.Tests && !strings.HasSuffix(p.ID, ".test]") { + continue + } + + if p.Name == parsingCfg.pkgName { + pkg = p + break + } + } + + if pkg == nil { + return nil, fmt.Errorf("package %s not found", parsingCfg.pkgName) + } + + targetType := pkg.Types.Scope().Lookup(parsingCfg.typeName) + if targetType == nil { + return nil, fmt.Errorf("type %s not found", parsingCfg.typeName) + } + + s, ok := targetType.Type().Underlying().(*types.Struct) + if !ok { + return nil, fmt.Errorf("expected struct, got %s", targetType.Type().Underlying().String()) + } + + docStringProvider := newDocStringFieldsProvider(parsingCfg.loadTests, parsingCfg.buildFlags) + + return parseStruct(s, targetType.Type().String(), docStringProvider) +} + +//nolint:gocyclo +func parseStruct(s *types.Struct, typeName string, docStringProvider *docStringFieldsProvider) ([]field, error) { + nameOwners := make(map[string]string) + + var parseRecursively func(*types.Struct, string) ([]field, error) + + parseStructField := func(t *types.Named, f *types.Var, typeName string) (field, error) { + if !f.Embedded() { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: "structs must be embedded", + } + } + + nextS, ok := t.Underlying().(*types.Struct) + if !ok { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: fmt.Sprintf("must be struct, got %s", t.Underlying().String()), + } + } + + if !f.Exported() { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: "must be exported", + } + } + + embeddedFields, err := parseRecursively(nextS, t.String()) + if err != nil { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: err.Error(), + } + } + + return field{ + name: f.Name(), + embeddedStruct: true, + embeddedStructFields: embeddedFields, + }, nil + } + + parseBasicTypeField := func(t *types.Basic, f *types.Var, typeName string) (field, error) { + if f.Embedded() { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: "embedded basic types are not allowed", + } + } + if !f.Exported() { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: "must be exported", + } + } + if _, allowed := allowedBasicKinds[t.Kind()]; !allowed { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: fmt.Sprintf("type of field must be one of %s, got %s", supportedKinds, f.Type().String()), + } + } + + comment, err := docStringProvider.getDocString(typeName, f.Name()) + if err != nil { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: err.Error(), + } + } + + return field{ + name: f.Name(), + fieldType: t.Kind(), + docString: comment, + }, nil + } + + parseSliceField := func(t *types.Slice, f *types.Var, typeName string) (field, error) { + // slices can't be embedded so we don't check for that here + if !f.Exported() { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: "must be exported", + } + } + + elemType, ok := t.Elem().(*types.Basic) + if !ok { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: fmt.Sprintf("type of field must be one of %s, got %s", supportedKinds, f.Type().String()), + } + } + + if _, allowed := allowedBasicKinds[elemType.Kind()]; !allowed { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: fmt.Sprintf("type of field must be one of %s, got %s", supportedKinds, f.Type().String()), + } + } + + comment, err := docStringProvider.getDocString(typeName, f.Name()) + if err != nil { + return field{}, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: err.Error(), + } + } + + return field{ + name: f.Name(), + fieldType: elemType.Kind(), + slice: true, + docString: comment, + }, nil + } + + parseRecursively = func(s *types.Struct, typeName string) ([]field, error) { + var fields []field + + for i := 0; i < s.NumFields(); i++ { + f := s.Field(i) + + var parsedField field + var err error + + switch t := f.Type().(type) { + case *types.Named: // when the field is a Struct + parsedField, err = parseStructField(t, f, typeName) + case *types.Basic: // when the field is a basic type like int, string, etc. + parsedField, err = parseBasicTypeField(t, f, typeName) + case *types.Slice: // when the field is a slice of basic type like []int. + parsedField, err = parseSliceField(t, f, typeName) + default: + err = parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: fmt.Sprintf("must be of embedded struct, basic type or slice of basic type, got %s", f.Type().String()), + } + } + + if err != nil { + return nil, err + } + + fields = append(fields, parsedField) + + if owner, exists := nameOwners[f.Name()]; exists { + return nil, parsingError{ + typeName: typeName, + fieldName: f.Name(), + msg: fmt.Sprintf("already exists in %s", owner), + } + } + + nameOwners[f.Name()] = typeName + } + + return fields, nil + } + + fields, err := parseRecursively(s, typeName) + if err != nil { + return nil, fmt.Errorf("failed to parse struct: %w", err) + } + + return fields, nil +} + +// allowedBasicKinds is a map of allowed basic types. +// Includes all supported types from go.opentelemetry.io/otel/attribute +// except for int. +// Since int size is platform dependent and because the size is required for Avro scheme, we don't use int. +var allowedBasicKinds = map[types.BasicKind]struct{}{ + types.Int64: {}, + types.Float64: {}, + types.String: {}, + types.Bool: {}, +} + +var supportedKinds = func() string { + kindsToString := map[types.BasicKind]string{ + types.Int64: "int64", + types.Float64: "float64", + types.String: "string", + types.Bool: "bool", + } + + kinds := make([]string, 0, len(allowedBasicKinds)) + + for k := range allowedBasicKinds { + s, exist := kindsToString[k] + if !exist { + panic(fmt.Sprintf("unexpected basic kind %v", k)) + } + + kinds = append(kinds, s) + } + + sort.Strings(kinds) + + return strings.Join(kinds, ", ") +}() diff --git a/cmd/generator/parser_test.go b/cmd/generator/parser_test.go new file mode 100644 index 0000000..75ec1f9 --- /dev/null +++ b/cmd/generator/parser_test.go @@ -0,0 +1,369 @@ +//go:build generator + +package main + +import ( + "go/types" + "testing" + + . "github.com/onsi/gomega" + + "github.com/nginxinc/telemetry-exporter/cmd/generator/tests" +) + +type DataUnexportedBasicTypeField struct { + clusterID string //nolint:unused +} + +type DataUnexportedSliceField struct { + someStrings []string //nolint:unused +} + +type DataUnexportedEmbeddedStructField struct { + someStruct //nolint:unused +} + +//nolint:unused +type someStruct struct{} + +type SomeStruct struct{} + +type DataNotEmbeddedStructField struct { + SomeField SomeStruct //nolint:unused +} + +type SomeInterface interface{} + +type DataEmbeddedInterface struct { + SomeInterface +} + +type IntType int64 + +type UnsupportedEmbeddedType struct { + IntType +} + +type EmbeddedBasicType struct { + int64 //nolint:unused +} + +type UnsupportedBasicType struct { + Counter int +} + +type MissingBasicFieldDocString struct { + Counter int64 // doc string above is missing +} + +type MissingSliceFieldDocString struct { + Counters []int64 // doc string above is missing +} + +type EmptyFieldDocString struct { + /* + */ + Counter int64 // empty doc string +} + +type UnsupportedSliceType struct { + Structs []SomeStruct +} + +type UnsupportedBasicTypeSlice struct { + Counters []int +} + +type DuplicateFields struct { + // Counter is a counter. + Counter int64 + EmbeddedDuplicateFields +} + +type EmbeddedDuplicateFields struct { + // Counter is a counter. + Counter int64 +} + +func TestParseErrors(t *testing.T) { + tests := []struct { + name string + expectedErrMsg string + typeName string + }{ + { + name: "not a struct", + expectedErrMsg: "expected struct, got interface{}", + typeName: "SomeInterface", + }, + { + name: "unexported field", + expectedErrMsg: "field clusterID: must be exported", + typeName: "DataUnexportedBasicTypeField", + }, + { + name: "unexported slice field", + expectedErrMsg: "field someStrings: must be exported", + typeName: "DataUnexportedSliceField", + }, + { + name: "unexported embedded struct", + expectedErrMsg: "field someStruct: must be exported", + typeName: "DataUnexportedEmbeddedStructField", + }, + { + name: "not embedded struct", + expectedErrMsg: "field SomeField: structs must be embedded", + typeName: "DataNotEmbeddedStructField", + }, + { + name: "embedded interface", + expectedErrMsg: "field SomeInterface: must be struct, got interface{}", + typeName: "DataEmbeddedInterface", + }, + { + name: "unsupported embedded type", + expectedErrMsg: "field IntType: must be struct, got int", + typeName: "UnsupportedEmbeddedType", + }, + { + name: "embedded basic type", + expectedErrMsg: "field int64: embedded basic types are not allowed", + typeName: "EmbeddedBasicType", + }, + { + name: "unsupported basic type", + expectedErrMsg: "field Counter: type of field must be one of bool, float64, int64, string, got int", + typeName: "UnsupportedBasicType", + }, + { + name: "missing field doc string", + expectedErrMsg: "field Counter: doc string not found", + typeName: "MissingBasicFieldDocString", + }, + { + name: "missing slice field doc string", + expectedErrMsg: "field Counters: doc string not found", + typeName: "MissingSliceFieldDocString", + }, + { + name: "empty field doc string", + expectedErrMsg: "field Counter: doc string not found", + typeName: "EmptyFieldDocString", + }, + { + name: "unsupported slice type", + expectedErrMsg: "field Structs: type of field must be one of bool, float64, int64, string, " + + "got []github.com/nginxinc/telemetry-exporter/cmd/generator.SomeStruct", + typeName: "UnsupportedSliceType", + }, + { + name: "unsupported basic type slice", + expectedErrMsg: "field Counters: type of field must be one of bool, float64, int64, string, got []int", + typeName: "UnsupportedBasicTypeSlice", + }, + { + name: "duplicate fields", + expectedErrMsg: "field Counter: already exists in " + + "github.com/nginxinc/telemetry-exporter/cmd/generator.DuplicateFields", + typeName: "DuplicateFields", + }, + { + name: "type not found", + expectedErrMsg: "type NotFoundType not found", + typeName: "NotFoundType", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewGomegaWithT(t) + + cfg := parsingConfig{ + pkgName: "main", + typeName: test.typeName, + loadTests: true, + buildFlags: []string{"-tags=generator"}, + } + + _, err := parse(cfg) + + g.Expect(err).To(MatchError(ContainSubstring(test.expectedErrMsg))) + }) + } +} + +func TestParseLoadingFailures(t *testing.T) { + g := NewGomegaWithT(t) + + cfg := parsingConfig{ + pkgName: "notfound", + typeName: "Data", + loadTests: true, + buildFlags: []string{"-tags=generator"}, + } + + _, err := parse(cfg) + + g.Expect(err).To(MatchError(ContainSubstring("package notfound not found"))) +} + +func TestParseSuccess(t *testing.T) { + g := NewGomegaWithT(t) + + cfg := parsingConfig{ + pkgName: "tests", + typeName: "Data", + loadPattern: "github.com/nginxinc/telemetry-exporter/cmd/generator/tests", + buildFlags: []string{"-tags=generator"}, + } + + _ = tests.Data{} // depends on the type being defined + + expectedEmbeddedStructFields := []field{ + { + docString: "AnotherSomeString is a string field.", + name: "AnotherSomeString", + fieldType: types.String, + slice: false, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "AnotherSomeInt is an int64 field.", + name: "AnotherSomeInt", + fieldType: types.Int64, + slice: false, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "AnotherSomeFloat is a float64 field.", + name: "AnotherSomeFloat", + fieldType: types.Float64, + slice: false, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "AnotherSomeBool is a bool field.", + name: "AnotherSomeBool", + fieldType: types.Bool, + slice: false, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "AnotherSomeStrings is a slice of strings.", + name: "AnotherSomeStrings", + fieldType: types.String, + slice: true, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "AnotherSomeInts is a slice of int64.", + name: "AnotherSomeInts", + fieldType: types.Int64, + slice: true, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "AnotherSomeFloats is a slice of float64.", + name: "AnotherSomeFloats", + fieldType: types.Float64, + slice: true, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "AnotherSomeBools is a slice of bool.", + name: "AnotherSomeBools", + fieldType: types.Bool, + slice: true, + embeddedStruct: false, + embeddedStructFields: nil, + }, + } + + expectedFields := []field{ + { + docString: "SomeString is a string field.", + name: "SomeString", + fieldType: types.String, + slice: false, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "SomeInt is an int64 field.", + name: "SomeInt", + fieldType: types.Int64, + slice: false, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "SomeFloat is a float64 field.\nMore comments.", + name: "SomeFloat", + fieldType: types.Float64, + slice: false, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "SomeBool is a bool field.", + name: "SomeBool", + fieldType: types.Bool, + slice: false, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "SomeStrings is a slice of strings.", + name: "SomeStrings", + fieldType: types.String, + slice: true, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "SomeInts is a slice of int64.", + name: "SomeInts", + fieldType: types.Int64, + slice: true, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "SomeFloats is a slice of float64.", + name: "SomeFloats", + fieldType: types.Float64, + slice: true, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "SomeBools is a slice of bool.", + name: "SomeBools", + fieldType: types.Bool, + slice: true, + embeddedStruct: false, + embeddedStructFields: nil, + }, + { + docString: "", + name: "Data2", + fieldType: 0, + slice: false, + embeddedStruct: true, + embeddedStructFields: expectedEmbeddedStructFields, + }, + } + + fields, err := parse(cfg) + + g.Expect(err).To(BeNil()) + g.Expect(fields).To(Equal(expectedFields)) +} diff --git a/cmd/generator/scheme.go b/cmd/generator/scheme.go new file mode 100644 index 0000000..df06259 --- /dev/null +++ b/cmd/generator/scheme.go @@ -0,0 +1,116 @@ +//go:build generator + +package main + +import ( + "fmt" + "go/types" + "io" + "strings" + "text/template" +) + +const schemeTemplate = `@namespace("{{ .Namespace }}") protocol {{ .Protocol }} { + @df_datatype("{{ .DataFabricDataType }}") record {{ .Record }} { + /** The field that identifies what type of data this is. */ + string dataType; + /** The time the event occurred */ + long eventTime; + /** The time our edge ingested the event */ + long ingestTime; + + {{ range .Fields }} + /** {{ .Comment }} */ + {{ .Type }} {{ .Name }} = null; + {{ end }} + } +} +` + +func getAvroPrimitiveType(kind types.BasicKind) string { + switch kind { + case types.Int64: + return "long" + case types.Float64: + return "double" + case types.String: + return "string" + case types.Bool: + return "boolean" + default: + panic(fmt.Sprintf("unexpected kind %v", kind)) + } +} + +func getAvroFieldName(name string) string { + return strings.ToLower(name[:1]) + name[1:] +} + +type schemeGen struct { + Namespace string + Protocol string + DataFabricDataType string + Record string + Fields []schemeField +} + +type schemeField struct { + Comment string + Type string + Name string +} + +type schemeGenConfig struct { + namespace string + protocol string + dataFabricDataType string + record string + fields []field +} + +func generateScheme(writer io.Writer, cfg schemeGenConfig) error { + var schemeFields []schemeField + + var createSchemeFields func([]field) + createSchemeFields = func(fields []field) { + for _, f := range fields { + if f.slice { + schemeFields = append(schemeFields, schemeField{ + Comment: f.docString, + Type: fmt.Sprintf("union {null, array<%s>}", getAvroPrimitiveType(f.fieldType)), + Name: getAvroFieldName(f.name), + }) + } else if f.embeddedStruct { + createSchemeFields(f.embeddedStructFields) + } else { + schemeFields = append(schemeFields, schemeField{ + Comment: f.docString, + Type: getAvroPrimitiveType(f.fieldType) + "?", + Name: getAvroFieldName(f.name), + }) + } + } + } + + createSchemeFields(cfg.fields) + + sg := schemeGen{ + Namespace: cfg.namespace, + Protocol: cfg.protocol, + DataFabricDataType: cfg.dataFabricDataType, + Record: cfg.record, + Fields: schemeFields, + } + + funcMap := template.FuncMap{ + "getAttributeType": getAttributeType, + } + + tmpl := template.Must(template.New("scheme").Funcs(funcMap).Parse(schemeTemplate)) + + if err := tmpl.Execute(writer, sg); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + + return nil +} diff --git a/cmd/generator/scheme_test.go b/cmd/generator/scheme_test.go new file mode 100644 index 0000000..0ec5ab1 --- /dev/null +++ b/cmd/generator/scheme_test.go @@ -0,0 +1,44 @@ +//go:build generator + +package main + +import ( + "bytes" + "testing" + + . "github.com/onsi/gomega" + + "github.com/nginxinc/telemetry-exporter/cmd/generator/tests" +) + +func TestGenerateScheme(t *testing.T) { + g := NewGomegaWithT(t) + + parseCfg := parsingConfig{ + pkgName: "tests", + typeName: "Data", + loadPattern: "github.com/nginxinc/telemetry-exporter/cmd/generator/tests", + buildFlags: []string{"-tags=generator"}, + } + + _ = tests.Data{} // depends on the type being defined + + fields, err := parse(parseCfg) + + g.Expect(err).ToNot(HaveOccurred()) + + var buf bytes.Buffer + + schemeCfg := schemeGenConfig{ + namespace: "gateway.nginx.org", + protocol: "avro", + dataFabricDataType: "telemetry", + record: parseCfg.typeName, + fields: fields, + } + + err = generateScheme(&buf, schemeCfg) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(buf.Bytes()).ToNot(BeEmpty()) +} diff --git a/cmd/generator/tests/data.avdl b/cmd/generator/tests/data.avdl new file mode 100644 index 0000000..36c96af --- /dev/null +++ b/cmd/generator/tests/data.avdl @@ -0,0 +1,61 @@ +@namespace("gateway.nginx.org") protocol NGFProductTelemetry { + @df_datatype("ngf-product-telemetry") record Data { + /** The field that identifies what type of data this is. */ + string dataType; + /** The time the event occurred */ + long eventTime; + /** The time our edge ingested the event */ + long ingestTime; + + + /** SomeString is a string field. */ + string? someString = null; + + /** SomeInt is an int64 field. */ + long? someInt = null; + + /** SomeFloat is a float64 field. +More comments. */ + double? someFloat = null; + + /** SomeBool is a bool field. */ + boolean? someBool = null; + + /** SomeStrings is a slice of strings. */ + union {null, array} someStrings = null; + + /** SomeInts is a slice of int64. */ + union {null, array} someInts = null; + + /** SomeFloats is a slice of float64. */ + union {null, array} someFloats = null; + + /** SomeBools is a slice of bool. */ + union {null, array} someBools = null; + + /** AnotherSomeString is a string field. */ + string? anotherSomeString = null; + + /** AnotherSomeInt is an int64 field. */ + long? anotherSomeInt = null; + + /** AnotherSomeFloat is a float64 field. */ + double? anotherSomeFloat = null; + + /** AnotherSomeBool is a bool field. */ + boolean? anotherSomeBool = null; + + /** AnotherSomeStrings is a slice of strings. */ + union {null, array} anotherSomeStrings = null; + + /** AnotherSomeInts is a slice of int64. */ + union {null, array} anotherSomeInts = null; + + /** AnotherSomeFloats is a slice of float64. */ + union {null, array} anotherSomeFloats = null; + + /** AnotherSomeBools is a slice of bool. */ + union {null, array} anotherSomeBools = null; + + } +} diff --git a/cmd/generator/tests/data.go b/cmd/generator/tests/data.go new file mode 100644 index 0000000..d69b8ef --- /dev/null +++ b/cmd/generator/tests/data.go @@ -0,0 +1,35 @@ +//go:build generator + +package tests + +import "github.com/nginxinc/telemetry-exporter/cmd/generator/tests/subtests" + +// Data includes a field of each supported data type. +// We use this struct to test the generation of code and scheme. +// We also use it to test that the generated code compiles and runs as expected. +// +//go:generate go run -tags generator github.com/nginxinc/telemetry-exporter/cmd/generator -type=Data -build-tags=generator -scheme -scheme-protocol=NGFProductTelemetry -scheme-df-datatype=ngf-product-telemetry +//nolint:govet +type Data struct { + // SomeString is a string field. + SomeString string + /* SomeInt is an int64 field. */ + SomeInt int64 + // SomeFloat is a float64 field. + // More comments. + SomeFloat float64 + // SomeBool is a bool field. + SomeBool bool + /* + SomeStrings is a slice of strings. + */ + SomeStrings []string + // SomeInts is a slice of int64. + SomeInts []int64 + // SomeFloats is a slice of float64. + SomeFloats []float64 + // SomeBools is a slice of bool. + SomeBools []bool + + subtests.Data2 +} diff --git a/cmd/generator/tests/data_attributes_generated.go b/cmd/generator/tests/data_attributes_generated.go new file mode 100644 index 0000000..2ed36dd --- /dev/null +++ b/cmd/generator/tests/data_attributes_generated.go @@ -0,0 +1,29 @@ +//go:build generator +package tests +/* +This is a generated file. DO NOT EDIT. +*/ + +import ( + "go.opentelemetry.io/otel/attribute" + "github.com/nginxinc/telemetry-exporter/pkg/telemetry" +) + +func (d *Data) Attributes() []attribute.KeyValue { + var attrs []attribute.KeyValue + + attrs = append(attrs, attribute.String("SomeString", d.SomeString)) + attrs = append(attrs, attribute.Int64("SomeInt", d.SomeInt)) + attrs = append(attrs, attribute.Float64("SomeFloat", d.SomeFloat)) + attrs = append(attrs, attribute.Bool("SomeBool", d.SomeBool)) + attrs = append(attrs, attribute.StringSlice("SomeStrings", d.SomeStrings)) + attrs = append(attrs, attribute.Int64Slice("SomeInts", d.SomeInts)) + attrs = append(attrs, attribute.Float64Slice("SomeFloats", d.SomeFloats)) + attrs = append(attrs, attribute.BoolSlice("SomeBools", d.SomeBools)) + attrs = append(attrs, d.Data2.Attributes()...) + + + return attrs +} + +var _ telemetry.Exportable = (*Data)(nil) diff --git a/cmd/generator/tests/data_test.go b/cmd/generator/tests/data_test.go new file mode 100644 index 0000000..aa82628 --- /dev/null +++ b/cmd/generator/tests/data_test.go @@ -0,0 +1,89 @@ +//go:build generator + +package tests + +import ( + "testing" + + . "github.com/onsi/gomega" + "go.opentelemetry.io/otel/attribute" + + "github.com/nginxinc/telemetry-exporter/cmd/generator/tests/subtests" +) + +func TestData_Attributes(t *testing.T) { + g := NewGomegaWithT(t) + + data := Data{ + SomeString: "some string", + SomeInt: 42, + SomeFloat: 3.14, + SomeBool: true, + SomeStrings: []string{"a", "b", "c"}, + SomeInts: []int64{1, 2, 3}, + SomeFloats: []float64{1.1, 2.2, 3.3}, + SomeBools: []bool{true, false, true}, + Data2: subtests.Data2{ + AnotherSomeString: "another string", + AnotherSomeInt: 24, + AnotherSomeFloat: 1.41, + AnotherSomeBool: false, + AnotherSomeStrings: []string{"d", "e", "f"}, + AnotherSomeInts: []int64{4, 5, 6}, + AnotherSomeFloats: []float64{4.4, 5.5, 6.6}, + AnotherSomeBools: []bool{false, true, false}, + }, + } + + expectedAttributes := []attribute.KeyValue{ + attribute.String("SomeString", "some string"), + attribute.Int64("SomeInt", 42), + attribute.Float64("SomeFloat", 3.14), + attribute.Bool("SomeBool", true), + attribute.StringSlice("SomeStrings", []string{"a", "b", "c"}), + attribute.Int64Slice("SomeInts", []int64{1, 2, 3}), + attribute.Float64Slice("SomeFloats", []float64{1.1, 2.2, 3.3}), + attribute.BoolSlice("SomeBools", []bool{true, false, true}), + attribute.String("AnotherSomeString", "another string"), + attribute.Int64("AnotherSomeInt", 24), + attribute.Float64("AnotherSomeFloat", 1.41), + attribute.Bool("AnotherSomeBool", false), + attribute.StringSlice("AnotherSomeStrings", []string{"d", "e", "f"}), + attribute.Int64Slice("AnotherSomeInts", []int64{4, 5, 6}), + attribute.Float64Slice("AnotherSomeFloats", []float64{4.4, 5.5, 6.6}), + attribute.BoolSlice("AnotherSomeBools", []bool{false, true, false}), + } + + attributes := data.Attributes() + + g.Expect(attributes).To(ConsistOf(expectedAttributes)) +} + +func TestData_AttributesEmpty(t *testing.T) { + g := NewGomegaWithT(t) + + data := Data{} + + expectedAttributes := []attribute.KeyValue{ + attribute.String("SomeString", ""), + attribute.Int64("SomeInt", 0), + attribute.Float64("SomeFloat", 0), + attribute.Bool("SomeBool", false), + attribute.StringSlice("SomeStrings", []string{}), + attribute.Int64Slice("SomeInts", []int64{}), + attribute.Float64Slice("SomeFloats", []float64{}), + attribute.BoolSlice("SomeBools", []bool{}), + attribute.String("AnotherSomeString", ""), + attribute.Int64("AnotherSomeInt", 0), + attribute.Float64("AnotherSomeFloat", 0), + attribute.Bool("AnotherSomeBool", false), + attribute.StringSlice("AnotherSomeStrings", []string{}), + attribute.Int64Slice("AnotherSomeInts", []int64{}), + attribute.Float64Slice("AnotherSomeFloats", []float64{}), + attribute.BoolSlice("AnotherSomeBools", []bool{}), + } + + attributes := data.Attributes() + + g.Expect(attributes).To(ConsistOf(expectedAttributes)) +} diff --git a/cmd/generator/tests/subtests/data2.go b/cmd/generator/tests/subtests/data2.go new file mode 100644 index 0000000..b9de68f --- /dev/null +++ b/cmd/generator/tests/subtests/data2.go @@ -0,0 +1,28 @@ +//go:build generator + +package subtests + +// Data2 is a struct that can be exported by a struct in another package to test cross-package referencing +// when generating code and scheme. +// Data2 includes a field of each supported data type except an embedded struct. +// +//go:generate go run -tags generator github.com/nginxinc/telemetry-exporter/cmd/generator -type=Data2 -build-tags=generator +//nolint:govet +type Data2 struct { + // AnotherSomeString is a string field. + AnotherSomeString string + // AnotherSomeInt is an int64 field. + AnotherSomeInt int64 + // AnotherSomeFloat is a float64 field. + AnotherSomeFloat float64 + // AnotherSomeBool is a bool field. + AnotherSomeBool bool + // AnotherSomeStrings is a slice of strings. + AnotherSomeStrings []string + // AnotherSomeInts is a slice of int64. + AnotherSomeInts []int64 + // AnotherSomeFloats is a slice of float64. + AnotherSomeFloats []float64 + // AnotherSomeBools is a slice of bool. + AnotherSomeBools []bool +} diff --git a/cmd/generator/tests/subtests/data2_attributes_generated.go b/cmd/generator/tests/subtests/data2_attributes_generated.go new file mode 100644 index 0000000..003450a --- /dev/null +++ b/cmd/generator/tests/subtests/data2_attributes_generated.go @@ -0,0 +1,28 @@ +//go:build generator +package subtests +/* +This is a generated file. DO NOT EDIT. +*/ + +import ( + "go.opentelemetry.io/otel/attribute" + "github.com/nginxinc/telemetry-exporter/pkg/telemetry" +) + +func (d *Data2) Attributes() []attribute.KeyValue { + var attrs []attribute.KeyValue + + attrs = append(attrs, attribute.String("AnotherSomeString", d.AnotherSomeString)) + attrs = append(attrs, attribute.Int64("AnotherSomeInt", d.AnotherSomeInt)) + attrs = append(attrs, attribute.Float64("AnotherSomeFloat", d.AnotherSomeFloat)) + attrs = append(attrs, attribute.Bool("AnotherSomeBool", d.AnotherSomeBool)) + attrs = append(attrs, attribute.StringSlice("AnotherSomeStrings", d.AnotherSomeStrings)) + attrs = append(attrs, attribute.Int64Slice("AnotherSomeInts", d.AnotherSomeInts)) + attrs = append(attrs, attribute.Float64Slice("AnotherSomeFloats", d.AnotherSomeFloats)) + attrs = append(attrs, attribute.BoolSlice("AnotherSomeBools", d.AnotherSomeBools)) + + + return attrs +} + +var _ telemetry.Exportable = (*Data2)(nil) diff --git a/go.mod b/go.mod index d3415f2..737be05 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.23.1 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.23.1 go.opentelemetry.io/otel/sdk v1.23.1 + golang.org/x/tools v0.17.0 ) require ( @@ -28,7 +29,6 @@ require ( golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.17.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect google.golang.org/grpc v1.61.0 // indirect