Add support for markdown generation (#3100)

This change contains a CLI tool, called 'docsgen', that generates markdown files for collector
components. The markdown files present the configuration metadata extracted by the `configschema`
API in a human readable form that can be used to manually configure the collector.

This change also makes some modifications to the package formerly knows as `schemagen`, renaming
it to `configschema` and exporting some things, because it no longer generates a schema yaml file, but
rather provides an equivalent API used by docsgen.

Also, this PR includes one sample, generated .md document: `receiver/otlpreceiver/config.md`

cmd/configschema/configschema/README.md

@@ -0,0 +1,5 @@
+# ConfigSchema API
+
+This package contains an API that can be used to introspect the configuration
+struct of a collector component. It can be used to generate documentation or
+tools to help users configure the collector.

cmd/schemagen/schemagen/comments.go → cmd/configschema/configschema/comments.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package schemagen
+package configschema
 
 import (
 	"go/ast"
@@ -23,12 +23,12 @@ import (
 )
 
 // commentsForStruct returns a map of fieldname -> comment for a struct
-func commentsForStruct(v reflect.Value, env env) map[string]string {
+func commentsForStruct(v reflect.Value, dr DirResolver) map[string]string {
 	elem := v
 	if v.Kind() == reflect.Ptr {
 		elem = v.Elem()
 	}
-	dir := packageDir(elem.Type(), env)
+	dir := dr.PackageDir(elem.Type())
 	name := trimPackage(elem)
 	return commentsForStructName(dir, name)
 }
@@ -48,13 +48,20 @@ func commentsForStructName(packageDir, structName string) map[string]string {
 	comments := map[string]string{}
 	for _, pkg := range pkgs {
 		for _, file := range pkg.Files {
-			if obj, ok := file.Scope.Objects[structName]; ok {
-				if ts, ok := obj.Decl.(*ast.TypeSpec); ok {
-					if st, ok := ts.Type.(*ast.StructType); ok {
-						for _, field := range st.Fields.List {
-							if field.Doc != nil {
-								if name := fieldName(field); name != "" {
-									comments[name] = field.Doc.Text()
+			for _, decl := range file.Decls {
+				if gd, ok := decl.(*ast.GenDecl); ok {
+					for _, spec := range gd.Specs {
+						if ts, ok := spec.(*ast.TypeSpec); ok {
+							if ts.Name.Name == structName {
+								if structComments := gd.Doc.Text(); structComments != "" {
+									comments["_struct"] = structComments
+								}
+								if st, ok := ts.Type.(*ast.StructType); ok {
+									for _, field := range st.Fields.List {
+										if name := fieldName(field); name != "" {
+											comments[name] = field.Doc.Text()
+										}
+									}
 								}
 							}
 						}

cmd/schemagen/schemagen/comments_test.go → cmd/configschema/configschema/comments_test.go

@@ -12,19 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package schemagen
+package configschema
 
 import (
 	"reflect"
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestFieldComments(t *testing.T) {
 	v := reflect.ValueOf(testStruct{})
-	comments := commentsForStruct(v, testEnv())
-	require.EqualValues(t, map[string]string{
-		"Duration": "embedded, package qualified\n",
-	}, comments)
+	comments := commentsForStruct(v, testDR())
+	assert.Equal(t, "embedded, package qualified comment\n", comments["Duration"])
+	assert.Equal(t, "testStruct comment\n", comments["_struct"])
 }

cmd/schemagen/schemagen/libfor_test.go → cmd/configschema/configschema/common_test.go

@@ -12,20 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package schemagen
+package configschema
 
 import "time"
 
 type testPerson struct {
 	Name string
 }
 
+// testStruct comment
 type testStruct struct {
 	One   string `mapstructure:"one"`
 	Two   int    `mapstructure:"two"`
 	Three uint   `mapstructure:"three"`
 	Four  bool   `mapstructure:"four"`
-	// embedded, package qualified
+	// embedded, package qualified comment
 	time.Duration `mapstructure:"duration"`
 	Squashed      testPerson    `mapstructure:",squash"`
 	PersonPtr     *testPerson   `mapstructure:"person_ptr"`
@@ -35,9 +36,9 @@ type testStruct struct {
 	Ignored       string        `mapstructure:"-"`
 }
 
-func testEnv() env {
-	return env{
-		srcRoot:    "../../..",
-		moduleName: defaultModule,
+func testDR() DirResolver {
+	return DirResolver{
+		SrcRoot:    "../../..",
+		ModuleName: DefaultModule,
 	}
 }

cmd/configschema/configschema/configs.go

@@ -0,0 +1,125 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package configschema
+
+import (
+	"fmt"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config"
+)
+
+const (
+	receiver  = "receiver"
+	extension = "extension"
+	processor = "processor"
+	exporter  = "exporter"
+)
+
+// CfgInfo contains a component config instance, as well as its group name and
+// type.
+type CfgInfo struct {
+	// the name of the component group, e.g. "receiver"
+	Group string
+	// the component type, e.g. "otlpreceiver.Config"
+	Type config.Type
+	// an instance of the component's configuration struct
+	CfgInstance interface{}
+}
+
+// GetAllCfgInfos accepts a Factories struct, then creates and returns a CfgInfo
+// for each of its components.
+func GetAllCfgInfos(components component.Factories) []CfgInfo {
+	var out []CfgInfo
+	for _, f := range components.Receivers {
+		out = append(out, CfgInfo{
+			Type:        f.Type(),
+			Group:       receiver,
+			CfgInstance: f.CreateDefaultConfig(),
+		})
+	}
+	for _, f := range components.Extensions {
+		out = append(out, CfgInfo{
+			Type:        f.Type(),
+			Group:       extension,
+			CfgInstance: f.CreateDefaultConfig(),
+		})
+	}
+	for _, f := range components.Processors {
+		out = append(out, CfgInfo{
+			Type:        f.Type(),
+			Group:       processor,
+			CfgInstance: f.CreateDefaultConfig(),
+		})
+	}
+	for _, f := range components.Exporters {
+		out = append(out, CfgInfo{
+			Type:        f.Type(),
+			Group:       exporter,
+			CfgInstance: f.CreateDefaultConfig(),
+		})
+	}
+	return out
+}
+
+// GetCfgInfo accepts a Factories struct, then creates and returns the default
+// config for the component specified by the passed-in componentType and
+// componentName.
+func GetCfgInfo(components component.Factories, componentType, componentName string) (CfgInfo, error) {
+	t := config.Type(componentName)
+	switch componentType {
+	case receiver:
+		f := components.Receivers[t]
+		if f == nil {
+			return CfgInfo{}, fmt.Errorf("unknown %s name %q", componentType, componentName)
+		}
+		return CfgInfo{
+			Type:        f.Type(),
+			Group:       componentType,
+			CfgInstance: f.CreateDefaultConfig(),
+		}, nil
+	case processor:
+		f := components.Processors[t]
+		if f == nil {
+			return CfgInfo{}, fmt.Errorf("unknown %s name %q", componentType, componentName)
+		}
+		return CfgInfo{
+			Type:        f.Type(),
+			Group:       componentType,
+			CfgInstance: f.CreateDefaultConfig(),
+		}, nil
+	case exporter:
+		f := components.Exporters[t]
+		if f == nil {
+			return CfgInfo{}, fmt.Errorf("unknown %s name %q", componentType, componentName)
+		}
+		return CfgInfo{
+			Type:        f.Type(),
+			Group:       componentType,
+			CfgInstance: f.CreateDefaultConfig(),
+		}, nil
+	case extension:
+		f := components.Extensions[t]
+		if f == nil {
+			return CfgInfo{}, fmt.Errorf("unknown %s name %q", componentType, componentName)
+		}
+		return CfgInfo{
+			Type:        f.Type(),
+			Group:       componentType,
+			CfgInstance: f.CreateDefaultConfig(),
+		}, nil
+	}
+	return CfgInfo{}, fmt.Errorf("unknown component type %q", componentType)
+}

cmd/schemagen/schemagen/cli_single_test.go → cmd/configschema/configschema/configs_test.go

@@ -12,30 +12,30 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package schemagen
+package configschema
 
 import (
-	"io/ioutil"
-	"path"
-	"path/filepath"
-	"reflect"
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
 
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/service/defaultcomponents"
 )
 
+func TestGetAllConfigs(t *testing.T) {
+	cfgs := GetAllCfgInfos(testComponents())
+	require.NotNil(t, cfgs)
+}
+
 func TestCreateReceiverConfig(t *testing.T) {
-	cfg, err := getConfig(testComponents(), "receiver", "otlp")
+	cfg, err := GetCfgInfo(testComponents(), "receiver", "otlp")
 	require.NoError(t, err)
 	require.NotNil(t, cfg)
 }
 
 func TestCreateProcesorConfig(t *testing.T) {
-	cfg, err := getConfig(testComponents(), "processor", "filter")
+	cfg, err := GetCfgInfo(testComponents(), "processor", "filter")
 	require.NoError(t, err)
 	require.NotNil(t, cfg)
 }
@@ -64,29 +64,13 @@ func TestGetConfig(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			cfg, err := getConfig(testComponents(), test.componentType, test.name)
+			cfg, err := GetCfgInfo(testComponents(), test.componentType, test.name)
 			require.NoError(t, err)
 			require.NotNil(t, cfg)
 		})
 	}
 }
 
-func TestCreateSingleSchemaFile(t *testing.T) {
-	e := testEnv()
-	tempDir := t.TempDir()
-	e.yamlFilename = func(reflect.Type, env) string {
-		return path.Join(tempDir, schemaFilename)
-	}
-	createSingleSchemaFile(testComponents(), "exporter", "otlp", e)
-	file, err := ioutil.ReadFile(filepath.Clean(path.Join(tempDir, schemaFilename)))
-	require.NoError(t, err)
-	fld := field{}
-	err = yaml.Unmarshal(file, &fld)
-	require.NoError(t, err)
-	require.Equal(t, "*otlpexporter.Config", fld.Type)
-	require.NotNil(t, fld.Fields)
-}
-
 func testComponents() component.Factories {
 	components, err := defaultcomponents.Components()
 	if err != nil {