[mdatagen] update mdatagen to document internal telemetry (open-telem…

…etry#10170)

This allows end users to see what telemetry each component should be
emitting.

---------

Signed-off-by: Alex Boten <223565+codeboten@users.noreply.github.com>

.chloggen/codeboten_mdatagen-docs-internal-metrics.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: mdatagen
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note:  generate documentation for internal telemetry
+
+# One or more tracking issues or pull requests related to the change
+issues: [10170]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

cmd/mdatagen/internal/samplereceiver/documentation.md

@@ -112,3 +112,23 @@ metrics:
 | string.resource.attr_disable_warning | Resource attribute with any string value. | Any Str | true |
 | string.resource.attr_remove_warning | Resource attribute with any string value. | Any Str | false |
 | string.resource.attr_to_be_removed | Resource attribute with any string value. | Any Str | true |
+
+## Internal Telemetry
+
+The following telemetry is emitted by this component.
+
+### batch_size_trigger_send
+
+Number of times the batch was sent due to a size trigger
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### request_duration
+
+Duration of request
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| s | Histogram | Double |

cmd/mdatagen/main.go

@@ -85,13 +85,19 @@ func run(ymlPath string) error {
 		}
 	}
 
-	if len(md.Telemetry.Metrics) != 0 {
-		if err = generateFile(filepath.Join(tmplDir, "telemetry.go.tmpl"),
-			filepath.Join(codeDir, "generated_telemetry.go"), md, "metadata"); err != nil {
-			return err
-		}
-		if err = generateFile(filepath.Join(tmplDir, "telemetry_test.go.tmpl"),
-			filepath.Join(codeDir, "generated_telemetry_test.go"), md, "metadata"); err != nil {
+	toGenerate := map[string]string{}
+
+	if len(md.Telemetry.Metrics) != 0 { // if there are telemetry metrics, generate telemetry specific files
+		toGenerate[filepath.Join(tmplDir, "telemetry.go.tmpl")] = filepath.Join(codeDir, "generated_telemetry.go")
+		toGenerate[filepath.Join(tmplDir, "telemetry_test.go.tmpl")] = filepath.Join(codeDir, "generated_telemetry_test.go")
+	}
+
+	if len(md.Metrics) != 0 || len(md.Telemetry.Metrics) != 0 { // if there's metrics or internal metrics, generate documentation for them
+		toGenerate[filepath.Join(tmplDir, "documentation.md.tmpl")] = filepath.Join(ymlDir, "documentation.md")
+	}
+
+	for tmpl, dst := range toGenerate {
+		if err = generateFile(tmpl, dst, md, "metadata"); err != nil {
 			return err
 		}
 	}
@@ -103,45 +109,30 @@ func run(ymlPath string) error {
 	if err = os.MkdirAll(filepath.Join(codeDir, "testdata"), 0700); err != nil {
 		return fmt.Errorf("unable to create output directory %q: %w", filepath.Join(codeDir, "testdata"), err)
 	}
-	if err = generateFile(filepath.Join(tmplDir, "testdata", "config.yaml.tmpl"),
-		filepath.Join(codeDir, "testdata", "config.yaml"), md, "metadata"); err != nil {
-		return err
-	}
 
-	if err = generateFile(filepath.Join(tmplDir, "config.go.tmpl"),
-		filepath.Join(codeDir, "generated_config.go"), md, "metadata"); err != nil {
-		return err
-	}
-	if err = generateFile(filepath.Join(tmplDir, "config_test.go.tmpl"),
-		filepath.Join(codeDir, "generated_config_test.go"), md, "metadata"); err != nil {
-		return err
+	toGenerate = map[string]string{
+		filepath.Join(tmplDir, "testdata", "config.yaml.tmpl"): filepath.Join(codeDir, "testdata", "config.yaml"),
+		filepath.Join(tmplDir, "config.go.tmpl"):               filepath.Join(codeDir, "generated_config.go"),
+		filepath.Join(tmplDir, "config_test.go.tmpl"):          filepath.Join(codeDir, "generated_config_test.go"),
 	}
 
-	if len(md.ResourceAttributes) > 0 {
-		if err = generateFile(filepath.Join(tmplDir, "resource.go.tmpl"),
-			filepath.Join(codeDir, "generated_resource.go"), md, "metadata"); err != nil {
-			return err
-		}
-		if err = generateFile(filepath.Join(tmplDir, "resource_test.go.tmpl"),
-			filepath.Join(codeDir, "generated_resource_test.go"), md, "metadata"); err != nil {
-			return err
-		}
+	if len(md.ResourceAttributes) > 0 { // only generate resource files if resource attributes are configured
+		toGenerate[filepath.Join(tmplDir, "resource.go.tmpl")] = filepath.Join(codeDir, "generated_resource.go")
+		toGenerate[filepath.Join(tmplDir, "resource_test.go.tmpl")] = filepath.Join(codeDir, "generated_resource_test.go")
 	}
 
-	if len(md.Metrics) == 0 {
-		return nil
+	if len(md.Metrics) > 0 { // only generate metrics if metrics are present
+		toGenerate[filepath.Join(tmplDir, "metrics.go.tmpl")] = filepath.Join(codeDir, "generated_metrics.go")
+		toGenerate[filepath.Join(tmplDir, "metrics_test.go.tmpl")] = filepath.Join(codeDir, "generated_metrics_test.go")
 	}
 
-	if err = generateFile(filepath.Join(tmplDir, "metrics.go.tmpl"),
-		filepath.Join(codeDir, "generated_metrics.go"), md, "metadata"); err != nil {
-		return err
-	}
-	if err = generateFile(filepath.Join(tmplDir, "metrics_test.go.tmpl"),
-		filepath.Join(codeDir, "generated_metrics_test.go"), md, "metadata"); err != nil {
-		return err
+	for tmpl, dst := range toGenerate {
+		if err = generateFile(tmpl, dst, md, "metadata"); err != nil {
+			return err
+		}
 	}
 
-	return generateFile(filepath.Join(tmplDir, "documentation.md.tmpl"), filepath.Join(ymlDir, "documentation.md"), md, "metadata")
+	return nil
 }
 
 func templatize(tmplFile string, md metadata) *template.Template {
@@ -159,6 +150,9 @@ func templatize(tmplFile string, md metadata) *template.Template {
 				"metricInfo": func(mn metricName) metric {
 					return md.Metrics[mn]
 				},
+				"telemetryInfo": func(mn metricName) metric {
+					return md.Telemetry.Metrics[mn]
+				},
 				"parseImportsRequired": func(metrics map[metricName]metric) bool {
 					for _, m := range metrics {
 						if m.Data().HasMetricInputType() {

cmd/mdatagen/templates/documentation.md.tmpl

@@ -32,6 +32,28 @@
 
 {{- end }}
 
+{{- end -}}
+
+{{- define "telemetry-documentation" -}}
+{{- $metricName := . }}
+{{- $metric := $metricName | telemetryInfo -}}
+
+### {{ $metricName }}
+
+{{ $metric.Description }}
+
+{{- if $metric.ExtendedDocumentation }}
+
+{{ $metric.ExtendedDocumentation }}
+
+{{- end }}
+
+| Unit | Metric Type | Value Type |{{ if $metric.Data.HasMonotonic }} Monotonic |{{ end }}
+| ---- | ----------- | ---------- |{{ if $metric.Data.HasMonotonic }} --------- |{{ end }}
+| {{ $metric.Unit }} | {{ $metric.Data.Type }} | {{ $metric.Data.MetricValueType }} |
+{{- if $metric.Data.HasMonotonic }} {{ $metric.Data.Monotonic }} |{{ end }}
+
+
 {{- end -}}
 
 [comment]: <> (Code generated by mdatagen. DO NOT EDIT.)
@@ -43,6 +65,8 @@
 **Parent Component:** {{ .Parent }}
 {{- end }}
 
+{{- if .Metrics }}
+
 ## Default Metrics
 
 The following metrics are emitted by default. Each of them can be disabled by applying the following configuration:
@@ -53,6 +77,8 @@ metrics:
     enabled: false
 ```
 
+{{- end }}
+
 {{- range $metricName, $metric := .Metrics }}
 {{- if $metric.Enabled }}
 
@@ -96,3 +122,19 @@ metrics:
 {{- end }}
 
 {{- end }}
+
+{{- if .Telemetry.Metrics }}
+
+## Internal Telemetry
+
+The following telemetry is emitted by this component.
+
+{{- range $metricName, $metric := .Telemetry.Metrics }}
+{{- if $metric.Enabled }}
+
+{{ template "telemetry-documentation" $metricName }}
+
+{{- end }}
+{{- end }}
+
+{{- end }}

exporter/exporterhelper/documentation.md

@@ -0,0 +1,79 @@
+[comment]: <> (Code generated by mdatagen. DO NOT EDIT.)
+
+# exporterhelper
+
+## Internal Telemetry
+
+The following telemetry is emitted by this component.
+
+### exporter_enqueue_failed_log_records
+
+Number of log records failed to be added to the sending queue.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### exporter_enqueue_failed_metric_points
+
+Number of metric points failed to be added to the sending queue.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### exporter_enqueue_failed_spans
+
+Number of spans failed to be added to the sending queue.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### exporter_send_failed_log_records
+
+Number of log records in failed attempts to send to destination.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### exporter_send_failed_metric_points
+
+Number of metric points in failed attempts to send to destination.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### exporter_send_failed_spans
+
+Number of spans in failed attempts to send to destination.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### exporter_sent_log_records
+
+Number of log record successfully sent to destination.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### exporter_sent_metric_points
+
+Number of metric points successfully sent to destination.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### exporter_sent_spans
+
+Number of spans successfully sent to destination.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |

processor/batchprocessor/documentation.md

@@ -0,0 +1,39 @@
+[comment]: <> (Code generated by mdatagen. DO NOT EDIT.)
+
+# batch
+
+## Internal Telemetry
+
+The following telemetry is emitted by this component.
+
+### processor_batch_batch_send_size
+
+Number of units in the batch
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| 1 | Histogram | Int |
+
+### processor_batch_batch_send_size_bytes
+
+Number of bytes in batch that was sent
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| By | Histogram | Int |
+
+### processor_batch_batch_size_trigger_send
+
+Number of times the batch was sent due to a size trigger
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### processor_batch_timeout_trigger_send
+
+Number of times the batch was sent due to a timeout trigger
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |