[Exporter/OpenSearch] OpenSearch exporter setup

.chloggen/opensearch-exporter.yaml

@@ -0,0 +1,9 @@
+change_type: new_component
+
+component: opensearchexporter
+
+note: exports OpenTelemetry signals to [OpenSearch](https://opensearch.org/).
+
+issues: [23611]
+
+subtext:

.github/CODEOWNERS

@@ -67,6 +67,7 @@ exporter/logzioexporter/                                 @open-telemetry/collect
 exporter/lokiexporter/                                   @open-telemetry/collector-contrib-approvers @gramidt @gouthamve @jpkrohling @kovrus @mar4uk
 exporter/mezmoexporter/                                  @open-telemetry/collector-contrib-approvers @dashpole @billmeyer @gjanco
 exporter/opencensusexporter/                             @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers
+exporter/opensearchexporter/                             @open-telemetry/collector-contrib-approvers @Aneurysm9 @MitchellGale @MaxKsyunz @YANG-DB
 exporter/parquetexporter/                                @open-telemetry/collector-contrib-approvers @atoulme
 exporter/prometheusexporter/                             @open-telemetry/collector-contrib-approvers @Aneurysm9
 exporter/prometheusremotewriteexporter/                  @open-telemetry/collector-contrib-approvers @Aneurysm9 @rapphil

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -61,6 +61,7 @@ body:
       - exporter/loki
       - exporter/mezmo
       - exporter/opencensus
+      - exporter/opensearch
       - exporter/parquet
       - exporter/prometheus
       - exporter/prometheusremotewrite

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -55,6 +55,7 @@ body:
       - exporter/loki
       - exporter/mezmo
       - exporter/opencensus
+      - exporter/opensearch
       - exporter/parquet
       - exporter/prometheus
       - exporter/prometheusremotewrite

.github/ISSUE_TEMPLATE/other.yaml

@@ -55,6 +55,7 @@ body:
       - exporter/loki
       - exporter/mezmo
       - exporter/opencensus
+      - exporter/opensearch
       - exporter/parquet
       - exporter/prometheus
       - exporter/prometheusremotewrite

.github/dependabot.yml

@@ -227,6 +227,11 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
+  - package-ecosystem: "gomod"
+    directory: "/exporter/opensearchexporter"
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
   - package-ecosystem: "gomod"
     directory: "/exporter/parquetexporter"
     schedule:
@@ -1097,8 +1102,3 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
-  - package-ecosystem: "gomod"
-    directory: "/receiver/syslogreceiver"
-    schedule:
-      interval: "weekly"
-      day: "wednesday"

exporter/opensearchexporter/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

exporter/opensearchexporter/README.md

@@ -0,0 +1,66 @@
+# OpenSearch Exporter
+
+| Status                   |           |
+| ------------------------ |-----------|
+| Stability                | [devel]   |
+| Supported pipeline types | traces    |
+| Distributions            | [contrib] |
+
+This exporter supports sending OpenTelemetry signals as documents to [OpenSearch](https://www.opensearch.org).
+
+The documents are sent using [observability catalog](https://github.com/opensearch-project/opensearch-catalog/tree/main/schema/observability) schema.
+
+## Configuration options
+
+- `endpoints`: List of OpenSearch URLs. If endpoints is missing, the
+  OPENSEARCH_URL environment variable will be used.
+- `num_workers` (optional): Number of workers publishing bulk requests concurrently.
+- `dataset` (default=`default`) a user-provided label. It is used to construct the name of the destination index or data stream.
+- `namespace` (default=`namespace`)a user-provided label. It is used to construct the name of the destination index or data stream.
+- `flush`: Event bulk buffer flush settings
+    - `bytes` (default=5242880): Write buffer flush limit.
+    - `interval` (default=30s): Write buffer time limit.
+- `retry`: Event retry settings
+    - `enabled` (default=true): Enable/Disable event retry on error. Retry
+      support is enabled by default.
+    - `max_requests` (default=3): Number of HTTP request retries.
+    - `initial_interval` (default=100ms): Initial waiting time if a HTTP request failed.
+    - `max_interval` (default=1m): Max waiting time if a HTTP request failed.
+
+### HTTP settings
+
+- `read_buffer_size` (default=0): Read buffer size.
+- `write_buffer_size` (default=0): Write buffer size used when.
+- `timeout` (default=90s): HTTP request time limit.
+- `headers` (optional): Headers to be send with each HTTP request.
+
+### Security and Authentication settings
+
+- `user` (optional): Username used for HTTP Basic Authentication.
+- `password` (optional): Password used for HTTP Basic Authentication.
+
+### TLS settings
+- `ca_file` (optional): Root Certificate Authority (CA) certificate, for
+  verifying the server's identity, if TLS is enabled.
+- `cert_file` (optional): Client TLS certificate.
+- `key_file` (optional): Client TLS key.
+- `insecure` (optional): In gRPC when set to true, this is used to disable the client transport security. In HTTP, this disables verifying the server's certificate chain and host name.
+- `insecure_skip_verify` (optional): Will enable TLS but not verify the certificate.
+  is enabled.
+
+## Example
+
+```yaml
+exporters:
+  opensearch/trace:
+    endpoints: [https://opensearch.example.com:9200]
+# ······
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [opensearch/trace]
+      processors: [batch]
+```
+[devel]:https://github.com/open-telemetry/opentelemetry-collector#development
+[contrib]:https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib

exporter/opensearchexporter/config.go

@@ -0,0 +1,122 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package opensearchexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/opensearchexporter"
+
+import (
+	"errors"
+	"os"
+	"time"
+
+	"go.opentelemetry.io/collector/config/configopaque"
+	"go.opentelemetry.io/collector/config/configtls"
+)
+
+const (
+	defaultOpenSearchEnvName = "OPENSEARCH_URL"
+)
+
+// Config defines configuration for OpenSearch exporter.
+type Config struct {
+
+	// Endpoints holds the OpenSearch URLs the exporter should send events to.
+	//
+	// OPENSEARCH_URL environment variable is not set.
+	Endpoints []string `mapstructure:"endpoints"`
+
+	// NumWorkers configures the number of workers publishing bulk requests.
+	NumWorkers int `mapstructure:"num_workers"`
+
+	HTTPClientSettings `mapstructure:",squash"`
+	Retry              RetrySettings `mapstructure:"retry"`
+	Flush              FlushSettings `mapstructure:"flush"`
+	Namespace          string        `mapstructure:"namespace"`
+	Dataset            string        `mapstructure:"dataset"`
+}
+
+type HTTPClientSettings struct {
+	Authentication AuthenticationSettings `mapstructure:",squash"`
+
+	// ReadBufferSize for HTTP client. See http.Transport.ReadBufferSize.
+	ReadBufferSize int `mapstructure:"read_buffer_size"`
+
+	// WriteBufferSize for HTTP client. See http.Transport.WriteBufferSize.
+	WriteBufferSize int `mapstructure:"write_buffer_size"`
+
+	// Timeout configures the HTTP request timeout.
+	Timeout time.Duration `mapstructure:"timeout"`
+
+	// Headers allows users to configure optional HTTP headers that
+	// will be sent with each HTTP request.
+	Headers map[string]string `mapstructure:"headers,omitempty"`
+
+	configtls.TLSClientSetting `mapstructure:"tls,omitempty"`
+}
+
+// AuthenticationSettings defines user authentication related settings.
+type AuthenticationSettings struct {
+	// User is used to configure HTTP Basic Authentication.
+	User string `mapstructure:"user"`
+
+	// Password is used to configure HTTP Basic Authentication.
+	Password configopaque.String `mapstructure:"password"`
+}
+
+// FlushSettings  defines settings for configuring the write buffer flushing
+// policy in the OpenSearch exporter. The exporter sends a bulk request with
+// all events already serialized into the send-buffer.
+type FlushSettings struct {
+	// Bytes sets the send buffer flushing limit.
+	Bytes int `mapstructure:"bytes"`
+
+	// Interval configures the max age of a document in the send buffer.
+	Interval time.Duration `mapstructure:"interval"`
+}
+
+// RetrySettings defines settings for the HTTP request retries in the OpenSearch exporter.
+// Failed sends are retried with exponential backoff.
+type RetrySettings struct {
+	// Enabled allows users to disable retry without having to comment out all settings.
+	Enabled bool `mapstructure:"enabled"`
+
+	// MaxRequests configures how often an HTTP request is retried before it is assumed to be failed.
+	MaxRequests int `mapstructure:"max_requests"`
+
+	// InitialInterval configures the initial waiting time if a request failed.
+	InitialInterval time.Duration `mapstructure:"initial_interval"`
+
+	// MaxInterval configures the max waiting time if consecutive requests failed.
+	MaxInterval time.Duration `mapstructure:"max_interval"`
+}
+
+type MappingsSettings struct {
+	// Mode configures the field mappings.
+	Mode string `mapstructure:"mode"`
+
+	// Try to find and remove duplicate fields
+	Dedup bool `mapstructure:"dedup"`
+
+	Dedot bool `mapstructure:"dedot"`
+}
+
+var (
+	errConfigNoEndpoint    = errors.New("endpoints must be specified")
+	errConfigEmptyEndpoint = errors.New("endpoints must not include empty entries")
+)
+
+// Validate validates the opensearch server configuration.
+func (cfg *Config) Validate() error {
+	if len(cfg.Endpoints) == 0 {
+		if os.Getenv(defaultOpenSearchEnvName) == "" {
+			return errConfigNoEndpoint
+		}
+	}
+
+	for _, endpoint := range cfg.Endpoints {
+		if endpoint == "" {
+			return errConfigEmptyEndpoint
+		}
+	}
+
+	return nil
+}

exporter/opensearchexporter/factory.go

@@ -0,0 +1,58 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package opensearchexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/opensearchexporter"
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/exporter"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
+	"go.opentelemetry.io/collector/pdata/ptrace"
+)
+
+const (
+	// The value of "type" key in configuration.
+	typeStr = "opensearch"
+	// The stability level of the exporter.
+	stability = component.StabilityLevelDevelopment
+)
+
+// NewFactory creates a factory for OpenSearch exporter.
+func NewFactory() exporter.Factory {
+	return exporter.NewFactory(
+		typeStr,
+		createDefaultConfig,
+		exporter.WithTraces(createTracesExporter, stability),
+	)
+}
+
+func createDefaultConfig() component.Config {
+	return &Config{
+		HTTPClientSettings: HTTPClientSettings{
+			Timeout: 90 * time.Second,
+		},
+		Namespace: "",
+		Dataset:   "",
+		Retry: RetrySettings{
+			Enabled:         true,
+			MaxRequests:     3,
+			InitialInterval: 100 * time.Millisecond,
+			MaxInterval:     1 * time.Minute,
+		},
+	}
+}
+
+func createTracesExporter(ctx context.Context,
+	set exporter.CreateSettings,
+	cfg component.Config) (exporter.Traces, error) {
+
+	return exporterhelper.NewTracesExporter(ctx, set, cfg, func(ctx context.Context, ld ptrace.Traces) error {
+		return nil
+	},
+		exporterhelper.WithShutdown(func(ctx context.Context) error {
+			return nil
+		}))
+}

exporter/opensearchexporter/go.mod

@@ -0,0 +1,47 @@
+module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/opensearchexporter
+
+go 1.19
+
+require (
+	go.opentelemetry.io/collector/component v0.81.0
+	go.opentelemetry.io/collector/config/configopaque v0.81.0
+	go.opentelemetry.io/collector/config/configtls v0.81.0
+	go.opentelemetry.io/collector/exporter v0.81.0
+	go.opentelemetry.io/collector/pdata v1.0.0-rcv0013
+)
+
+require (
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/knadh/koanf v1.5.0 // indirect
+	github.com/knadh/koanf/v2 v2.0.1 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/collector v0.81.0 // indirect
+	go.opentelemetry.io/collector/config/configtelemetry v0.81.0 // indirect
+	go.opentelemetry.io/collector/confmap v0.81.0 // indirect
+	go.opentelemetry.io/collector/consumer v0.81.0 // indirect
+	go.opentelemetry.io/collector/extension v0.81.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.0.0-rcv0013 // indirect
+	go.opentelemetry.io/collector/processor v0.81.0 // indirect
+	go.opentelemetry.io/collector/receiver v0.81.0 // indirect
+	go.opentelemetry.io/otel v1.16.0 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/net v0.11.0 // indirect
+	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/text v0.10.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/grpc v1.56.1 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+)