[exporter/datasetexporter]: Initial Commit

.chloggen/add-datasetexporter.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datasetexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add new DataSet exporter.
+
+# One or more tracking issues related to the change
+issues: [20660]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

.github/CODEOWNERS

@@ -47,6 +47,7 @@ exporter/clickhouseexporter/                             @open-telemetry/collect
 exporter/cassandraexporter/                              @open-telemetry/collector-contrib-approvers @atoulme @emreyalvac
 exporter/coralogixexporter/                              @open-telemetry/collector-contrib-approvers @oded-dd @povilasv @matej-g
 exporter/datadogexporter/                                @open-telemetry/collector-contrib-approvers @mx-psi @gbbr @dineshg13 @liustanley @songy23
+exporter/datasetexporter/                                @open-telemetry/collector-contrib-approvers @atoulme @martin-majlis-s1
 exporter/dynatraceexporter/                              @open-telemetry/collector-contrib-approvers @dyladan @arminru @evan-bradley
 exporter/elasticsearchexporter/                          @open-telemetry/collector-contrib-approvers @JaredTan95
 exporter/f5cloudexporter/                                @open-telemetry/collector-contrib-approvers @gramidt

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -41,6 +41,7 @@ body:
       - exporter/clickhouse
       - exporter/coralogix
       - exporter/datadog
+      - exporter/dataset
       - exporter/dynatrace
       - exporter/elasticsearch
       - exporter/f5cloud

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -35,6 +35,7 @@ body:
       - exporter/clickhouse
       - exporter/coralogix
       - exporter/datadog
+      - exporter/dataset
       - exporter/dynatrace
       - exporter/elasticsearch
       - exporter/f5cloud

.github/ISSUE_TEMPLATE/other.yaml

@@ -35,6 +35,7 @@ body:
       - exporter/clickhouse
       - exporter/coralogix
       - exporter/datadog
+      - exporter/dataset
       - exporter/dynatrace
       - exporter/elasticsearch
       - exporter/f5cloud

.github/dependabot.yml

@@ -127,6 +127,11 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
+  - package-ecosystem: "gomod"
+    directory: "/exporter/datasetexporter"
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
   - package-ecosystem: "gomod"
     directory: "/exporter/dynatraceexporter"
     schedule:
@@ -1097,8 +1102,3 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
-  - package-ecosystem: "gomod"
-    directory: "/receiver/zipkinreceiver"
-    schedule:
-      interval: "weekly"
-      day: "wednesday"

cmd/otelcontribcol/builder-config.yaml

@@ -42,6 +42,7 @@ exporters:
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/cassandraexporter v0.76.3
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/coralogixexporter v0.76.3
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter v0.76.3
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datasetexporter v0.76.3
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/dynatraceexporter v0.76.3
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter v0.76.3
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/f5cloudexporter v0.76.3
@@ -321,6 +322,7 @@ replaces:
   - github.com/open-telemetry/opentelemetry-collector-contrib/processor/routingprocessor => ../../processor/routingprocessor
   - github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatorateprocessor => ../../processor/deltatorateprocessor
   - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter => ../../exporter/datadogexporter
+  - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datasetexporter => ../../exporter/datasetexporter
   - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => ../../internal/aws/containerinsight
   - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => ../../internal/aws/awsutil
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zookeeperreceiver => ../../receiver/zookeeperreceiver

cmd/otelcontribcol/exporters_test.go

@@ -46,6 +46,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/coralogixexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datasetexporter"
 	dtconf "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/dynatraceexporter/config"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/f5cloudexporter"
@@ -335,6 +336,15 @@ func TestDefaultExporters(t *testing.T) {
 				return cfg
 			},
 		},
+		{
+			exporter: "dataset",
+			getConfigFn: func() component.Config {
+				cfg := expFactories["dataset"].CreateDefaultConfig().(*datasetexporter.Config)
+				cfg.DatasetURL = "https://" + endpoint
+				cfg.APIKey = "secret-key"
+				return cfg
+			},
+		},
 		{
 			exporter: "dynatrace",
 			getConfigFn: func() component.Config {

cmd/otelcontribcol/go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter v0.76.3
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/coralogixexporter v0.76.3
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter v0.76.3
+	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datasetexporter v0.76.3
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/dynatraceexporter v0.76.3
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter v0.76.3
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/f5cloudexporter v0.76.3
@@ -926,6 +927,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/delt
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter => ../../exporter/datadogexporter
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datasetexporter => ../../exporter/datasetexporter
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => ../../internal/aws/containerinsight
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => ../../internal/aws/awsutil

exporter/datasetexporter/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

exporter/datasetexporter/README.md

@@ -0,0 +1,62 @@
+# DataSet Exporter
+
+<!-- status autogenerated section -->
+| Status                   |           |
+| ------------------------ |-----------|
+| Stability                | [development]   |
+| Supported pipeline types | logs, traces   |
+| Distributions            | [contrib] |
+
+[development]: https://github.com/open-telemetry/opentelemetry-collector#development
+[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
+<!-- end autogenerated section -->
+
+This exporter sends logs to [DataSet](https://www.dataset.com/).
+
+See the [Getting Started](https://app.scalyr.com/help/getting-started) guide.
+
+## Configuration
+
+### Required Settings
+
+- `dataset_url` (no default): The URL of the DataSet API that ingests the data. Most likely https://app.scalyr.com. If not specified env variable `DATASET_URL` is used.
+- `api_key` (no default): The "Log Write" API Key required to use API. Instructions how to get [API key](https://app.scalyr.com/help/api-keys). If not specified env variable `DATASET_API_KEY` is used.
+
+### Optional Settings
+
+- `max_delay_ms` (default = "15000"): The maximum delay between sending batches from the same source.
+- `group_by` (default = []): The list of attributes based on which events should be grouped.
+- `retry_on_failure`: See [retry_on_failure](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md)
+- `sending_queue`: See [sending_queue](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md)
+- `timeout`: See [timeout](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md)
+
+
+### Example
+
+```yaml
+
+exporters:
+  dataset:
+    # DataSet API URL
+    dataset_url: https://app.scalyr.com
+    # API Key
+    api_key: your_api_key
+    # Send batch to the API at least every 15s
+    max_delay_ms: 15000
+    # Group data based on these attributes
+    group_by:
+      - attributes.container_id
+      - attributes.log.file.path
+      - body.map.kubernetes.container_hash
+      - body.map.kubernetes.pod_id
+      - body.map.kubernetes.docker_id
+      - body.map.stream
+
+service:
+  pipelines:
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      # add dataset among your exporters
+      exporters: [dataset]
+```

exporter/datasetexporter/config.go

@@ -0,0 +1,91 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package datasetexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datasetexporter"
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	"go.opentelemetry.io/collector/confmap"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
+)
+
+const MaxDelayMs = "15000"
+
+type Config struct {
+	DatasetURL                     string   `mapstructure:"dataset_url"`
+	APIKey                         string   `mapstructure:"api_key"`
+	MaxDelayMs                     string   `mapstructure:"max_delay_ms"`
+	GroupBy                        []string `mapstructure:"group_by"`
+	exporterhelper.RetrySettings   `mapstructure:"retry_on_failure"`
+	exporterhelper.QueueSettings   `mapstructure:"sending_queue"`
+	exporterhelper.TimeoutSettings `mapstructure:"timeout"`
+}
+
+func (c *Config) Unmarshal(conf *confmap.Conf) error {
+	if err := conf.Unmarshal(c, confmap.WithErrorUnused()); err != nil {
+		return fmt.Errorf("cannot unmarshal config: %w", err)
+	}
+
+	if len(c.DatasetURL) == 0 {
+		c.DatasetURL = os.Getenv("DATASET_URL")
+	}
+	if len(c.APIKey) == 0 {
+		c.APIKey = os.Getenv("DATASET_API_KEY")
+	}
+
+	if len(c.MaxDelayMs) == 0 {
+		c.MaxDelayMs = MaxDelayMs
+	}
+
+	return nil
+}
+
+// Validate checks if all required fields in Config are set and have valid values.
+// If any of the required fields are missing or have invalid values, it returns an error.
+func (c *Config) Validate() error {
+	if c.APIKey == "" {
+		return fmt.Errorf("api_key is required")
+	}
+	if c.DatasetURL == "" {
+		return fmt.Errorf("dataset_url is required")
+	}
+
+	_, err := strconv.Atoi(c.MaxDelayMs)
+	if err != nil {
+		return fmt.Errorf(
+			"max_delay_ms must be integer, but %s was used: %w",
+			c.MaxDelayMs,
+			err,
+		)
+	}
+
+	return nil
+}
+
+// String returns a string representation of the Config object.
+// It includes all the fields and their values in the format "field_name: field_value".
+func (c *Config) String() string {
+	s := ""
+	s += fmt.Sprintf("%s: %s; ", "DatasetURL", c.DatasetURL)
+	s += fmt.Sprintf("%s: %s; ", "MaxDelayMs", c.MaxDelayMs)
+	s += fmt.Sprintf("%s: %s; ", "GroupBy", c.GroupBy)
+	s += fmt.Sprintf("%s: %+v; ", "RetrySettings", c.RetrySettings)
+	s += fmt.Sprintf("%s: %+v; ", "QueueSettings", c.QueueSettings)
+	s += fmt.Sprintf("%s: %+v", "TimeoutSettings", c.TimeoutSettings)
+
+	return s
+}