diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7acbc70212ef..badeb51d484f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,7 @@ stages: - manaul-jobs - build - push + - kafka-poc-deploy - staging-deploy - prod-deploy @@ -81,6 +82,22 @@ build-ci-image: # # For debugging # - aws sts get-caller-identity - bash $SCRIPT $NAMESPACE $VALUES +kafka-poc-deploy: + stage: kafka-poc-deploy + tags: ["runner:docker", "size:large"] + image: $CI_IMAGE + dependencies: + - push-image-staging + rules: + - if: '$CI_COMMIT_REF_NAME =~ /-kafka-poc$/' + script: + - >- + TEMP_AWS_ACCESS_KEY_ID=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.sand-eks-deploy-api-key --with-decryption --query Parameter.Value --out text) + - >- + TEMP_AWS_SECRET_ACCESS_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.sand-eks-deploy-access-key --with-decryption --query Parameter.Value --out text) + - export AWS_ACCESS_KEY_ID=$TEMP_AWS_ACCESS_KEY_ID + - export AWS_SECRET_ACCESS_KEY=$TEMP_AWS_SECRET_ACCESS_KEY + - bash ./ci/scripts/kafka-poc.sh staging-deploy-demo-eks: !!merge <<: *staging-deploy variables: diff --git a/ci/Dockerfile b/ci/Dockerfile index 56b68654a7c8..0e023cecbbba 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -12,7 +12,7 @@ RUN clean-apt install make \ RUN apt-get update && \ apt-get -y install default-jre-headless -ADD https://github.com/open-telemetry/opentelemetry-java-contrib/releases/download/v1.27.0/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar +ADD ./ci/opentelemetry-jmx-metrics-1.33.0-alpha-SNAPSHOT.jar /opt/opentelemetry-jmx-metrics.jar COPY --from=build /src/bin/otelcontribcol_* /otelcol-contrib USER 1001 diff --git a/ci/opentelemetry-jmx-metrics-1.33.0-alpha-SNAPSHOT.jar b/ci/opentelemetry-jmx-metrics-1.33.0-alpha-SNAPSHOT.jar new file mode 100644 index 000000000000..ee0e5603fac4 Binary files /dev/null and b/ci/opentelemetry-jmx-metrics-1.33.0-alpha-SNAPSHOT.jar differ diff --git a/ci/scripts/kafka-poc.sh b/ci/scripts/kafka-poc.sh new file mode 100644 index 000000000000..4a00edfe44a2 --- /dev/null +++ b/ci/scripts/kafka-poc.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# This script is used to deploy collector on demo account cluster + +set -euo pipefail +IFS=$'\n\t' +set -x + +install_collector() { + release_name="opentelemetry-collector" + + # if repo already exists, helm 3+ will skip + helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts + helm repo update open-telemetry + # --install will run `helm install` if not already present. + helm --debug upgrade "${release_name}" -n "kafka-poc" open-telemetry/opentelemetry-collector --install \ + -f ./ci/values.yaml \ + -f "./ci/values-kafka-poc.yaml" \ + --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \ + --set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib" + helm list --all-namespaces + + install_deployment +} + +install_deployment() { + release_name_deployment="opentelemetry-collector-deployment" + + # --install collector that fetches jmx metrics. The jmx receiver cannot be used in the daemonset deployment + # as this would lead to duplicate metrics. + helm --debug upgrade "${release_name_deployment}" -n "kafka-poc" open-telemetry/opentelemetry-collector --install \ + -f ./ci/values-jmx.yaml \ + --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \ + --set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib" \ + --set nodeSelector.alpha\\.eksctl\\.io/nodegroup-name=kafka-poc-ng + +} + +########################################################################################################### +clusterName="dd-otel" +clusterArn="arn:aws:eks:us-east-1:601427279990:cluster/${clusterName}" + +aws eks --region us-east-1 update-kubeconfig --name "${clusterName}" +kubectl config use-context "${clusterArn}" + +install_collector diff --git a/ci/values-jmx.yaml b/ci/values-jmx.yaml index b44b566924e5..1d51567a5c14 100644 --- a/ci/values-jmx.yaml +++ b/ci/values-jmx.yaml @@ -15,7 +15,7 @@ extraEnvs: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: http://$(OTEL_COLLECTOR_NAME):4317 - name: OTEL_RESOURCE_ATTRIBUTES - value: kafka_source=deploymentcol + value: kafka_source=otelcol config: receivers: jmx: diff --git a/ci/values-kafka-poc.yaml b/ci/values-kafka-poc.yaml new file mode 100644 index 000000000000..2239a5bb2123 --- /dev/null +++ b/ci/values-kafka-poc.yaml @@ -0,0 +1,6 @@ +nodeSelector: + alpha.eksctl.io/nodegroup-name: kafka-poc-ng +clusterRole: + name: "opentelemetry-collector-kafka" + clusterRoleBinding: + name: opentelemetry-collector-kafka \ No newline at end of file diff --git a/ci/values.yaml b/ci/values.yaml index 237af780c9fc..f8d3bf7155c0 100644 --- a/ci/values.yaml +++ b/ci/values.yaml @@ -90,27 +90,46 @@ config: attributes: actions: - key: log.file.path - pattern: \/var\/log\/pods\/otel_opentelemetry-demo-(?P.*?)-.* + pattern: \/var\/log\/pods\/${env:OTEL_K8S_NAMESPACE}_opentelemetry-demo-(?P.*?)-.* action: extract - key: log.file.path - pattern: \/var\/log\/pods\/otel_opentelemetry-demo-(?P.*?)-.* + pattern: \/var\/log\/pods\/${env:OTEL_K8S_NAMESPACE}_opentelemetry-demo-(?P.*?)-.* action: extract - key: service from_attribute: service_name action: upsert attributes/kafkasrc: include: - match_type: regexp + match_type: strict attributes: - - {key: "source", value: 'kafka|frauddetectionservice|orderproducer'} + - key: "source" + value: 'kafka' actions: # this makes sure the source for logs that come from kafka|frauddetectionservice|orderproducer is - # set to kafka. The OOTB box kafka dashboard filters logs on source:kafka. This will still not work - # as it will be a log attribute and can only be searched by @source:kafka. We would need to support - # attributes_as_tags or find another solution for this to work. - - key: source - action: update - value: "kafka" + # set to kafka. The OOTB box kafka dashboard filters logs on source:kafka. + - key: ddtags + action: upsert + value: "source:kafka" + attributes/producer: + include: + match_type: strict + attributes: + - key: "source" + value: 'orderproducer' + actions: + - key: ddtags + action: upsert + value: "source:kafka" + attributes/consumer: + include: + match_type: strict + attributes: + - key: "source" + value: 'frauddetectionservice' + actions: + - key: ddtags + action: upsert + value: "source:kafka" memory_limiter: check_interval: 1s limit_mib: 500 @@ -192,5 +211,5 @@ config: processors: [resourcedetection, k8sattributes, batch] exporters: [datadog] logs: - processors: [memory_limiter, resourcedetection, k8sattributes, attributes, attributes/kafkasrc, batch] + processors: [memory_limiter, resourcedetection, k8sattributes, batch, attributes, attributes/kafkasrc, attributes/producer, attributes/consumer] exporters: [datadog] diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod index 4bb0921bf915..74039cdd9693 100644 --- a/cmd/otelcontribcol/go.mod +++ b/cmd/otelcontribcol/go.mod @@ -275,8 +275,9 @@ require ( github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.8.1 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.8.3 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.8.3 // indirect - github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.8.1 // indirect - github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.8.1 // indirect + // https://github.com/DataDog/opentelemetry-mapping-go/pull/217 + github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.8.4-0.20231214134201-f0bd121d625d // indirect + github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.8.3 // indirect github.com/DataDog/sketches-go v1.4.3 // indirect github.com/DataDog/zstd v1.5.2 // indirect github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 // indirect diff --git a/cmd/otelcontribcol/go.sum b/cmd/otelcontribcol/go.sum index 4e12c1d0eb6f..852865fe30c9 100644 --- a/cmd/otelcontribcol/go.sum +++ b/cmd/otelcontribcol/go.sum @@ -185,15 +185,15 @@ github.com/DataDog/gohai v0.0.0-20220718130825-1776f9beb9cc h1:gtlKB6B50/UEuFm1L github.com/DataDog/gohai v0.0.0-20220718130825-1776f9beb9cc/go.mod h1:oyPC4jWHHjVVNjslDAKp8EqfQBaSmODjHt4HCX+C+9Q= github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.8.1 h1:16YkGLmeJH830m3dG0owtdSMXDkmTZdRd1EWZ1XIO6M= github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.8.1/go.mod h1:re1FCVuKWovntr4bODIUe+iYT2Al0gbS0WK726XLz7w= -github.com/DataDog/opentelemetry-mapping-go/pkg/internal/sketchtest v0.8.1 h1:0GyngySyxePf1RozAdm8p51uxT/kDsxqhzWPZGNrqVg= +github.com/DataDog/opentelemetry-mapping-go/pkg/internal/sketchtest v0.8.3 h1:x0Hc8zoGaw3JzvA9/j4BYwX+LTDeZvzR1DAtLmZWOhw= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.8.3 h1:oFW083L4oIyrYL06/3yCU5lnslaj8haSmQidFDK9rQk= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.8.3/go.mod h1:8jOAAH5PrNN6zICpu65M7mNPAeOQsW5Wk/sq80txutA= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.8.3 h1:Kp4byxXZG8HmrbCH6rVhq/yWnW/ddKt0TqtjWzlj4N0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.8.3/go.mod h1:oN2FZ3z4WiBkfsexz6Uc/IU+yjlQ+xrMpDmVnj8fF/Y= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.8.1 h1:TbZDzUrW8iahevDY5Xw+j1EfrX8XZdeg/M6UYBl9mxg= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.8.1/go.mod h1:nVIKiAwP/Zd8IFxeHhEAiirJRH1kOOoZ315HQeSEkng= -github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.8.1 h1:4cyzwoXrnjElTIlsFbpyXQkAJs3+6ly9HExOu9lrup4= -github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.8.1/go.mod h1:3AEt8/uCtbpzjPOPPxIBDcR1qcUq2hBbrkrdE/qNfb0= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.8.4-0.20231214134201-f0bd121d625d h1:iBOfirvCTBmyOIryFV4kF6XRy8Hv9RBS0sn9yzkz0uI= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.8.4-0.20231214134201-f0bd121d625d/go.mod h1:E80nIOZcDOSCC/oc95EaNm5LxXmihI35hGSC7wvSSME= +github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.8.3 h1:3iLIuB1c7+uBJJ2dWOTLdwrdWmBQLfpa98oiSFCW52I= +github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.8.3/go.mod h1:Fq3Xmw5wsdm1rui0k9UiYZmlGLR+JnoZ2L4zhLTb0uo= github.com/DataDog/sketches-go v1.4.3 h1:ZB9nijteJRFUQixkQfatCqASartGNfiolIlMiEv3u/w= github.com/DataDog/sketches-go v1.4.3/go.mod h1:XR0ns2RtEEF09mDKXiKZiQg+nfZStrq1ZuL1eezeZe0= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= diff --git a/receiver/jmxreceiver/config.go b/receiver/jmxreceiver/config.go index 3d755992f542..289fbc1b6f31 100644 --- a/receiver/jmxreceiver/config.go +++ b/receiver/jmxreceiver/config.go @@ -183,6 +183,9 @@ func (c *Config) validateJar(supportedJarDetails map[string]supportedJar, jar st return fmt.Errorf("error hashing file: %w", err) } + // Skip hash validation for POC. + return nil + jarDetails, ok := supportedJarDetails[hash] if !ok { return errors.New("jar hash does not match known versions")