.gitlab-ci.yml

default:
  image: 'docker.repo.splunkdev.net/ci-cd/ci-container:debian-buster'

stages:
  - sast-oss-scan
  - build
  - sign
  - package
  - cve-scan
  - release
  - choco-release
  - github-release

include:
  # https://cd.splunkdev.com/prodsec/scp-scanning/gitlab-checkmarx
  - project: 'prodsec/scp-scanning/gitlab-checkmarx'
    ref: latest
    file: '/templates/.sast_scan.yml'
  # https://cd.splunkdev.com/prodsec/whitesource
  - project: 'prodsec/whitesource'
    ref: latest
    file: '/templates/.whitesource.yml'

semgrep:
  stage: sast-oss-scan
  extends: .sast_scan
  retry: 2
  variables:
    SAST_SCANNER: "Semgrep"
    SEMGREP_EXCLUDE: "examples,internal/buildscripts,tests,*_test.go,cmd/otelcol/Dockerfile.windows,deployments/ansible/molecule"
    alert_mode: "policy"
  after_script:
    - echo "Check results at $CI_PIPELINE_URL/security"
  only:
    - main
    - schedules

whitesource:
  extends: .whitesource
  stage: sast-oss-scan
  retry: 2
  variables:
    ws_user_key: ${WHITESOURCE_USER_KEY}
    product_id: ${WHITESOURCE_ID}
    config: go-modules
    config_overlay: $CI_PROJECT_DIR/.whitesource.conf
  only:
    - main
    - schedules
  # allow_failure: false
  cache:
    key: "whitesource-go1.17.2"
    paths:
      - /usr/local/go
      - /go/pkg/mod
  before_script:
    - mkdir -p /lib64
    - ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2
    - |
      set -e
      if [ ! -d /usr/local/go ]; then
        pushd /usr/local
        wget https://golang.org/dl/go1.17.2.linux-amd64.tar.gz
        tar xzf go1.17.2.linux-amd64.tar.gz
        popd
      fi
    - export GOROOT=/usr/local/go
    - export GOPATH=/go
    - export PATH=${GOROOT}/bin:$PATH
  script:
    - bash /whitesource/templates/template-gitlab.sh
    - |
      # Check the report and fail if high vulns or rejected components were found
      set -e
      report="whitesource-results/json-artifacts/report_metadata.json"
      if [ -f "$report" ]; then
        high_vulns="$( jq '."Total High Vulns"' $report )"
        if [ "$high_vulns" = "null" ]; then
          echo "Failed to get 'Total High Vulns' from $report"
          exit 1
        fi
        rejected="$( jq '."Total Components Rejected"' $report )"
        if [ "$rejected" = "null" ]; then
          echo "Failed to get 'Total Components Rejected' from $report"
          exit 1
        fi
        if [ "$high_vulns" -gt 0 ] || [ "$rejected" -gt 0 ]; then
          exit 1
        fi
      else
        echo "$report not found"
        exit 1
      fi

.go-cache:
  image: 'docker-hub.repo.splunkdev.net/golang:1.17.2'
  variables:
    GOPATH: "$CI_PROJECT_DIR/.go"
  before_script:
    - mkdir -p $GOPATH
    - make install-tools
    - export PATH=$GOPATH/bin:$PATH
  cache:
    key: "go-cache"
    paths:
      - .go/pkg/mod
      - .go/bin

compile:
  extends: .go-cache
  only:
    variables:
      - $CI_COMMIT_BRANCH == "main"
      - $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - schedules
  stage: build
  parallel:
    matrix:
      - TARGET: [binaries-darwin_amd64, binaries-linux_amd64, binaries-linux_arm64, binaries-windows_amd64]
  script: make $TARGET
  after_script:
    - if [ -e bin/otelcol ]; then rm -f bin/otelcol; fi  # remove the symlink
    - if [ -e bin/translatesfx ]; then rm -f bin/translatesfx; fi  # remove the symlink
    - if [ -e bin/migratecheckpoint ]; then rm -f bin/migratecheckpoint; fi  # remove the symlink
  artifacts:
    paths:
      - bin/otelcol_*
      - bin/translatesfx_*
      - bin/migratecheckpoint_*

.sign-release-cache:
  image: 'docker.repo.splunkdev.net/ci-cd/ci-container:python-3.9'
  variables:
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  cache:
    key: "sign-release-pip-cache"
    paths:
      - .cache/pip
  before_script:
    - pip3 install -r internal/buildscripts/packaging/release/requirements.txt

.sign-release:
  only:
    variables:
      - $CI_COMMIT_BRANCH == "main"
      - $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - schedules
  retry: 2
  script:
    - |
      set -ex
      STAGE="test"
      if [[ "${CI_COMMIT_TAG:-}" =~ beta ]]; then
        STAGE="beta"
      elif [[ "${CI_COMMIT_TAG:-}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
        STAGE="release"
      fi
      for path in ${PATHS:-}; do
        if [ ! -f "$path" ]; then
          echo "$path not found!"
          exit 1
        fi
        python3 internal/buildscripts/packaging/release/sign_release.py --force --stage=$STAGE --path=$path ${OPTIONS:-}
      done

sign-exe:
  extends:
    - .sign-release-cache
    - .sign-release
  stage: sign
  parallel:
    matrix:
      - TARGET: [otelcol, translatesfx]
  variables:
    PATHS: bin/${TARGET}_windows_amd64.exe
    OPTIONS: "--no-push"
  after_script:
    - mkdir -p dist/signed
    - mv bin/signed/${TARGET}_windows_amd64.exe dist/signed/
  artifacts:
    paths:
      - dist/signed/${TARGET}_windows_amd64.exe

sign-osx:
  extends:
    - .sign-release-cache
    - .sign-release
  stage: sign
  parallel:
    matrix:
      - TARGET: [otelcol, translatesfx]
  variables:
    PATHS: bin/${TARGET}_darwin_amd64
    OPTIONS: "--no-push"
  after_script:
    - mkdir -p dist/signed
    - mv bin/signed/${TARGET}_darwin_amd64 dist/signed/
  artifacts:
    paths:
      - dist/signed/${TARGET}_darwin_amd64

build-linux-image:
  only:
    variables:
      - $CI_COMMIT_BRANCH == "main"
      - $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - schedules
  stage: package
  dependencies:
    - compile
  parallel:
    matrix:
      - ARCH: [amd64, arm64]
  retry: 2
  script:
    - make docker-otelcol ARCH=${ARCH} SKIP_COMPILE=true
    - arch=$( docker inspect --format='{{.Architecture}}' otelcol:${ARCH} )
    - if [[ "$arch" != "$ARCH" ]]; then exit 1; fi
  after_script:
    - mkdir -p dist
    - docker save -o dist/otelcol-${ARCH}.tar otelcol:${ARCH}
  artifacts:
    paths:
      - dist/otelcol-*.tar

.build-deb-rpm:
  only:
    variables:
      - $CI_COMMIT_BRANCH == "main"
      - $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - schedules
  stage: package
  dependencies:
    - compile
  parallel:
    matrix:
      - ARCH: [amd64, arm64]
  before_script:
    - ./internal/buildscripts/packaging/fpm/install-deps.sh
  script:
    - ./internal/buildscripts/packaging/fpm/${PKG_TYPE}/build.sh "${CI_COMMIT_TAG:-}" "$ARCH" "./dist"

build-deb:
  extends: .build-deb-rpm
  variables:
    PKG_TYPE: deb
  artifacts:
    paths:
      - dist/*.deb

build-rpm:
  extends: .build-deb-rpm
  variables:
    PKG_TYPE: rpm
  artifacts:
    paths:
      - dist/*.rpm

build-msi:
  only:
    variables:
      - $CI_COMMIT_BRANCH == "main"
      - $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - schedules
  stage: package
  dependencies:
    - sign-exe
  before_script:
    # build the MSI with the signed exe
    - mkdir -p bin
    - cp -f dist/signed/otelcol_windows_amd64.exe bin/otelcol_windows_amd64.exe
    - cp -f dist/signed/translatesfx_windows_amd64.exe bin/translatesfx_windows_amd64.exe
  script:
    - make msi SKIP_COMPILE=true VERSION=${CI_COMMIT_TAG:-}
  artifacts:
    paths:
      - dist/*.msi

push-linux-image:
  only:
    variables:
      - $CI_COMMIT_BRANCH == "main"
      - $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - schedules
  stage: release
  dependencies:
    - build-linux-image
  retry: 2
  before_script:
    - docker load -i dist/otelcol-amd64.tar
    - docker load -i dist/otelcol-arm64.tar
  script:
    - docker login -u $CIRCLECI_QUAY_USERNAME -p $CIRCLECI_QUAY_PASSWORD quay.io
    - |
      # Set env vars
      set -e
      if [[ -n "${CI_COMMIT_TAG:-}" ]]; then
        IMAGE_NAME="quay.io/signalfx/splunk-otel-collector"
        MANIFEST_TAG=${CI_COMMIT_TAG#v}
      else
        IMAGE_NAME="quay.io/signalfx/splunk-otel-collector-dev"
        MANIFEST_TAG=$CI_COMMIT_SHA
      fi
    - |
      # Tag and push images
      set -e
      for arch in "amd64" "arm64"; do
        ARCH_TAG="${MANIFEST_TAG}-${arch}"
        echo "Tagging and pushing ${IMAGE_NAME}:${ARCH_TAG}"
        docker tag otelcol:${arch} ${IMAGE_NAME}:${ARCH_TAG}
        docker push ${IMAGE_NAME}:${ARCH_TAG}
        if [[ "${CI_COMMIT_BRANCH:-}" = "main" ]] || [[ "${CI_COMMIT_TAG:-}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
          # only push latest tag for main and stable releases
          LATEST_TAG="latest-${arch}"
          echo "Tagging and pushing ${IMAGE_NAME}:${LATEST_TAG}"
          docker tag ${IMAGE_NAME}:${ARCH_TAG} ${IMAGE_NAME}:${LATEST_TAG}
          docker push ${IMAGE_NAME}:${LATEST_TAG}
        fi
      done
    - |
      # Create and push image manifests
      set -e
      echo "Creating and pushing ${IMAGE_NAME}:${MANIFEST_TAG} manifest"
      docker manifest create ${IMAGE_NAME}:${MANIFEST_TAG} --amend ${IMAGE_NAME}:${MANIFEST_TAG}-amd64 --amend ${IMAGE_NAME}:${MANIFEST_TAG}-arm64
      docker manifest push ${IMAGE_NAME}:${MANIFEST_TAG}
      if [[ "${CI_COMMIT_BRANCH:-}" = "main" ]] || [[ "${CI_COMMIT_TAG:-}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
        # only push latest manifest tag for main and stable releases
        echo "Creating and pushing ${IMAGE_NAME}:latest manifest"
        docker manifest create ${IMAGE_NAME}:latest --amend ${IMAGE_NAME}:latest-amd64 --amend ${IMAGE_NAME}:latest-arm64
        docker manifest push ${IMAGE_NAME}:latest
      fi
    - docker pull ${IMAGE_NAME}:${MANIFEST_TAG}
    - docker inspect --format='{{.RepoDigests}}' ${IMAGE_NAME}:${MANIFEST_TAG} | tee dist/linux_digest.txt
    - docker manifest inspect ${IMAGE_NAME}:${MANIFEST_TAG} | tee dist/manifest_digest.txt
  artifacts:
    paths:
      - dist/linux_digest.txt
      - dist/manifest_digest.txt

build-push-windows-image:
  only:
    variables:
      - $CI_COMMIT_BRANCH == "main"
      - $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - schedules
  stage: release
  dependencies:
    - sign-exe
  tags:
    - windows
  retry: 2
  before_script:
    - Copy-Item .\dist\signed\otelcol_windows_amd64.exe .\cmd\otelcol\otelcol.exe
    - Copy-Item .\dist\signed\translatesfx_windows_amd64.exe .\cmd\otelcol\translatesfx.exe
  script:
    - docker login -u $env:CIRCLECI_QUAY_USERNAME -p $env:CIRCLECI_QUAY_PASSWORD quay.io
    - |
      $ErrorActionPreference = 'Stop'
      if ($env:CI_COMMIT_TAG) {
        $IMAGE_NAME = "quay.io/signalfx/splunk-otel-collector-windows"
        $IMAGE_TAG = $env:CI_COMMIT_TAG.TrimStart("v")
      } else {
        $IMAGE_NAME = "quay.io/signalfx/splunk-otel-collector-windows-dev"
        $IMAGE_TAG = $env:CI_COMMIT_SHA
      }
      $SMART_AGENT_RELEASE = $((Get-Content internal\buildscripts\packaging\smart-agent-release.txt).TrimStart("v"))
      echo "Building ${IMAGE_NAME}:${IMAGE_TAG}"
      docker build -t ${IMAGE_NAME}:${IMAGE_TAG} --build-arg SMART_AGENT_RELEASE=${SMART_AGENT_RELEASE} -f .\cmd\otelcol\Dockerfile.windows .\cmd\otelcol\
      echo "Pushing ${IMAGE_NAME}:${IMAGE_TAG}"
      docker push ${IMAGE_NAME}:${IMAGE_TAG}
      if ($env:CI_COMMIT_BRANCH -eq "main" -or $env:CI_COMMIT_TAG -match '^v\d+\.\d+\.\d+$') {
        # only push latest tag for main and stable releases
        echo "Tagging and pushing ${IMAGE_NAME}:latest"
        docker tag ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:latest
        docker push ${IMAGE_NAME}:latest
      }
    - docker inspect --format='{{.RepoDigests}}' ${IMAGE_NAME}:${IMAGE_TAG} | Tee-Object -FilePath dist/windows_digest.txt
  after_script:
    - docker image prune --all --force
  artifacts:
    paths:
      - dist/windows_digest.txt

release-debs:
  extends:
    - .sign-release-cache
    - .sign-release
  stage: release
  dependencies:
    - build-deb
  variables:
    PATHS: dist/*.deb
  after_script:
    # stage the debs to be picked up by the github-release job
    - mkdir -p dist/signed
    - cp dist/*.deb dist/signed/
  artifacts:
    paths:
      - dist/signed/*.deb

release-rpms:
  extends:
    - .sign-release-cache
    - .sign-release
  stage: release
  dependencies:
    - build-rpm
  variables:
    PATHS: dist/*.rpm
  artifacts:
    paths:
      - dist/signed/*.rpm

# only sign the MSI; disable push to S3 until we get a token
release-msi:
  extends:
    - .sign-release-cache
    - .sign-release
  stage: release
  dependencies:
    - build-msi
  variables:
    PATHS: dist/*.msi
    OPTIONS: "--no-push"
  artifacts:
    paths:
      - dist/signed/*.msi

choco-release:
  only:
    variables:
      - $CI_COMMIT_BRANCH == "main"
      - $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - schedules
  stage: choco-release
  dependencies:
    - release-msi
  tags:
    - windows
  script:
    - |
      $ErrorActionPreference = 'Stop'
      Set-PSDebug -Trace 1
      $msi_file_name = Resolve-Path .\dist\signed\splunk-otel-collector*.msi | Split-Path -leaf
      if ($msi_file_name -match '(\d+\.\d+\.\d+)(\.\d+)?') {
        $version = $matches[0]
      } else {
        throw "Failed to get version from $msi_file_name"
      }
      .\internal\buildscripts\packaging\choco\make.ps1 build_choco -Version $version -BuildDir .\dist\signed
    - Test-Path .\dist\signed\splunk-otel-collector.${version}.nupkg
    - |
      # Only push the choco package for stable release tags
      if ($env:CI_COMMIT_TAG -match '^v\d+\.\d+\.\d+$') {
        choco push -k $env:CHOCO_TOKEN .\dist\signed\splunk-otel-collector.${version}.nupkg
      }
  artifacts:
    paths:
      - dist/signed/*.nupkg

# release installer scripts; disabled until we get a token for pushing to S3
.release-installers:
  extends:
    - .sign-release-cache
  only:
    variables:
      - $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+.*/
  stage: release
  script:
    - python3 internal/buildscripts/packaging/release/sign_release.py --force --installers

github-release:
  extends: .go-cache
  only:
    variables:
      - $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - schedules
  stage: github-release
  dependencies:
    - compile
    - sign-exe
    - sign-osx
    - release-debs
    - release-rpms
    - release-msi
    - push-linux-image
    - build-push-windows-image
    - choco-release
  script:
    - mkdir -p dist/assets
    - cp bin/otelcol_linux_* dist/assets/
    - cp bin/translatesfx_linux_* dist/assets/
    - cp dist/signed/* dist/assets/
    - pushd dist/assets && shasum -a 256 * > checksums.txt && popd
    - release_notes="$( ./internal/buildscripts/packaging/release/gh-release-notes.sh "$CI_COMMIT_TAG" )"
    - ghr -t "$GITHUB_TOKEN" -u signalfx -r splunk-otel-collector -n "$CI_COMMIT_TAG" -b "$release_notes" --replace "$CI_COMMIT_TAG" dist/assets/
  artifacts:
    when: always
    paths:
      - dist/assets

.ansible:
  image: 'cimg/python:3.9'
  only:
    - /^ansible-v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - branches
    - schedules
  variables:
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  cache:
    key: "ansible-pip-cache"
    paths:
      - .cache/pip

ansible-build:
  extends: .ansible
  stage: build
  artifacts:
    paths:
      - dist/
  before_script:
    - pip3 install ansible==3.4.0
  script:
    - ansible-galaxy collection build ./deployments/ansible --output-path ./dist

ansible-release:
  extends: .ansible
  stage: release
  before_script:
    - pip3 install ansible==3.4.0 yq==2.12.0
  script:
    - export COLLECTION_VERSION=$(cat ./deployments/ansible/galaxy.yml | yq .version -r)
    - ansible-galaxy collection publish ./dist/signalfx-splunk_otel_collector-${COLLECTION_VERSION}.tar.gz --token=${ANSIBLE_GALAXY_TOKEN} 

puppet-release:
  image: 'docker-hub.repo.splunkdev.net/ruby:2.6-buster'
  stage: release
  only:
    variables:
      - $CI_COMMIT_TAG =~ /^puppet-v[0-9]+\.[0-9]+\.[0-9]+.*/
  except:
    - schedules
  before_script:
    - gem install bundler
    - cd deployments/puppet
    - bundle install
    - bundle exec rake module:clean
  script:
    - bundle exec rake module:push
  artifacts:
    paths:
      - deployments/puppet/pkg/*.tar.gz

cve-scan:
  extends: .go-cache
  stage: cve-scan
  retry: 2
  only:
    - main
    - schedules
  before_script:
    - apt-get update
    - apt-get install -y ca-certificates curl gnupg lsb-release
    - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
    - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
    - apt-get update
    - apt-get install -y docker-ce-cli docker-scan-plugin
  script:
    - |
      if [ -f dist/otelcol-amd64.tar ]; then
        docker load -i dist/otelcol-amd64.tar
        docker tag otelcol:amd64 otelcol:latest
      else
        make docker-otelcol
      fi
    - docker scan --accept-license --login --token ${SNYK_AUTH_TOKEN}
    - docker scan --severity high otelcol
  after_script:
    - |
      if [ "$CI_JOB_STATUS" != "success" ]; then
        curl -X POST ${SLACK_WEBHOOK_URL} -H 'Content-Type: application/json' \
          --data "{\"blocks\": [{\"type\": \"section\",\"text\": {\"type\": \"mrkdwn\",\"text\": \"*@here Gitlab Job #${CI_JOB_ID}*\"}},{\"type\": \"section\",\"text\": {\"type\": \"mrkdwn\",\"text\": \"*:ghost: Vulnerability scan failed on splunk-otel-collector*\"},\"accessory\": {\"type\": \"button\",\"text\": {\"type\": \"plain_text\",\"text\": \"More Info\",\"emoji\": true},\"style\": \"danger\",\"url\": \"${CI_JOB_URL}\",\"action_id\": \"button-action\"}}]}"
      fi