azure-pipelines.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

trigger:
  branches:
    include:
      - '*'  # must quote since "*" is a YAML reserved character; we want a string

pool:
  vmImage: 'ubuntu-16.04'

variables:
  MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
  MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
  SPARK_VERSION: '2.4.4'
  HADOOP_VERSION: '2.7'
  SPARK_HOME: $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)

stages:
  - stage: test
    displayName: "Run tests"
    jobs:
      - job: unit_tests_spark_client
        steps:
          - task: Maven@3
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'clean install'
              options: -DskipTests
              publishJUnitResults: false
              javaHomeOption: 'JDKVersion'
              jdkVersionOption: '1.8'
              jdkArchitectureOption: 'x64'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
          - task: Cache@2
            inputs:
              key: 'maven | "$(Agent.OS)" | **/pom.xml'
              restoreKeys: |
                maven | "$(Agent.OS)"
                maven
              path: $(MAVEN_CACHE_FOLDER)
            displayName: Cache Maven local repo
          - task: Maven@3
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Punit-tests -pl hudi-client/hudi-spark-client
              publishJUnitResults: false
              testResultsFiles: '**/surefire-reports/TEST-*.xml'
              testRunTitle: 'unit tests spark client'
              javaHomeOption: 'JDKVersion'
              jdkVersionOption: '1.8'
              jdkArchitectureOption: 'x64'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
      - job: unit_tests_utilities
        steps:
          - task: Maven@3
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'clean install'
              options: -DskipTests
              publishJUnitResults: false
              javaHomeOption: 'JDKVersion'
              jdkVersionOption: '1.8'
              jdkArchitectureOption: 'x64'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
          - task: Cache@2
            inputs:
              key: 'maven | "$(Agent.OS)" | **/pom.xml'
              restoreKeys: |
                maven | "$(Agent.OS)"
                maven
              path: $(MAVEN_CACHE_FOLDER)
            displayName: Cache Maven local repo
          - task: Maven@3
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Punit-tests -pl hudi-utilities
              publishJUnitResults: false
              testResultsFiles: '**/surefire-reports/TEST-*.xml'
              testRunTitle: 'unit tests utilities'
              javaHomeOption: 'JDKVersion'
              jdkVersionOption: '1.8'
              jdkArchitectureOption: 'x64'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
      - job: unit_tests_other_modules
        steps:
          - task: Maven@3
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'clean install'
              options: -DskipTests
              publishJUnitResults: false
              javaHomeOption: 'JDKVersion'
              jdkVersionOption: '1.8'
              jdkArchitectureOption: 'x64'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
          - task: Cache@2
            inputs:
              key: 'maven | "$(Agent.OS)" | **/pom.xml'
              restoreKeys: |
                maven | "$(Agent.OS)"
                maven
              path: $(MAVEN_CACHE_FOLDER)
            displayName: Cache Maven local repo
          - task: Maven@3
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Punit-tests -pl !hudi-utilities,!hudi-client/hudi-spark-client
              publishJUnitResults: false
              testResultsFiles: '**/surefire-reports/TEST-*.xml'
              testRunTitle: 'unit tests other modules'
              javaHomeOption: 'JDKVersion'
              jdkVersionOption: '1.8'
              jdkArchitectureOption: 'x64'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
      - job: functional_tests
        steps:
          - task: Maven@3
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Pfunctional-tests
              publishJUnitResults: false
              testResultsFiles: '**/surefire-reports/TEST-*.xml'
              testRunTitle: 'functional tests'
              javaHomeOption: 'JDKVersion'
              jdkVersionOption: '1.8'
              jdkArchitectureOption: 'x64'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
      - job: integration_tests
        steps:
          - script: |
              echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)'
              wget https://archive.apache.org/dist/spark/spark-$(SPARK_VERSION)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -O $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz
              tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz -C $(Pipeline.Workspace)/
              mkdir /tmp/spark-events/
              mvn -Pintegration-tests verify