tests.yml

name: Tests

on:
  push:
    paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE']
    branches:
    - 'integration'
    - 'release/version*'
  pull_request:
    paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE', '.github/workflows/microservice*.yaml']
  workflow_dispatch:
  
env:
  JAVA_VERSION: '11'
  JAVA_DISTRIBUTION: 'zulu' #This is the default on v1 of the action for 1.8
  MAVEN_OPTS: "-Djansi.force=true -Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true -XX:ThreadStackSize=1m"

jobs:
  # Runs the pom sorter and code formatter to ensure that the code
  # is formatted and poms are sorted according to project rules. If changes are found,
  # they are committed back to the branch
  check-code-formatting:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Set up JDK ${{env.JAVA_VERSION}}
      uses: actions/setup-java@v4
      with:
        distribution: ${{env.JAVA_DISTRIBUTION}}
        java-version: ${{env.JAVA_VERSION}}
        maven-version: 3.9.5
        cache: 'maven'
    - name: Extract branch name
      shell: bash
      run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
      id: extract_branch
    - name: Format code
      env:
        USER_NAME: ${{ secrets.USER_NAME }}
        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
      run: |
        mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -ntp "-Dstyle.color=always" clean formatter:format sortpom:sort impsort:sort -Dmaven.build.cache.enabled=false -Pautoformat
        git status
        git diff-index --quiet HEAD || (echo "Modified files found. Creating new commit with formatting fixes" && echo "diffs_found=true" >> "$GITHUB_ENV")
    - name: Commit Changes
      run: |
        if [ "$diffs_found" = true ]; then
          git config --global user.name "GitHub Actions"
          git config --global user.email "datawave@github.com"
          git pull origin ${{ steps.extract_branch.outputs.branch }} --rebase --autostash
          git checkout -b ${{ steps.extract_branch.outputs.branch }}
          git commit -am "GitHub Actions: Fix Formatting"
          git push origin ${{ steps.extract_branch.outputs.branch }}
        else
          echo "Nothing to do"
        fi
        
  # Build the code and run the unit/integration tests.
  build-and-test:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Set up JDK ${{env.JAVA_VERSION}}
      uses: actions/setup-java@v4
      with:
        distribution: ${{env.JAVA_DISTRIBUTION}}
        java-version: ${{env.JAVA_VERSION}}
        maven-version: 3.9.5
        cache: 'maven'
    - name: Build and Run Unit Tests
      env:
        USER_NAME: ${{ secrets.USER_NAME }}
        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
      run: |
        RUN_TESTS="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -ntp "-Dstyle.color=always" -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -Ddeploy -Ddist -T1C clean verify"
        $RUN_TESTS \
          || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \
          || { echo "***** TESTS FAILED. Attempting final retry."; $RUN_TESTS; }

  # This step can be uncommented to support feature development.  When developing a 
  # feature which spans multiple repos, you can update the poms to use the latest 
  # SNAPSHOT across the board, and run this action as a sanity check to ensure 
  # everything is working in unison.

  # # Build the code and run the microservice unit/integration tests.
  # build-and-test-microservices:
  #   runs-on: ubuntu-latest
  #   steps:
  #     - name: Checkout Code
  #       uses: actions/checkout@v4
  #       with:
  #         submodules: 'recursive'
  #     - name: Set up JDK ${{env.JAVA_VERSION}}
  #       uses: actions/setup-java@v4
  #       with:
  #         distribution: ${{env.JAVA_DISTRIBUTION}}
  #         java-version: ${{env.JAVA_VERSION}}
  #     - uses: actions/cache@v4
  #       with:
  #         path: ~/.m2/repository
  #         key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }}
  #         restore-keys: |
  #           ${{ runner.os }}-maven-build-
  #           ${{ runner.os }}-maven-format-
  #           ${{ runner.os }}-maven-
  #     - name: Build Project
  #       env:
  #         USER_NAME: ${{ secrets.USER_NAME }}
  #         ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
  #       run: |
  #         BUILD="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -DskipServices -Ddeploy -Ddist -T1C -pl "-:config-service" clean install -DskipTests"
  #         $BUILD
  #     - name: Run Microservice Unit Tests
  #       env:
  #         USER_NAME: ${{ secrets.USER_NAME }}
  #         ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
  #       run: |
  #         RUN_TESTS="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Dmaven.build.cache.enabled=false verify"
  #         cd microservices
  #         $RUN_TESTS \
  #           || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \
  #           || { echo "***** TESTS FAILED. Attempting final retry."; $RUN_TESTS; }

  quickstart-build-and-test:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Set up JDK ${{env.JAVA_VERSION}}
      uses: actions/setup-java@v4
      with:
        distribution: ${{env.JAVA_DISTRIBUTION}}
        java-version: ${{env.JAVA_VERSION}}
        maven-version: 3.9.5
        cache: 'maven'
    # Allow us to use the "--squash" option below
    - name: Turn on Docker experimental features and move Docker data root
      run: |
        if [[ -f /etc/docker/daemon.json ]]; then
          sudo sed -ri 's|\s*}\s*$|, "experimental": true, "data-root": "/mnt/docker" }|' /etc/docker/daemon.json
        else
          echo $'{\n    "experimental": true, "data-root": "/mnt/docker"\n}' | sudo tee /etc/docker/daemon.json
        fi
        sudo systemctl restart docker
        echo Docker Experimental Features: $(docker version -f '{{.Server.Experimental}}')
    # Builds the quickstart docker image and run the query tests
    - name: Quickstart Query Tests
      env:
        DW_DATAWAVE_BUILD_COMMAND: "mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -ntp -Dstyle.color=always -Dmaven.build.cache.enabled=false -Pdev -Ddeploy -Dtar -DskipTests clean package"
        DOCKER_BUILD_OPTS: "--squash --force-rm"
        USER_NAME: ${{ secrets.USER_NAME }}
        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
      run: |
        TAG=$(mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -q -N -Dmaven.build.cache.enabled=false -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec)
        contrib/datawave-quickstart/docker/docker-build.sh ${TAG} --docker-opts "${DOCKER_BUILD_OPTS}"

  compose-build-and-test-latest-snapshots:
    runs-on: ubuntu-latest
    steps:
      - name: Free up some space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          sudo rm -rf $AGENT_TOOLSDIRECTORY
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          submodules: 'recursive'
      - name: Set up JDK ${{env.JAVA_VERSION}}
        uses: actions/setup-java@v4
        with:
          distribution: ${{env.JAVA_DISTRIBUTION}}
          java-version: ${{env.JAVA_VERSION}}
          maven-version: 3.9.5
          cache: 'maven'
      # Builds the quickstart and microservice docker images and runs a query test
      - name: Docker Compose Query Tests
        env:
          USER_NAME: ${{ secrets.USER_NAME }}
          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
        run: |
          # update datawave dependencies to use the latest snapshots
          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e versions:update-properties versions:update-parent -DallowSnapshots=true -Dincludes=gov.nsa.*
          
          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean install
          
          # free up some space so that we don't run out
          docker system prune -f
          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean
          
          cd docker
          ./bootstrap.sh
          
          attempt=0
          max_attempts=20
          while [ $attempt -lt $max_attempts ]; do
            attempt=$((attempt+1))
          
            echo "Starting docker compose (Attempt ${attempt}/${max_attempts})"
            nohup docker compose up -d --no-recreate < /dev/null > compose.out 2>&1 &
            sleep 60s
            cat compose.out
          
            # check to see if the query service is running
            QUERY="$(docker compose ps --status running --services | grep query || true)"
          
            if [ "$QUERY" == "query" ] ; then
              echo "Docker compose started successfully"
              break
            elif [ $attempt -eq $max_attempts ] ; then
              echo "Failed to start docker compose"
              exit 1
            fi
          done
          
          cd scripts
          ./testAll.sh

      - name: Dump Logs
        if: failure()
        run: |
          cd docker
          docker compose logs

  compose-build-and-test:
    runs-on: ubuntu-latest
    steps:
      - name: Free up some space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          sudo rm -rf $AGENT_TOOLSDIRECTORY
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          submodules: 'recursive'
      - name: Set up JDK ${{env.JAVA_VERSION}}
        uses: actions/setup-java@v4
        with:
          distribution: ${{env.JAVA_DISTRIBUTION}}
          java-version: ${{env.JAVA_VERSION}}
      - uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-build-
            ${{ runner.os }}-maven-format-
            ${{ runner.os }}-maven-
      # Builds the quickstart and microservice docker images and runs a query test
      - name: Docker Compose Query Tests
        env:
          USER_NAME: ${{ secrets.USER_NAME }}
          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
        run: |
          # set some bogus URLs to trigger dependency download via maven
          DIST_URLS="-Durl.zookeeper=https://bogus.apache.org/zookeeper/zookeeper-3.7.2/apache-zookeeper-3.7.2-bin.tar.gz.tar.gz \
                     -Durl.accumulo=https://bogus.apache.org/accumulo/2.1.3/accumulo-2.1.3-bin.tar.gz \
                     -Durl.wildfly=https://bogus.jboss.org/wildfly/17.0.1.Final/wildfly-17.0.1.Final.tar.gz \
                     -Durl.hadoop=https://bogus.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz \
                     -Durl.maven=https://bogus.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz"
          
          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean install
          # free up some space so that we don't run out
          docker system prune -f
          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean
          cd docker
          ./bootstrap.sh
          attempt=0
          max_attempts=20
          while [ $attempt -lt $max_attempts ]; do
            attempt=$((attempt+1))
            echo "Starting docker compose (Attempt ${attempt}/${max_attempts})"
            nohup docker compose up -d --no-recreate < /dev/null > compose.out 2>&1 &
            sleep 60s
            cat compose.out
            # check to see if the query service is running
            QUERY="$(docker compose ps --status running --services | grep query || true)"
            if [ "$QUERY" == "query" ] ; then
              echo "Docker compose started successfully"
              break
            elif [ $attempt -eq $max_attempts ] ; then
              echo "Failed to start docker compose"
              exit 1
            fi
          done
          cd scripts
          ./testAll.sh
      - name: Dump Logs
        if: failure()
        run: |
          cd docker
          docker compose logs

    # Here's an example of how you'd deploy the image to the github package registry.
    # We don't want to do this by default since packages on github cannot be deleted
    # or overwritten. So this could only be done for tags, however it seems the quickstart
    # image may also exceed the maximum size allowed by github.
    # - name: Deploy Quickstart Image
    #   env:
    #     IMAGE_REGISTRY: "docker.pkg.github.com"
    #     IMAGE_USERNAME: "brianloss"
    #     IMAGE_NAME: "datawave/quickstart"
    #     USER_NAME: ${{ secrets.USER_NAME }}
    #     ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
    #   run: |
    #     # Set up env vars
    #     TAG=$(mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -q -N -Dmaven.build.cache.enabled=false -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec)
    #     REMOTE_IMAGE_NAME="${IMAGE_REGISTRY}/${IMAGE_USERNAME}/${IMAGE_NAME}"
    #     # Log in to the package registry
    #     echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com --username ${GITHUB_ACTOR} --password-stdin
    #     # Tag and push the image
    #     docker tag ${IMAGE_NAME}:${TAG} ${REMOTE_IMAGE_NAME}:${TAG}
    #     docker images
    #     docker push ${REMOTE_IMAGE_NAME}:${TAG}