CI.yml

name: CI

on:
  push:
  pull_request:

env:
  python-version: '3.11'
  java-version: '11'
  gradle-version: '8.0.2'
  node-version: '18.x'

jobs:
  workflow-info:
    runs-on: ubuntu-latest
    steps:
      - name: Create JSON Artifact
        run: |
          echo '{' > workflow-info.json
          echo '  "pr_number": "${{ github.event.pull_request.number }}",' >> workflow-info.json
          echo '  "commit": "${{ github.event.pull_request.head.sha || github.event.after || github.sha }}",' >> workflow-info.json
          echo '  "branch": "${{ github.ref }}",' >> workflow-info.json
          echo '  "commit_parent": "${{ github.event.pull_request.base.sha || github.event.before || github.base.sha }}",' >> workflow-info.json
          echo '  "build_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",' >> workflow-info.json
          echo '  "build": "${{ github.run_id }}"' >> workflow-info.json
          echo '}' >> workflow-info.json
          echo "workflow-info.json created"
      - name: Upload JSON Artifact
        uses: actions/upload-artifact@v4
        with:
          name: workflow-info
          path: workflow-info.json


  style-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-java@v4
        with:
          java-version: ${{ env.java-version }}
          distribution: "corretto"
      - uses: gradle/actions/setup-gradle@v4
        with:
          gradle-version: ${{ env.gradle-version }}
      - name: Run Gradle Build
        run: ./gradlew spotlessCheck
        env:
          OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''

  python-lint:
    runs-on: ubuntu-latest
    steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ env.python-version }}
     - name: Install dependencies
       run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install flake8
     - name: Analysing the code with flake8
       run: |
        flake8 $(git ls-files '*.py')

  console-python-tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        py-project:
          - lib/console_link
          - cluster_tools
          - console_api
    env:
      WORKING_DIR: ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/${{ matrix.py-project }}
    defaults:
      run:
        working-directory: ${{ env.WORKING_DIR }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.python-version }}
      - run: |
          python3 -m pip install --upgrade pipenv
          pipenv install --deploy --dev
          pipenv run test
          pipenv run coverage xml
      - name: Get Sanitized Name
        env:
          PY_PROJECT: ${{ matrix.py-project }}
        run: echo "SANITIZED_PY_PROJECT=${PY_PROJECT/\//-}" >> $GITHUB_ENV
      - name: Upload Coverage Reports
        uses: actions/upload-artifact@v4
        with:
          if-no-files-found: error
          name: coverage-reports-python-tests-${{ env.SANITIZED_PY_PROJECT }}
          path: ${{ env.WORKING_DIR }}/coverage.xml


  gradle-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-java@v4
        with:
          java-version: ${{ env.java-version }}
          distribution: 'corretto'
      - uses: gradle/actions/setup-gradle@v4
        with:
          gradle-version: ${{ env.gradle-version }}
          gradle-home-cache-cleanup: true
      - name: Run Gradle Build
        run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace
        env:
          OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''

      - name: Run Tests with Coverage
        run: ./gradlew mergeJacocoReports -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace
        env:
          OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''

      - name: Detect Memory Dumps
        if: failure()
        run: |
          if find . -type f -name "*.hprof" | grep -q '.'; then
            echo "::group::Memory Dumps Detected"
            echo "::warning::Memory dumps were found and uploaded as artifacts. Review these files to diagnose OOM issues."
            echo "To download and inspect these files, navigate to 'Actions' -> 'Artifacts'."
            echo "::endgroup::"
          fi

      - name: Upload memory dump
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          if-no-files-found: ignore
          name: memory-dumps
          path: ./**/*.hprof

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-reports-gradle-tests
          path: |
            **/build/reports/tests/
            **/reports/jacoco/mergedReport/
      - name: Upload Coverage Reports
        uses: actions/upload-artifact@v4
        with:
          if-no-files-found: error
          name: coverage-reports-gradle-tests
          path: ./**/jacocoMergedReport.xml


  python-e2e-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-java@v4
        with:
          java-version: ${{ env.java-version }}
          distribution: 'corretto'
      - uses: gradle/actions/setup-gradle@v4
        with:
          gradle-version: ${{ env.gradle-version }}
      - uses: actions/setup-python@v5
        with:
         python-version: ${{ env.python-version }}
      - name: Generate Cache Key from Dockerfiles
        id: generate_cache_key
        run: |
          files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \))
          file_contents=$(cat $files)
          key=$(echo "${file_contents}" | sha1sum | awk '{print $1}')
          echo "key=${key}" >> "$GITHUB_OUTPUT"
      - name: Cache Docker Images
        uses: ScribeMD/docker-cache@0.5.0
        with:
          key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }}
          read-only: true
      - name: Start Docker Solution
        run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace
        env:
          OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
      - name: Run E2E test script
        working-directory: TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test
        run: |
          docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s
      - name: Collect Docker, OpenSearch Benchmark, and Shared Logs
        if: always()
        run: |
          mkdir -p logs/docker logs/opensearch_benchmark_logs logs/shared_logs_output
          for container in $(docker ps -aq); do
            container_name=$(docker inspect --format '{{.Name}}' $container | sed 's/\///')
            docker logs $container > logs/docker/${container_name}_logs.txt 2>&1
          done
          docker cp $(docker ps --filter "name=migration-console" -q):/root/.benchmark/logs logs/opensearch_benchmark_logs
          docker cp $(docker ps --filter "name=migration-console" -q):/shared-logs-output logs/shared_logs_output
      - name: Upload Logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-test-logs
          path: |
            logs/docker
            logs/opensearch_benchmark_logs
            logs/shared_logs_output
      - name: Clean up migrations docker images before caching
        run: |
          docker stop $(docker ps -q) && docker system prune --volumes -f
          docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^migrations/' | xargs -I {} docker image rm {}

  node-tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        npm-project:
          - ./deployment/cdk/opensearch-service-migration
          - ./deployment/migration-assistant-solution
    defaults:
      run:
        working-directory: ${{ matrix.npm-project }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.node-version }}
      - name: Install NPM dependencies
        run: npm ci
      - name: Run CDK Jest Tests (using mocked images)
        run: npm run test

  link-checker:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: lychee Link Checker
        id: lychee
        uses: lycheeverse/lychee-action@v1
        with:
          args: --verbose --accept=200,403,429  "**/*.html" "**/*.md" "**/*.txt" "**/*.json"
            --offline
            --exclude "file:///github/workspace/*"
            --exclude "http://localhost*"
            --exclude "https://localhost*"
            --exclude "http://capture-proxy*"
            --exclude "https://capture-proxy*"
            --exclude-mail
          fail: true
        env:
          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}


  all-ci-checks-pass:
    needs:
      - console-python-tests
      - gradle-tests
      - link-checker
      - node-tests
      - python-e2e-tests
      - python-lint
      - style-check
    if: always()
    runs-on: ubuntu-latest
    steps:
      - if: ${{ contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'skipped') }}
        run: |
          echo "One or more job cancelled, failed, or skipped" && exit 1
      - run: |
          echo '## :heavy_check_mark: All continous integration checks pass' >> $GITHUB_STEP_SUMMARY