Skip to content
CI

Use --exclude-tasks=check-cluster-health for OSB #3210

Sign in to view logs

Use --exclude-tasks=check-cluster-health for OSB

Use --exclude-tasks=check-cluster-health for OSB #3210

Workflow file for this run

.github/workflows/CI.yml at 7b1d01c
name: CI
on:
push:
pull_request:
env:
python-version: '3.11'
java-version: '11'
gradle-version: '8.0.2'
node-version: '18.x'
jobs:
workflow-info:
runs-on: ubuntu-latest
steps:
- name: Create JSON Artifact
run: |
echo '{' > workflow-info.json
echo ' "pr_number": "${{ github.event.pull_request.number }}",' >> workflow-info.json
echo ' "commit": "${{ github.event.pull_request.head.sha || github.event.after || github.sha }}",' >> workflow-info.json
echo ' "branch": "${{ github.ref }}",' >> workflow-info.json
echo ' "commit_parent": "${{ github.event.pull_request.base.sha || github.event.before || github.base.sha }}",' >> workflow-info.json
echo ' "build_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",' >> workflow-info.json
echo ' "build": "${{ github.run_id }}"' >> workflow-info.json
echo '}' >> workflow-info.json
echo "workflow-info.json created"
- name: Upload JSON Artifact
uses: actions/upload-artifact@v4
with:
name: workflow-info
path: workflow-info.json
style-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
java-version: ${{ env.java-version }}
distribution: "corretto"
- uses: gradle/actions/setup-gradle@v4
with:
gradle-version: ${{ env.gradle-version }}
- name: Run Gradle Build
run: ./gradlew spotlessCheck
env:
OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
python-lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ env.python-version }}
- name: Install dependencies
run: |
python3 -m pip install --upgrade pip
python3 -m pip install flake8
- name: Analysing the code with flake8
run: |
flake8 $(git ls-files '*.py')
console-python-tests:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
py-project:
- lib/console_link
- cluster_tools
- console_api
env:
WORKING_DIR: ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/${{ matrix.py-project }}
defaults:
run:
working-directory: ${{ env.WORKING_DIR }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ env.python-version }}
- run: |
python3 -m pip install --upgrade pipenv
pipenv install --deploy --dev
pipenv run test
pipenv run coverage xml
- name: Get Sanitized Name
env:
PY_PROJECT: ${{ matrix.py-project }}
run: echo "SANITIZED_PY_PROJECT=${PY_PROJECT/\//-}" >> $GITHUB_ENV
- name: Upload Coverage Reports
uses: actions/upload-artifact@v4
with:
if-no-files-found: error
name: coverage-reports-python-tests-${{ env.SANITIZED_PY_PROJECT }}
path: ${{ env.WORKING_DIR }}/coverage.xml
gradle-tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
java-version: ${{ env.java-version }}
distribution: 'corretto'
- uses: gradle/actions/setup-gradle@v4
with:
gradle-version: ${{ env.gradle-version }}
gradle-home-cache-cleanup: true
- name: Run Gradle Build
run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace
env:
OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
- name: Run Tests with Coverage
run: ./gradlew mergeJacocoReports -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace
env:
OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
- name: Detect Memory Dumps
if: failure()
run: |
if find . -type f -name "*.hprof" | grep -q '.'; then
echo "::group::Memory Dumps Detected"
echo "::warning::Memory dumps were found and uploaded as artifacts. Review these files to diagnose OOM issues."
echo "To download and inspect these files, navigate to 'Actions' -> 'Artifacts'."
echo "::endgroup::"
fi
- name: Upload memory dump
if: failure()
uses: actions/upload-artifact@v4
with:
if-no-files-found: ignore
name: memory-dumps
path: ./**/*.hprof
- uses: actions/upload-artifact@v4
if: always()
with:
name: test-reports-gradle-tests
path: |
**/build/reports/tests/
**/reports/jacoco/mergedReport/
- name: Upload Coverage Reports
uses: actions/upload-artifact@v4
with:
if-no-files-found: error
name: coverage-reports-gradle-tests
path: ./**/jacocoMergedReport.xml
python-e2e-tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
java-version: ${{ env.java-version }}
distribution: 'corretto'
- uses: gradle/actions/setup-gradle@v4
with:
gradle-version: ${{ env.gradle-version }}
- uses: actions/setup-python@v5
with:
python-version: ${{ env.python-version }}
- name: Generate Cache Key from Dockerfiles
id: generate_cache_key
run: |
files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \))
file_contents=$(cat $files)
key=$(echo "${file_contents}" | sha1sum | awk '{print $1}')
echo "key=${key}" >> "$GITHUB_OUTPUT"
- name: Cache Docker Images
uses: ScribeMD/docker-cache@0.5.0
with:
key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }}
read-only: true
- name: Start Docker Solution
run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace
env:
OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
- name: Run E2E test script
working-directory: TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test
run: |
docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s
- name: Collect Docker, OpenSearch Benchmark, and Shared Logs
if: always()
run: |
mkdir -p logs/docker logs/opensearch_benchmark_logs logs/shared_logs_output
for container in $(docker ps -aq); do
container_name=$(docker inspect --format '{{.Name}}' $container | sed 's/\///')
docker logs $container > logs/docker/${container_name}_logs.txt 2>&1
done
docker cp $(docker ps --filter "name=migration-console" -q):/root/.benchmark/logs logs/opensearch_benchmark_logs
docker cp $(docker ps --filter "name=migration-console" -q):/shared-logs-output logs/shared_logs_output
- name: Upload Logs
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-test-logs
path: |
logs/docker
logs/opensearch_benchmark_logs
logs/shared_logs_output
- name: Clean up migrations docker images before caching
run: |
docker stop $(docker ps -q) && docker system prune --volumes -f
docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^migrations/' | xargs -I {} docker image rm {}
node-tests:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
npm-project:
- ./deployment/cdk/opensearch-service-migration
- ./deployment/migration-assistant-solution
defaults:
run:
working-directory: ${{ matrix.npm-project }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: ${{ env.node-version }}
- name: Install NPM dependencies
run: npm ci
- name: Run CDK Jest Tests (using mocked images)
run: npm run test
link-checker:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: lychee Link Checker
id: lychee
uses: lycheeverse/lychee-action@v1
with:
args: --verbose --accept=200,403,429 "**/*.html" "**/*.md" "**/*.txt" "**/*.json"
--offline
--exclude "file:///github/workspace/*"
--exclude "http://localhost*"
--exclude "https://localhost*"
--exclude "http://capture-proxy*"
--exclude "https://capture-proxy*"
--exclude-mail
fail: true
env:
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
all-ci-checks-pass:
needs:
- console-python-tests
- gradle-tests
- link-checker
- node-tests
- python-e2e-tests
- python-lint
- style-check
if: always()
runs-on: ubuntu-latest
steps:
- if: ${{ contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'skipped') }}
run: |
echo "One or more job cancelled, failed, or skipped" && exit 1
- run: |
echo '## :heavy_check_mark: All continous integration checks pass' >> $GITHUB_STEP_SUMMARY