Skip to content

Restructure Html2Parquet with its own dpk_ namespace #294

Restructure Html2Parquet with its own dpk_ namespace

Restructure Html2Parquet with its own dpk_ namespace #294

Workflow file for this run

name: Test/build KFP
on:
workflow_dispatch:
push:
branches:
- "dev"
- "releases/**"
tags:
- "*"
paths:
- ".make.*"
- "kfp/**"
- "!kfp/**.md"
- "!kfp/**/test/**"
- "!kfp/**/doc/**"
- "data-processing-lib/**"
- "!data-processing-lib/**.md"
- "!data-processing-lib/**/.gitignore"
- "!data-processing-lib/**/test/**"
- "!data-processing-lib/**/test-data/**"
- "!**.md"
- "!**/doc/**"
- "!**/images/**"
- "!**/.gitignore"
pull_request:
branches:
- "dev"
- "releases/**"
paths:
- ".make.*"
- "kfp/**"
- "!kfp/**/test/**"
- "!kfp/**.md"
- "!kfp/**/doc/**"
- "data-processing-lib/**"
- "!data-processing-lib/**/test/**"
- "!data-processing-lib/**/test-data/**"
- "!data-processing-lib/**.md"
- "!data-processing-lib/**/doc/**"
- "!data-processing-lib/**/.gitignore"
- "!**.md"
- "!**/doc/**"
- "!**/images/**"
- "!**/.gitignore"
# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
check_if_push_images:
# check whether the Docker images should be pushed to the remote repository
# The images are pushed if it is a merge to dev branch or a new tag is created.
# The latter being part of the release process.
# The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file.
runs-on: ubuntu-22.04
outputs:
publish_images: ${{ steps.version.outputs.publish_images }}
steps:
- id: version
run: |
publish_images='false'
if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ;
then
publish_images='true'
fi
if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ;
then
publish_images='true'
fi
echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
test-kfp-v1:
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Free up space in github runner
# Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
run: |
df -h
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Import environment variables
run: |
cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
echo "REPOROOT=$PWD" >> $GITHUB_ENV
echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
- name: Test KFP libs (shared and v1) and run a workflow
timeout-minutes: 120
run: |
$PWD/scripts/workflow_helper.sh install-tools
make -C kfp/kfp_support_lib test
$PWD/scripts/workflow_helper.sh test-workflow "transforms/universal/noop"
echo "Run noop completed"
test-kfp-v2:
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Free up space in github runner
# Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
run: |
df -h
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Import environment variables
run: |
cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
echo "REPOROOT=$PWD" >> $GITHUB_ENV
echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
echo "KFPv2=1" >> $GITHUB_ENV
- name: Test KFP libs (shared and v2) and run a workflow
timeout-minutes: 120
run: |
$PWD/scripts/workflow_helper.sh install-tools
make -C kfp/kfp_support_lib test
$PWD/scripts/workflow_helper.sh test-workflow "transforms/universal/noop"
echo "Run noop completed"
build-kfp-components:
needs: [check_if_push_images]
runs-on: ubuntu-22.04
timeout-minutes: 30
env:
DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Free up space in github runner
# Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
run: |
df -h
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Build
run: |
make -C kfp/kfp_ray_components DOCKER=docker image
make KFPv2=1 -C kfp/kfp_ray_components DOCKER=docker image
- name: Publish images
if: needs.check_if_push_images.outputs.publish_images == 'true'
run: make -C kfp/kfp_ray_components publish