Fix for inability to read some parquet files (issue #816) #354
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test/build KFP | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "dev" | |
| - "releases/**" | |
| tags: | |
| - "*" | |
| paths: | |
| - ".make.*" | |
| - "kfp/**" | |
| - "!kfp/**.md" | |
| - "!kfp/**/test/**" | |
| - "!kfp/**/doc/**" | |
| - "data-processing-lib/**" | |
| - "!data-processing-lib/**.md" | |
| - "!data-processing-lib/**/.gitignore" | |
| - "!data-processing-lib/**/test/**" | |
| - "!data-processing-lib/**/test-data/**" | |
| - "!**.md" | |
| - "!**/doc/**" | |
| - "!**/images/**" | |
| - "!**/.gitignore" | |
| pull_request: | |
| branches: | |
| - "dev" | |
| - "releases/**" | |
| paths: | |
| - ".make.*" | |
| - "kfp/**" | |
| - "!kfp/**/test/**" | |
| - "!kfp/**.md" | |
| - "!kfp/**/doc/**" | |
| - "data-processing-lib/**" | |
| - "!data-processing-lib/**/test/**" | |
| - "!data-processing-lib/**/test-data/**" | |
| - "!data-processing-lib/**.md" | |
| - "!data-processing-lib/**/doc/**" | |
| - "!data-processing-lib/**/.gitignore" | |
| - "!**.md" | |
| - "!**/doc/**" | |
| - "!**/images/**" | |
| - "!**/.gitignore" | |
| # taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check_if_push_images: | |
| # check whether the Docker images should be pushed to the remote repository | |
| # The images are pushed if it is a merge to dev branch or a new tag is created. | |
| # The latter being part of the release process. | |
| # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| publish_images: ${{ steps.version.outputs.publish_images }} | |
| steps: | |
| - id: version | |
| run: | | |
| publish_images='false' | |
| if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; | |
| then | |
| publish_images='true' | |
| fi | |
| if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; | |
| then | |
| publish_images='true' | |
| fi | |
| echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" | |
| test-kfp-v1: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Free up space in github runner | |
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| run: | | |
| df -h | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| df -h | |
| - name: Import environment variables | |
| run: | | |
| cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV | |
| echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV | |
| echo "REPOROOT=$PWD" >> $GITHUB_ENV | |
| echo "PATH=$PATH:/tmp" >> $GITHUB_ENV | |
| - name: Test KFP libs (shared and v1) and run a workflow | |
| timeout-minutes: 120 | |
| run: | | |
| $PWD/scripts/workflow_helper.sh install-tools | |
| make -C kfp/kfp_support_lib test | |
| $PWD/scripts/workflow_helper.sh test-workflow "transforms/universal/noop" | |
| echo "Run noop completed" | |
| test-kfp-v2: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Free up space in github runner | |
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| run: | | |
| df -h | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| df -h | |
| - name: Import environment variables | |
| run: | | |
| cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV | |
| echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV | |
| echo "REPOROOT=$PWD" >> $GITHUB_ENV | |
| echo "PATH=$PATH:/tmp" >> $GITHUB_ENV | |
| echo "KFPv2=1" >> $GITHUB_ENV | |
| - name: Test KFP libs (shared and v2) and run a workflow | |
| timeout-minutes: 120 | |
| run: | | |
| $PWD/scripts/workflow_helper.sh install-tools | |
| make -C kfp/kfp_support_lib test | |
| $PWD/scripts/workflow_helper.sh test-workflow "transforms/universal/noop" | |
| echo "Run noop completed" | |
| build-kfp-components: | |
| needs: [check_if_push_images] | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| env: | |
| DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} | |
| DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Free up space in github runner | |
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| run: | | |
| df -h | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| df -h | |
| - name: Build | |
| run: | | |
| make -C kfp/kfp_ray_components DOCKER=docker image | |
| make KFPv2=1 -C kfp/kfp_ray_components DOCKER=docker image | |
| - name: Publish images | |
| if: needs.check_if_push_images.outputs.publish_images == 'true' | |
| run: make -C kfp/kfp_ray_components publish |