test.yml

name: Test CI

on:
    workflow_dispatch:
    push:
        branches:
            - "dev"
    pull_request:
        branches:
            - "dev"
jobs:
    test-python-lib:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Test data-processing-lib/python
              run: |
                  make -C data-processing-lib/python DOCKER=docker venv test
    test-ray-lib:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Test data-processing-lib/ray
              run: |
                  make -C data-processing-lib/ray DOCKER=docker venv test
    test-spark-lib:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Test data-processing-lib/spark
              run: |
                  make -C data-processing-lib/spark DOCKER=docker venv test
    test-code:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Test Code Transforms
              run: |
                  make -C transforms/code DOCKER=docker venv test-src
    test-universal:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Test Universal Transforms
              run: |
                  make -C transforms/universal DOCKER=docker venv test-src
    test-tools:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Test tools
              run: |
                  make -C tools DOCKER=docker venv test
    test-kfp-lib:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Test KFP v1 lib
              run: |
                  source kind/requirements.env
                  export PATH=$PATH:/tmp/
                  curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
                  curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
                  chmod 700 /tmp/get_helm.sh
                  HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
                  chmod 777 /tmp/helm
                  chmod 777 /tmp/kind
                  curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
                  chmod 777 /tmp/kubectl
                  curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
                  chmod +x /tmp/mc
                  export DEPLOY_KUBEFLOW=0
                  make -C kind setup 
                  make -C kfp/kfp_support_lib build test
    test-kfp-workflow-run:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Free up space in github runner
              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
              run: |
                  df -h
                  sudo rm -rf "/usr/local/share/boost"
                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                  df -h
            - name: Test KFP v1 worflow run
              timeout-minutes: 120
              run: |
                  source kind/requirements.env
                  export PATH=$PATH:/tmp/
                  curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
                  curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
                  chmod 700 /tmp/get_helm.sh
                  HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
                  chmod 777 /tmp/helm
                  chmod 777 /tmp/kind
                  curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
                  chmod 777 /tmp/kubectl
                  curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
                  chmod +x /tmp/mc
                  export DEPLOY_KUBEFLOW=1
                  make -C kind setup
                  make -C kfp/kfp_support_lib test
                  make -C transforms/universal/noop/ workflow-build
                  make -C transforms/universal/noop workflow-test