Fixes after testing.

Signed-off-by: Revital Sur <eres@il.ibm.com>
IBM · Jul 2, 2024 · f7de624 · f7de624
1 parent a9d7209
commit f7de624
Show file tree

Hide file tree

Showing 5 changed files with 183 additions and 84 deletions.
diff --git a/.github/workflows/test-images.yml b/.github/workflows/test-images.yml
diff --git a/.github/workflows/test-src.yml → .github/workflows/test.yml b/.github/workflows/test-src.yml → .github/workflows/test.yml
@@ -1,4 +1,4 @@
-name: Test Code
+name: Build, Test and (Optionally) Push images
 
 on:
     workflow_dispatch:
@@ -11,6 +11,20 @@ on:
             - "dev"
             - "releases/**"
 jobs:
+    infer_version:
+        runs-on: ubuntu-22.04
+        outputs:
+            publish_images: ${{ steps.version.outputs.publish_images }}
+        steps:
+            - id: version
+              run: |
+                 publish_images='false'
+                 echo ${GITHUB_REPOSITORY}
+                 if  [[ ${GITHUB_REF} == refs/heads/dev &&  ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == 'IBM/data-prep-kit' ]] ;
+                 then
+                   publish_images='true'
+                 fi
+                 echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
     test-make:
         runs-on: ubuntu-22.04
         steps:
@@ -51,18 +65,6 @@ jobs:
             - name: Test Code Transforms
               run: |
                   make -C transforms/code DOCKER=docker venv test-src
-            - name: Save images
-              # if: ${{ github.event_name != 'pull_request' }}
-              run: |
-                  make -C transforms docker-save-image
-                  make -C kfp/kfp_ray_components docker-save-image
-            - name: Upload images
-              # if: ${{ github.event_name != 'pull_request' }}
-              uses: actions/upload-artifact@v3
-              with:
-                name: images
-                path: artifacts/*
-                retention-days: 1
     test-language:
         runs-on: ubuntu-22.04
         steps:
@@ -173,36 +175,185 @@ jobs:
                   header_text "Running ${transforms[$index]} workflow test"
                   make -C ${transforms[$index]} workflow-test
                   header_text "Run ${transforms[$index]} completed"
+    test-code-images:
+        needs: [infer_version]
+        runs-on: ubuntu-22.04
+        timeout-minutes: 30
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Free up space in github runner
+              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+              run: |
+                  df -h
+                  sudo rm -rf "/usr/local/share/boost"
+                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
+                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+                  df -h
+            - name: Test Code Transform Images
+              run: |
+                  make -C data-processing-lib DOCKER=docker image 
+                  make -C transforms/code DOCKER=docker test-image
+            - name: Print space
+               # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+              run: |
+                   df -h
+                   docker images
+            - name: Save images
+              if: needs.infer_version.outputs.publish_images == 'true'
+              run: make -C transforms/code docker-save-image
+            - name: Upload images
+              if: needs.infer_version.outputs.publish_images == 'true'
+              uses: actions/upload-artifact@v3
+              with:
+                name: images-code
+                path: artifacts/*
+                retention-days: 1
+                compression-level: 0
+    test-language-images:
+        needs: [infer_version]
+        runs-on: ubuntu-22.04
+        timeout-minutes: 30
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Free up space in github runner
+              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+              run: |
+                  df -h
+                  sudo rm -rf "/usr/local/share/boost"
+                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
+                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+                  df -h
+            - name: Test language Transform Images
+              run: |
+                  make -C data-processing-lib DOCKER=docker image
+                  make -C transforms/language DOCKER=docker test-image
+            - name: Print space
+              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+              run: df -h
+            - name: Save images
+              if: needs.infer_version.outputs.publish_images == 'true'
+              run: make -C transforms/language docker-save-image
+            - name: Upload images
+              if: needs.infer_version.outputs.publish_images == 'true'
+              uses: actions/upload-artifact@v3
+              with:
+                  name: images-language
+                  path: artifacts/*
+                  compression-level: 0
+    test-universal-images:
+        needs: [infer_version]
+        runs-on: ubuntu-22.04
+        timeout-minutes: 120
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Free up space in github runner
+              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+              run: |
+                  df -h
+                  sudo rm -rf /opt/ghc
+                  sudo rm -rf "/usr/local/share/boost"
+                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
+                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+                  df -h
+            - name: Test Universal Transform Images
+              run: |
+                  make -C data-processing-lib DOCKER=docker image 
+                  make -C transforms/universal DOCKER=docker test-image
+            - name: Print space
+               # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+              run: |
+                   df -h
+                   docker images
+            - name: Save images
+              if: ${{ needs.infer_version.outputs.publish_images == 'true' }}
+              run: make -C transforms/universal docker-save-image
+            - name: Upload images
+              if: ${{ needs.infer_version.outputs.publish_images == 'true' }}
+              uses: actions/upload-artifact@v3
+              with:
+                  name: images-universal
+                  path: artifacts/*
+                  compression-level: 0
+    build-kfp-components:
+      needs: [infer_version]
+      runs-on: ubuntu-22.04
+      timeout-minutes: 30
+      steps:
+          - name: Checkout
+            uses: actions/checkout@v4
+          - name: Free up space in github runner
+            # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+            run: |
+                df -h
+                sudo rm -rf "/usr/local/share/boost"
+                sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+                sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
+                sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+                df -h
+          - name: Build 
+            run: |
+               make -C kfp/kfp_ray_components DOCKER=docker image
+          - name: Print space
+            # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+            run: df -h
+          - name: Save images
+            if: ${{ needs.infer_version.outputs.publish_images == 'true' }}
+            run: make -C kfp/kfp_ray_components docker-save-image
+          - name: Upload images
+            if: ${{ needs.infer_version.outputs.publish_images == 'true' }}
+            uses: actions/upload-artifact@v3
+            with:
+                name: images-kfp
+                path: artifacts/*
+                compression-level: 0
+    test-tool-images:
+        runs-on: ubuntu-22.04
+        timeout-minutes: 30
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Build and Test Tool images
+              run: |
+                  make -C tools/ingest2parquet DOCKER=docker test-image
     push-images:
         name: Push images with latest tag
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
+        if: needs.infer_version.outputs.publish_images == 'true'
         env:
            DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
            DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
-        needs: [test-make, test-python-lib, test-ray-lib, test-kfp-v1]
-        # if: ${{ github.event_name != 'pull_request' && github.repository == 'IBM/data-prep-kit' }}
+        needs: [infer_version, test-ray-lib, test-spark-lib, test-code, test-language, test-universal, test-tools, test-kfp-v1, test-kfp-v2, test-code-images, test-language-images, test-universal-images, build-kfp-components]
         steps:
         - uses: actions/checkout@v3
+        - name: Free up space in github runner
+            # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+          run: |
+                df -h
+                sudo rm -rf "/usr/local/share/boost"
+                sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+                sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
+                sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+                df -h
         - name: Download artifact
           uses: actions/download-artifact@v3
           with:
-            path: artifacts/*
-            name: images
-        - name: Load images
+            path: artifacts/
+            pattern: images-*
+        - name: Print artifacts dir
+          run: ls -ltr artifacts/
+        - name: Load code images
           run: |
                make -C transforms docker-load-image
                make -C kfp-kfp_ray_componenets docker-load-image
-        - id: version
-          name: Infer version
-          run: |
-            publish_images='false'
-            if  [[ ${GITHUB_REF} == refs/heads/dev ]] ;
-            then
-              publish_images='true'
-            fi
-            echo ::set-output name=publish_images::$publish_images
         - name: Publish images
-          if: ${{ steps.version.outputs.publish_images == 'true' }}
+          if: needs.infer_version.outputs.publish_images == 'true'
 #           run: make -C transforms -C kfp-kfp_ray_componenets publish
           run: echo "make -C transforms -C kfp-kfp_ray_componenets publish"
 
+
diff --git a/.make.defaults b/.make.defaults
@@ -45,7 +45,7 @@ DOCKER_REGISTRY_ENDPOINT?=$(DOCKER_HOSTNAME)/$(DOCKER_NAMESPACE)
 DOCKER_LOCAL_IMAGE=$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
 DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
 DOCKER_SPARK_BASE_IMAGE_NAME=data-prep-kit-spark-3.5.1
-DOCKER_SPARK_BASE_IMAGE=$(DOCKER_SPARK_BASE_IMAGE_NAME):$(DPK_LIB_VERSION)
+DOCKER_SPARK_BASE_IMAGE=$(DOCKER_SPARK_BASE_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
 # Deprecated in favor of DOCKER_REMOTE_IMAGE
 DOCKER_IMAGE?=$(DOCKER_REMOTE_IMAGE)
 KIND_CLUSTER_NAME=dataprep
@@ -184,6 +184,7 @@ __check_defined = \
 
 .PHONY: .defaults.docker-save-image
 .defaults.docker-save-image:
+	mkdir -p ${ARTIFACTS_DIR}/
 	docker save -o ${ARTIFACTS_DIR}/${DOCKER_IMAGE_NAME}.tar $(DOCKER_REMOTE_IMAGE)
 
 .PHONY: .defaults.docker-load-image

diff --git a/data-processing-lib/spark/Makefile b/data-processing-lib/spark/Makefile
@@ -2,7 +2,6 @@
 REPOROOT=../..
 include $(REPOROOT)/.make.defaults
 SPARK_VERSION=3.5.1
-DOCKER_IMAGE_NAME=data-prep-kit-spark-$(SPARK_VERSION)
 DOCKER_IMAGE_VERSION=$(DPK_LIB_VERSION)
 DOCKER_IMAGE_LIB_NAME=data-prep-kit-spark
 TAG := "v${DPK_LIB_VERSION}"

diff --git a/kfp/kfp_ray_components/Makefile b/kfp/kfp_ray_components/Makefile
@@ -63,7 +63,7 @@ docker-load-image:
 
 .PHONY: docker-save-image
 docker-save-image:
-	@$(MAKE) .defaults.docker-load-image
+	@$(MAKE) .defaults.docker-save-image
 
 .PHONY: publish
 publish: