Resolving conflicts

.github/workflows/build-images.yml

@@ -0,0 +1,36 @@
+
+name: Build Transform Images
+
+on: 
+  workflow_dispatch:
+  push:
+    branches:
+      - "dev"
+  pull_request:
+    branches:
+      - "dev"
+jobs:
+  build-code:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Code Transforms
+      run: |
+         make -C transforms/code DOCKER=docker image test-image 
+  build-universal:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Universal Transforms
+      run: |
+         make -C transforms/universal DOCKER=docker image test-image
+  build-tools:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Universal Transforms
+      run: |
+         make -C tools/ingest2parquet DOCKER=docker image test-image

.github/workflows/build-library.yml

@@ -0,0 +1,24 @@
+
+name: Build Library
+
+on: 
+  workflow_dispatch:
+  push:
+    branches:
+      - "dev"
+  pull_request:
+    branches:
+      - "dev"
+jobs:
+  build-lib:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python:
+          - "3.11"
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build data-processing-lib
+      run: |
+         make -C data-processing-lib DOCKER=docker venv build

.github/workflows/deploy-docs.yml

@@ -0,0 +1,21 @@
+name: deploy docs
+
+on:
+    workflow_dispatch:
+    push:
+        branches:
+            - dev
+jobs:
+    deploy:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v2
+            - uses: actions/setup-python@v2
+              with:
+                  python-version: 3.x
+            - run: pip install mkdocs-material mkdocstrings[python] mkdocs-badges mkdocs-same-dir
+            - run: |
+                  # remove badges 
+                  cat README.md |sed  '/img\.shields\.io/d' > README_.md
+                  mv README_.md README.md  
+                  mkdocs gh-deploy --force

.github/workflows/deploy-library.yml

@@ -0,0 +1,59 @@
+name: Build, Test, and Upload PyPI package
+
+on:
+    release:
+        types:
+            - published
+    workflow_dispatch:
+
+permissions:
+    contents: read
+    # see https://docs.pypi.org/trusted-publishers/
+    id-token: write
+
+jobs:
+    build-package:
+        name: Build and check packages
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout 
+              uses: actions/checkout@v4
+              with:
+                  # for setuptools-scm
+                  fetch-depth: 0
+            - name: Build Package for pypi 
+              run: |
+                make -C data-processing-lib build
+    publish-test-pypi:
+        name: Publish packages to test.pypi.org
+        # disabled 
+        if: false
+        runs-on: ubuntu-latest
+        needs: build-package
+
+        steps:
+            - name: Fetch build artifacts
+              uses: actions/download-artifact@v4
+              with:
+                  name: Packages
+                  path: dist
+
+            - name: Upload to Test PyPI
+              uses: pypa/gh-action-pypi-publish@release/v1
+              with:
+                  repository-url: https://test.pypi.org/legacy/
+
+    publish-pypi:
+        name: Publish release to pypi.org
+        runs-on: ubuntu-latest
+        needs: build-package
+        # disabled as of now
+        if: false
+        steps:
+            - name: Fetch build artifacts
+              uses: actions/download-artifact@v4
+              with:
+                  name: Packages
+                  path: dist
+            - name: Upload to PyPI
+              uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/deploy-transforms.yml

@@ -0,0 +1,32 @@
+name: Build, Test, and upload Transform images to quay.io
+
+on:
+    release:
+        types:
+            - published
+    workflow_dispatch:
+
+jobs:
+    build-images:
+        name: Build and check images
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout 
+              uses: actions/checkout@v4
+              with:
+                  # for setuptools-scm
+                  fetch-depth: 0
+            - name: Build Package for pypi 
+              run: |
+                make -C transforms image test-image
+    publish-images:
+        name: Publish packages to quay.io
+        # disabled 
+        if: false
+        runs-on: ubuntu-latest
+        needs: build-images
+
+        steps:
+            - name: Push images to quay.io registry
+              run: |
+                make -C transforms publish

.github/workflows/test.yml

@@ -0,0 +1,73 @@
+name: Test CI
+
+on:
+    workflow_dispatch:
+    push:
+        branches:
+            - "dev"
+    pull_request:
+        branches:
+            - "dev"
+jobs:
+    test-lib:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Test data-processing-lib
+              run: |
+                  make -C data-processing-lib DOCKER=docker venv test
+    test-code:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Test Code Transforms
+              run: |
+                  make -C transforms/code DOCKER=docker venv test-src
+    test-universal:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Test Universal Transforms
+              run: |
+                  make -C transforms/universal DOCKER=docker venv test-src
+    test-tools:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Test tools
+              run: |
+                  make -C tools DOCKER=docker venv test
+    test-kfp-lib:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Test KFP lib
+              run: |
+                  source kind/requirements.env
+                  export PATH=$PATH:/tmp/
+                  curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
+                  curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+                  chmod 700 /tmp/get_helm.sh
+                  HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
+                  chmod 777 /tmp/helm
+                  chmod 777 /tmp/kind
+                  curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
+                  chmod 777 /tmp/kubectl
+                  curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
+                  chmod +x /tmp/mc
+                  export DEPLOY_KUBEFLOW=0
+                  make -C kind setup 
+                  make -C kfp/kfp_support_lib build test
+    test-kfp-compile:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Test KFP compile
+              run: |
+                  make -C kfp/transform_workflows venv build

.make.defaults

@@ -196,6 +196,21 @@ __check_defined = \
 	@# Help: Test an $(DOCKER_IMAGE) use test source inside the image. 
 	$(DOCKER) run -t --rm $(DOCKER_IMAGE) pytest -s test 
 
+.PHONY: .defaults.test-locals
+.defaults.test-locals::
+	@# Help: Run the *local*.py files in the src directory
+	@source venv/bin/activate;              \
+	export PYTHONPATH=src;                  \
+	files=$$(find src -name '*local*.py' );         \
+	for i in $$files; do                    \
+	    echo Executing: python $$i;         \
+	    python $$i;                         \
+	    if [ $$? -ne 0 ];   then            \
+		exit 1;                         \
+	    fi;                                 \
+	done   
+
+
 # This expects the image to already be built and so does not depending on .defaults.publish.
 # This allows others to define their own image building prior to publishing.
 .PHONY: .defaults.publish
@@ -219,3 +234,67 @@ __check_defined = \
 	pip install wheel;			\
 	pip install -r requirements.txt;	\
 
+.PHONY: .defaults.check.installed
+.defaults.check.installed::
+	@installed=$$(which $(CHECK_RUNNABLE));		\
+	if [ -z "$$installed" ]; then			\
+	    echo $(CHECK_RUNNABLE) must be installed;	\
+	    exit 1;					\
+	fi
+
+.PHONY: .defaults.minio.check
+.defaults.minio.check::
+	@# Help: Check that minio and the mc CLI is installed. 
+	$(MAKE) CHECK_RUNNABLE=minio .defaults.check.installed
+	$(MAKE) CHECK_RUNNABLE=mc .defaults.check.installed
+
+MINIO_DIR=/tmp/data-prep-lab
+MINIO_ALIAS=local
+# These are the credentials used by samples.
+MINIO_ADMIN_USER=localminioaccesskey
+MINIO_ADMIN_PWD= localminiosecretkey
+
+.PHONY: .defaults.minio.verify-running
+.defaults.minio.verify-running:
+	@z=$$(ps -elf | grep minio | grep -v grep);	\
+	if [ -z "$$z" ]; then				\
+	    echo "";					\
+	    echo "Please start minio (see make help)";	\
+	    echo "";					\
+	    exit 1;					\
+	fi
+
+.PHONY: .defaults.minio.stop
+.defaults.minio.stop::
+	$(MAKE) .defaults.minio.check
+	@# Help: Stop the minio server used for S3-based samples
+	-mc admin service stop $(MINIO_ALIAS) > /dev/null 2>&1
+	-mc alias rm $(MINIO_ALIAS)  > /dev/null 2>&1
+
+.PHONY: .defaults.minio.start
+.defaults.minio.start::
+	$(MAKE) .defaults.minio.stop
+	@# Help: Stop, then start, the minio server used for S3-based samples
+	rm -rf $(MINIO_DIR) 
+	mkdir  $(MINIO_DIR) 
+	echo Starting minio server			
+	minio server $(MINIO_DIR) &
+	sleep 1
+	-mc alias rm $(MINIO_ALIAS)  > /dev/null 2>&1
+	mc alias set $(MINIO_ALIAS) http://127.0.0.1:9000 minioadmin minioadmin
+	mc admin user svcacct add --access-key "$(MINIO_ADMIN_USER)" --secret-key "$(MINIO_ADMIN_PWD)" $(MINIO_ALIAS) minioadmin
+	mc alias ls $(MINIO_ALIAS)
+	@echo "Test minio server started" 
+
+.PHONY: .defaults.minio.load-test-data
+.defaults.minio.load-test-data::
+	@# Help: Load the minio server with S3 sample data from $(MINIO_SRC) to $(MINIO_DEST) 
+	@if [ -z "$(MINIO_SRC)" -o -z "$(MINIO_DEST)" ]; then	\
+		echo MINIO_SRC and MINIO_DEST must be defined when use $? target;\
+		exit 1;						\
+	fi
+	$(MAKE) .defaults.minio.check
+	@# Help: Load $(MINIO_SRC) into minio at local/test/$(MINIO_DEST) 
+	-mc mb  local/test > /dev/null 2>&1	# Igore if it already exists
+	mc cp --recursive $(MINIO_SRC) local/test/$(MINIO_DEST)
+

.make.versions

@@ -4,8 +4,8 @@
 ################################################################################
 
 # Data prep lab wheel version
-DPL_LIB_VERSION=0.1.3
-DPL_LIB_KFP_VERSION=0.1.4
+DPL_LIB_VERSION=0.1.5
+DPL_LIB_KFP_VERSION=0.1.6
 
 # Begin transform versions/tags
 BLOCKLIST_VERSION=0.2.0
@@ -21,6 +21,7 @@ MALWARE_VERSION=0.3
 PROGLANG_SELECT_VERSION=0.2.0
 CODE_QUALITY_VERSION=0.2.0
 DOC_QUALITY_VERSION=0.2.0
+INGEST_TO_PARQUET_VERSION=0.1.0
 
-KFP_DOCKER_VERSION=0.0.5
+KFP_DOCKER_VERSION=0.0.6
 

MAINTAINERS.md

@@ -4,3 +4,5 @@ David Wood - dawood@us.ibm.com
 
 Boris Lublinsky - blublinsky@ibm.com
 
+Revital Eres - eres@il.ibm.com
+