Workaround for pydantic 2.0 incompatibility, new AWS glue job runner, and new makefile targets.

Makefile

@@ -1,21 +1,37 @@
-clean:
-	rm -rf venv
-
-install:
+venv:
 	if [ ! -d "venv" ]; then \
   		if [[ '$(shell uname -m)' == 'arm64' ]]; then \
 			/usr/bin/python3 -m venv venv; \
 		else \
 			python -m venv venv; \
 		fi \
 	fi
+
+clean-build:
+	rm -rf dist
+	rm -rf build
+
+clean-venv:
+	rm -rf venv
+
+clean: clean-build clean-venv
+
+build: venv
+	venv/bin/python setup.py sdist bdist_wheel
+
+rebuild: clean-build build
+
+deploy-s3:
+	./s3-build-and-deploy.sh
+
+install: venv
 	venv/bin/pip install --upgrade pip
 	venv/bin/pip install -r dev-requirements.txt
 
-lint:
+lint: venv
 	venv/bin/pre-commit run --all-files
 
-test-integration:
+test-integration: install
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml kill
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml rm -f
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml up -d

deltacat/examples/common/fixtures.py

@@ -0,0 +1,54 @@
+import os
+import logging
+from deltacat import logs
+
+from deltacat.constants import (
+    DELTACAT_APP_LOG_LEVEL,
+    DELTACAT_SYS_LOG_LEVEL,
+    DELTACAT_APP_LOG_DIR,
+    DELTACAT_SYS_LOG_DIR,
+    DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
+    DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
+    DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
+    DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
+)
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def create_runtime_environment():
+    # read packages to install in the runtime environment from system environment vars
+    package = os.environ.get("PACKAGE")
+    logger.debug(f"Pip Install Packages: {package}")
+
+    runtime_environment = None
+    if package:
+        # read the stage (e.g. alpha, beta, dev, etc.) from the system environment vars
+        stage = os.environ["STAGE"]
+        logger.debug(f"Job Run Stage: {stage}")
+
+        # log the job run system environment for debugging
+        logger.debug(f"Job Run System Environment: {os.environ}")
+
+        worker_env_vars = {
+            # forward the STAGE environment variable to workers
+            "STAGE": stage,
+            # forward deltacat logging environment variables to workers
+            "DELTACAT_APP_LOG_LEVEL": DELTACAT_APP_LOG_LEVEL,
+            "DELTACAT_SYS_LOG_LEVEL": DELTACAT_SYS_LOG_LEVEL,
+            "DELTACAT_APP_LOG_DIR": DELTACAT_APP_LOG_DIR,
+            "DELTACAT_SYS_LOG_DIR": DELTACAT_SYS_LOG_DIR,
+            "DELTACAT_APP_INFO_LOG_BASE_FILE_NAME": DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
+            "DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME": DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
+            "DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME": DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
+            "DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME": DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
+        }
+
+        # setup runtime environment from system environment variables:
+        #   1. STAGE: The execution environment stage (e.g. alpha, beta, dev, etc.)
+        #   2. PACKAGE: Package to Pip Install. (e.g. signed S3 URL for DeltaCAT)
+        runtime_environment = {
+            "env_vars": worker_env_vars,
+            "pip": package,
+        }
+    return runtime_environment

deltacat/examples/hello_world.py

@@ -0,0 +1,18 @@
+import ray
+import deltacat
+
+
+@ray.remote
+def hello_worker():
+    print("Hello, Worker!")
+    print(f"Worker DeltaCAT Version: {deltacat.__version__}")
+
+
+def run():
+    print("Hello, Driver!")
+    print(f"Driver DeltaCAT Version: {deltacat.__version__}")
+    hello_worker.remote()
+
+
+if __name__ == "__main__":
+    run()

deltacat/examples/logging.py

@@ -0,0 +1,33 @@
+import os
+import sys
+import ray
+import logging
+
+from deltacat import logs
+from deltacat.examples.common.fixtures import create_runtime_environment
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+@ray.remote
+def logging_worker(var1, var2):
+    logger.debug(f"Worker System Environment: {os.environ}")
+    logger.debug(f"Worker Variable 1: {var1}")
+    logger.debug(f"Worker Variable 2: {var2}")
+
+
+def run(var1="default1", var2="default2", **kwargs):
+    logger.debug(f"Driver Variable 1: {var1}")
+    logger.debug(f"Driver Variable 2: {var2}")
+    logging_worker.remote(var1, var2)
+
+
+if __name__ == "__main__":
+    # create any runtime environment required to run the example
+    runtime_env = create_runtime_environment()
+
+    # initialize ray
+    ray.init(address="auto", runtime_env=runtime_env)
+
+    # run the example using os.environ as kwargs
+    run(**os.environ)

dev-requirements.txt

@@ -8,3 +8,6 @@ pre-commit == 2.20.0
 pytest == 7.2.0
 pytest-cov == 4.0.0
 requests-mock == 1.11.0
+pyiceberg @ git+https://github.com/apache/iceberg#subdirectory=python
+pydantic == 2.0
+ray[data] ~= 2.6