fix: Adding protobuf<5 as a required dependency due to snowflake limi…

…tations (#4537) * fix: Removed protobuf as a required dependency Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com> * fix: Removed install-protoc-dependencies target Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com> * fix: Ran lock python dependencies to correct dependencies Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com> * fix: Adding protobuf<5 as a required dependency due to snowflake limitations Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com> --------- Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com> Co-authored-by: Bhargav Dodla <bdodla@expediagroup.com>
feast-dev · Sep 19, 2024 · cecca83 · cecca83
1 parent 50b8f23
commit cecca83
Show file tree

Hide file tree

Showing 9 changed files with 163 additions and 153 deletions.
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -97,6 +97,7 @@ jobs:
         # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum.
         run: |
           pip install -U pip setuptools wheel twine
+          make install-protoc-dependencies
           make build-ui
           git status
           git restore go.mod go.sum

diff --git a/Makefile b/Makefile
@@ -52,13 +52,16 @@ install-python-ci-dependencies-uv-venv:
 	uv pip install --no-deps -e .
 	python setup.py build_python_protos --inplace
 
+install-protoc-dependencies:
+	pip install "protobuf<5" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1"
+
 lock-python-ci-dependencies:
 	uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
 
 package-protos:
 	cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
 
-compile-protos-python:
+compile-protos-python: install-protoc-dependencies
 	python setup.py build_python_protos --inplace
 
 install-python:
@@ -69,12 +72,14 @@ lock-python-dependencies:
 	uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py$(PYTHON_VERSION)-requirements.txt
 
 lock-python-dependencies-all:
-	pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt"
-	pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt"
-	pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt"
-	pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt"
-	pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt"
-	pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt"
+	# Remove all existing requirements because we noticed the lock file is not always updated correctly. Removing and running the command again ensures that the lock file is always up to date.
+	rm -r sdk/python/requirements/*
+	pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.9 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt"
+	pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt"
+	pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.10 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt"
+	pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt"
+	pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.11 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt"
+	pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt"
 
 benchmark-python:
 	IS_TEST=True python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data sdk/python/tests

diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt
@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt
+#    uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt
 aiobotocore==2.15.0
     # via feast (setup.py)
 aiohappyeyeballs==2.4.0
@@ -16,7 +16,7 @@ altair==4.2.2
     # via great-expectations
 annotated-types==0.7.0
     # via pydantic
-anyio==4.4.0
+anyio==4.5.0
     # via
     #   httpx
     #   jupyter-server
@@ -51,13 +51,13 @@ attrs==24.2.0
     #   aiohttp
     #   jsonschema
     #   referencing
-azure-core==1.30.2
+azure-core==1.31.0
     # via
     #   azure-identity
     #   azure-storage-blob
 azure-identity==1.17.1
     # via feast (setup.py)
-azure-storage-blob==12.22.0
+azure-storage-blob==12.23.0
     # via feast (setup.py)
 babel==2.16.0
     # via
@@ -144,11 +144,11 @@ cryptography==42.0.8
     #   types-redis
 cython==3.0.11
     # via thriftpy2
-dask[dataframe]==2024.8.2
+dask[dataframe]==2024.9.0
     # via
     #   feast (setup.py)
     #   dask-expr
-dask-expr==1.1.13
+dask-expr==1.1.14
     # via dask
 db-dtypes==1.3.0
     # via google-cloud-bigquery
@@ -158,7 +158,7 @@ decorator==5.1.1
     # via ipython
 defusedxml==0.7.1
     # via nbconvert
-deltalake==0.19.2
+deltalake==0.20.0
     # via feast (setup.py)
 deprecation==2.1.0
     # via python-keycloak
@@ -187,11 +187,11 @@ execnet==2.1.1
     # via pytest-xdist
 executing==2.1.0
     # via stack-data
-fastapi==0.114.1
+fastapi==0.115.0
     # via feast (setup.py)
 fastjsonschema==2.20.0
     # via nbformat
-filelock==3.16.0
+filelock==3.16.1
     # via
     #   snowflake-connector-python
     #   virtualenv
@@ -256,7 +256,7 @@ googleapis-common-protos[grpc]==1.65.0
     #   google-api-core
     #   grpc-google-iam-v1
     #   grpcio-status
-great-expectations==0.18.20
+great-expectations==0.18.21
     # via feast (setup.py)
 grpc-google-iam-v1==0.13.1
     # via google-cloud-bigtable
@@ -302,15 +302,15 @@ httpx==0.27.2
     #   feast (setup.py)
     #   jupyterlab
     #   python-keycloak
-ibis-framework[duckdb]==9.4.0
+ibis-framework[duckdb]==9.5.0
     # via
     #   feast (setup.py)
     #   ibis-substrait
 ibis-substrait==4.0.1
     # via feast (setup.py)
-identify==2.6.0
+identify==2.6.1
     # via pre-commit
-idna==3.8
+idna==3.10
     # via
     #   anyio
     #   httpx
@@ -321,7 +321,9 @@ idna==3.8
 imagesize==1.4.1
     # via sphinx
 importlib-metadata==8.5.0
-    # via dask
+    # via
+    #   build
+    #   dask
 iniconfig==2.0.0
     # via pytest
 ipykernel==6.29.5
@@ -372,7 +374,7 @@ jsonschema[format-nongpl]==4.23.0
     #   nbformat
 jsonschema-specifications==2023.12.1
     # via jsonschema
-jupyter-client==8.6.2
+jupyter-client==8.6.3
     # via
     #   ipykernel
     #   jupyter-server
@@ -438,7 +440,7 @@ mistune==3.0.2
     # via
     #   great-expectations
     #   nbconvert
-mmh3==4.1.0
+mmh3==5.0.0
     # via feast (setup.py)
 mock==2.0.0
     # via feast (setup.py)
@@ -506,6 +508,7 @@ packaging==24.1
     #   google-cloud-bigquery
     #   great-expectations
     #   gunicorn
+    #   ibis-framework
     #   ibis-substrait
     #   ipykernel
     #   jupyter-server
@@ -570,8 +573,9 @@ proto-plus==1.24.0
     #   google-cloud-bigquery-storage
     #   google-cloud-bigtable
     #   google-cloud-datastore
-protobuf==4.25.4
+protobuf==4.25.5
     # via
+    #   feast (setup.py)
     #   google-api-core
     #   google-cloud-bigquery-storage
     #   google-cloud-bigtable
@@ -590,11 +594,11 @@ psutil==5.9.0
     # via
     #   feast (setup.py)
     #   ipykernel
-psycopg[binary, pool]==3.2.1
+psycopg[binary, pool]==3.2.2
     # via feast (setup.py)
-psycopg-binary==3.2.1
+psycopg-binary==3.2.2
     # via psycopg
-psycopg-pool==3.2.2
+psycopg-pool==3.2.3
     # via psycopg
 ptyprocess==0.7.0
     # via
@@ -629,12 +633,12 @@ pybindgen==0.22.1
     # via feast (setup.py)
 pycparser==2.22
     # via cffi
-pydantic==2.9.1
+pydantic==2.9.2
     # via
     #   feast (setup.py)
     #   fastapi
     #   great-expectations
-pydantic-core==2.23.3
+pydantic-core==2.23.4
     # via pydantic
 pygments==2.18.0
     # via
@@ -739,7 +743,7 @@ referencing==0.35.1
     #   jsonschema
     #   jsonschema-specifications
     #   jupyter-events
-regex==2024.7.24
+regex==2024.9.11
     # via
     #   feast (setup.py)
     #   parsimonious
@@ -790,15 +794,15 @@ ruamel-yaml==0.17.40
     # via great-expectations
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-ruff==0.6.4
+ruff==0.6.5
     # via feast (setup.py)
 s3transfer==0.10.2
     # via boto3
 scipy==1.14.1
     # via great-expectations
 send2trash==1.8.3
     # via jupyter-server
-setuptools==74.1.2
+setuptools==75.1.0
     # via
     #   grpcio-tools
     #   jupyterlab
@@ -826,7 +830,7 @@ sniffio==1.3.1
     #   httpx
 snowballstemmer==2.2.0
     # via sphinx
-snowflake-connector-python[pandas]==3.12.1
+snowflake-connector-python[pandas]==3.12.2
     # via feast (setup.py)
 sortedcontainers==2.4.0
     # via snowflake-connector-python
@@ -846,9 +850,9 @@ sphinxcontrib-qthelp==2.0.0
     # via sphinx
 sphinxcontrib-serializinghtml==2.0.0
     # via sphinx
-sqlalchemy[mypy]==2.0.34
+sqlalchemy[mypy]==2.0.35
     # via feast (setup.py)
-sqlglot==25.18.0
+sqlglot==25.20.1
     # via ibis-framework
 sqlite-vec==0.1.1
     # via feast (setup.py)
@@ -858,7 +862,7 @@ stack-data==0.6.3
     # via ipython
 starlette==0.38.5
     # via fastapi
-substrait==0.22.0
+substrait==0.23.0
     # via ibis-substrait
 tabulate==0.9.0
     # via feast (setup.py)
@@ -939,15 +943,15 @@ types-python-dateutil==2.9.0.20240906
     # via
     #   feast (setup.py)
     #   arrow
-types-pytz==2024.1.0.20240417
+types-pytz==2024.2.0.20240913
     # via feast (setup.py)
-types-pyyaml==6.0.12.20240808
+types-pyyaml==6.0.12.20240917
     # via feast (setup.py)
 types-redis==4.6.0.20240903
     # via feast (setup.py)
 types-requests==2.30.0.0
     # via feast (setup.py)
-types-setuptools==74.1.0.20240907
+types-setuptools==75.1.0.20240917
     # via
     #   feast (setup.py)
     #   types-cffi
@@ -986,7 +990,7 @@ tzlocal==5.2
     #   trino
 uri-template==1.3.0
     # via jsonschema
-urllib3==2.2.2
+urllib3==2.2.3
     # via
     #   feast (setup.py)
     #   botocore
@@ -1038,5 +1042,5 @@ xmltodict==0.13.0
     # via moto
 yarl==1.11.1
     # via aiohttp
-zipp==3.20.1
+zipp==3.20.2
     # via importlib-metadata