From 8178190ea20513aa7ad14c7a133cfdcaaee94f3a Mon Sep 17 00:00:00 2001
From: Valentyn Tymofieiev <valentyn@google.com>
Date: Mon, 4 Dec 2023 16:32:17 -0800
Subject: [PATCH 1/3] Add missing Python 3.11 dependencies.

---
 .../workflows/update_python_dependencies.yml  | 12 +++++-----
 .../base_image_requirements_manual.txt        |  9 +++-----
 .../py310/base_image_requirements.txt         | 20 ++++++++---------
 .../py311/base_image_requirements.txt         | 22 +++++++++++--------
 .../py38/base_image_requirements.txt          | 22 +++++++++----------
 .../py39/base_image_requirements.txt          | 22 +++++++++----------
 .../container/run_generate_requirements.sh    | 10 +++++++++
 7 files changed, 65 insertions(+), 52 deletions(-)

diff --git a/.github/workflows/update_python_dependencies.yml b/.github/workflows/update_python_dependencies.yml
index b4b839c3204c9..48c75de30b98a 100644
--- a/.github/workflows/update_python_dependencies.yml
+++ b/.github/workflows/update_python_dependencies.yml
@@ -16,7 +16,7 @@
 # under the License.
 
 # This workflow will update python dependencies as part of the release process
-# And commit the changes into the branch release, creating a PR into the branch 
+# And commit the changes into the branch release, creating a PR into the branch
 
 name: Update Python Depedencies
 
@@ -38,7 +38,7 @@ jobs:
       properties: ${{ steps.test-properties.outputs.properties }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4        
+        uses: actions/checkout@v4
       - id: test-properties
         uses: ./.github/actions/setup-default-test-properties
 
@@ -57,6 +57,8 @@ jobs:
             3.9
           java-version: 8
           go-version: 1.21
+      - name: Install headers required by some python packages
+        run: sudo apt-get install -y libsnappy-dev
       - name: Update Python Dependencies
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
@@ -71,8 +73,8 @@ jobs:
         run: |
           branchName=weekly_update_python_dependencies_$(date +%s)
           git checkout -b $branchName
-          git add -A 
-          git diff-index --quiet HEAD || gitdiff=$? || echo $? 
+          git add -A
+          git diff-index --quiet HEAD || gitdiff=$? || echo $?
           if [[ $gitDiff != 0 ]]; then
             echo "Changes are ready to commit"
             git commit -m "Update Python Dependencies" --quiet
@@ -81,4 +83,4 @@ jobs:
             echo "Link of the new PR  $GITHUB_PR_URL"
           else
             echo "No changes on the files"
-          fi
\ No newline at end of file
+          fi
diff --git a/sdks/python/container/base_image_requirements_manual.txt b/sdks/python/container/base_image_requirements_manual.txt
index f2f3ea44b44c7..144512744f1b2 100644
--- a/sdks/python/container/base_image_requirements_manual.txt
+++ b/sdks/python/container/base_image_requirements_manual.txt
@@ -32,15 +32,12 @@ cython<1
 # some versions of libraries that launch Beam pipelines, like tensorflow-transform.
 # Leaving 'future' in our containers for now prevent breaking tft users.
 future
-# TODO(https://github.com/GoogleCloudPlatform/cloud-profiler-python/issues/127): Unpin python version.
-google-cloud-profiler;python_version<"3.11"
+google-cloud-profiler
 guppy3
 mmh3  # Optimizes execution of some Beam codepaths. TODO: Make it Beam's dependency.
 nltk  # Commonly used for natural language processing.
 nose==1.3.7  # For Dataflow internal testing. TODO: remove this.
-# TODO: Uncomment python version once python-snappy supports it.
-# https://github.com/apache/beam/issues/25985
-python-snappy;python_version<"3.11"  # Optimizes execution of some Beam codepaths.
+python-snappy  # Optimizes execution of some Beam codepaths.
 scipy
 scikit-learn
-build>=1.0,<2 # tool to build sdist from setup.py in stager.
\ No newline at end of file
+build>=1.0,<2 # tool to build sdist from setup.py in stager.
diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt
index a80e6611d9cf3..40ddbbe297ae7 100644
--- a/sdks/python/container/py310/base_image_requirements.txt
+++ b/sdks/python/container/py310/base_image_requirements.txt
@@ -32,7 +32,7 @@ charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.6
+cryptography==41.0.7
 Cython==0.29.36
 deprecation==2.1.0
 dill==0.3.1.1
@@ -43,16 +43,16 @@ exceptiongroup==1.2.0
 execnet==2.0.2
 fastavro==1.9.0
 fasteners==0.19
-freezegun==1.2.2
+freezegun==1.3.1
 future==0.18.3
 google-api-core==2.14.0
-google-api-python-client==2.108.0
+google-api-python-client==2.109.0
 google-apitools==0.5.31
-google-auth==2.23.4
+google-auth==2.24.0
 google-auth-httplib2==0.1.1
 google-cloud-aiplatform==1.36.4
 google-cloud-bigquery==3.13.0
-google-cloud-bigquery-storage==2.22.0
+google-cloud-bigquery-storage==2.23.0
 google-cloud-bigtable==2.21.0
 google-cloud-core==2.3.3
 google-cloud-datastore==2.18.0
@@ -83,7 +83,7 @@ iniconfig==2.0.0
 joblib==1.3.2
 Js2Py==0.74
 jsonschema==4.20.0
-jsonschema-specifications==2023.11.1
+jsonschema-specifications==2023.11.2
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
@@ -108,7 +108,7 @@ pycparser==2.21
 pydot==1.4.2
 PyHamcrest==2.1.0
 pyjsparser==2.7.1
-pymongo==4.6.0
+pymongo==4.6.1
 PyMySQL==1.1.0
 pyparsing==3.1.1
 pyproject_hooks==1.0.0
@@ -119,11 +119,11 @@ python-dateutil==2.8.2
 python-snappy==0.6.1
 pytz==2023.3.post1
 PyYAML==6.0.1
-referencing==0.31.0
+referencing==0.31.1
 regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
-rpds-py==0.13.1
+rpds-py==0.13.2
 rsa==4.9
 scikit-learn==1.3.2
 scipy==1.11.4
@@ -142,6 +142,6 @@ typing_extensions==4.8.0
 tzlocal==5.2
 uritemplate==4.1.1
 urllib3==2.1.0
-websocket-client==1.6.4
+websocket-client==1.7.0
 wrapt==1.16.0
 zstandard==0.22.0
diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt
index 04d9fc28c273e..c2e75ccc812c3 100644
--- a/sdks/python/container/py311/base_image_requirements.txt
+++ b/sdks/python/container/py311/base_image_requirements.txt
@@ -32,7 +32,7 @@ charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.6
+cryptography==41.0.7
 Cython==0.29.36
 deprecation==2.1.0
 dill==0.3.1.1
@@ -42,20 +42,22 @@ docopt==0.6.2
 execnet==2.0.2
 fastavro==1.9.0
 fasteners==0.19
-freezegun==1.2.2
+freezegun==1.3.1
 future==0.18.3
 google-api-core==2.14.0
+google-api-python-client==2.109.0
 google-apitools==0.5.31
-google-auth==2.23.4
+google-auth==2.24.0
 google-auth-httplib2==0.1.1
 google-cloud-aiplatform==1.36.4
 google-cloud-bigquery==3.13.0
-google-cloud-bigquery-storage==2.22.0
+google-cloud-bigquery-storage==2.23.0
 google-cloud-bigtable==2.21.0
 google-cloud-core==2.3.3
 google-cloud-datastore==2.18.0
 google-cloud-dlp==3.13.0
 google-cloud-language==2.11.1
+google-cloud-profiler==4.1.0
 google-cloud-pubsub==2.18.4
 google-cloud-pubsublite==1.8.3
 google-cloud-recommendations-ai==0.10.5
@@ -80,7 +82,7 @@ iniconfig==2.0.0
 joblib==1.3.2
 Js2Py==0.74
 jsonschema==4.20.0
-jsonschema-specifications==2023.11.1
+jsonschema-specifications==2023.11.2
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
@@ -105,7 +107,7 @@ pycparser==2.21
 pydot==1.4.2
 PyHamcrest==2.1.0
 pyjsparser==2.7.1
-pymongo==4.6.0
+pymongo==4.6.1
 PyMySQL==1.1.0
 pyparsing==3.1.1
 pyproject_hooks==1.0.0
@@ -113,13 +115,14 @@ pytest==7.4.3
 pytest-timeout==2.2.0
 pytest-xdist==3.5.0
 python-dateutil==2.8.2
+python-snappy==0.6.1
 pytz==2023.3.post1
 PyYAML==6.0.1
-referencing==0.31.0
+referencing==0.31.1
 regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
-rpds-py==0.13.1
+rpds-py==0.13.2
 rsa==4.9
 scikit-learn==1.3.2
 scipy==1.11.4
@@ -135,7 +138,8 @@ threadpoolctl==3.2.0
 tqdm==4.66.1
 typing_extensions==4.8.0
 tzlocal==5.2
+uritemplate==4.1.1
 urllib3==2.1.0
-websocket-client==1.6.4
+websocket-client==1.7.0
 wrapt==1.16.0
 zstandard==0.22.0
diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt
index f22bbfe7d7194..becb02c30e7a9 100644
--- a/sdks/python/container/py38/base_image_requirements.txt
+++ b/sdks/python/container/py38/base_image_requirements.txt
@@ -33,7 +33,7 @@ charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.6
+cryptography==41.0.7
 Cython==0.29.36
 deprecation==2.1.0
 dill==0.3.1.1
@@ -44,16 +44,16 @@ exceptiongroup==1.2.0
 execnet==2.0.2
 fastavro==1.9.0
 fasteners==0.19
-freezegun==1.2.2
+freezegun==1.3.1
 future==0.18.3
 google-api-core==2.14.0
-google-api-python-client==2.108.0
+google-api-python-client==2.109.0
 google-apitools==0.5.31
-google-auth==2.23.4
+google-auth==2.24.0
 google-auth-httplib2==0.1.1
 google-cloud-aiplatform==1.36.4
 google-cloud-bigquery==3.13.0
-google-cloud-bigquery-storage==2.22.0
+google-cloud-bigquery-storage==2.23.0
 google-cloud-bigtable==2.21.0
 google-cloud-core==2.3.3
 google-cloud-datastore==2.18.0
@@ -80,13 +80,13 @@ hdfs==2.7.3
 httplib2==0.22.0
 hypothesis==6.91.0
 idna==3.6
-importlib-metadata==6.8.0
+importlib-metadata==7.0.0
 importlib-resources==6.1.1
 iniconfig==2.0.0
 joblib==1.3.2
 Js2Py==0.74
 jsonschema==4.20.0
-jsonschema-specifications==2023.11.1
+jsonschema-specifications==2023.11.2
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
@@ -112,7 +112,7 @@ pycparser==2.21
 pydot==1.4.2
 PyHamcrest==2.1.0
 pyjsparser==2.7.1
-pymongo==4.6.0
+pymongo==4.6.1
 PyMySQL==1.1.0
 pyparsing==3.1.1
 pyproject_hooks==1.0.0
@@ -123,11 +123,11 @@ python-dateutil==2.8.2
 python-snappy==0.6.1
 pytz==2023.3.post1
 PyYAML==6.0.1
-referencing==0.31.0
+referencing==0.31.1
 regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
-rpds-py==0.13.1
+rpds-py==0.13.2
 rsa==4.9
 scikit-learn==1.3.2
 scipy==1.10.1
@@ -146,7 +146,7 @@ typing_extensions==4.8.0
 tzlocal==5.2
 uritemplate==4.1.1
 urllib3==2.1.0
-websocket-client==1.6.4
+websocket-client==1.7.0
 wrapt==1.16.0
 zipp==3.17.0
 zstandard==0.22.0
diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt
index f42b6f8c2b456..5186e8819f7ba 100644
--- a/sdks/python/container/py39/base_image_requirements.txt
+++ b/sdks/python/container/py39/base_image_requirements.txt
@@ -32,7 +32,7 @@ charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.6
+cryptography==41.0.7
 Cython==0.29.36
 deprecation==2.1.0
 dill==0.3.1.1
@@ -43,16 +43,16 @@ exceptiongroup==1.2.0
 execnet==2.0.2
 fastavro==1.9.0
 fasteners==0.19
-freezegun==1.2.2
+freezegun==1.3.1
 future==0.18.3
 google-api-core==2.14.0
-google-api-python-client==2.108.0
+google-api-python-client==2.109.0
 google-apitools==0.5.31
-google-auth==2.23.4
+google-auth==2.24.0
 google-auth-httplib2==0.1.1
 google-cloud-aiplatform==1.36.4
 google-cloud-bigquery==3.13.0
-google-cloud-bigquery-storage==2.22.0
+google-cloud-bigquery-storage==2.23.0
 google-cloud-bigtable==2.21.0
 google-cloud-core==2.3.3
 google-cloud-datastore==2.18.0
@@ -79,12 +79,12 @@ hdfs==2.7.3
 httplib2==0.22.0
 hypothesis==6.91.0
 idna==3.6
-importlib-metadata==6.8.0
+importlib-metadata==7.0.0
 iniconfig==2.0.0
 joblib==1.3.2
 Js2Py==0.74
 jsonschema==4.20.0
-jsonschema-specifications==2023.11.1
+jsonschema-specifications==2023.11.2
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
@@ -109,7 +109,7 @@ pycparser==2.21
 pydot==1.4.2
 PyHamcrest==2.1.0
 pyjsparser==2.7.1
-pymongo==4.6.0
+pymongo==4.6.1
 PyMySQL==1.1.0
 pyparsing==3.1.1
 pyproject_hooks==1.0.0
@@ -120,11 +120,11 @@ python-dateutil==2.8.2
 python-snappy==0.6.1
 pytz==2023.3.post1
 PyYAML==6.0.1
-referencing==0.31.0
+referencing==0.31.1
 regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
-rpds-py==0.13.1
+rpds-py==0.13.2
 rsa==4.9
 scikit-learn==1.3.2
 scipy==1.11.4
@@ -143,7 +143,7 @@ typing_extensions==4.8.0
 tzlocal==5.2
 uritemplate==4.1.1
 urllib3==2.1.0
-websocket-client==1.6.4
+websocket-client==1.7.0
 wrapt==1.16.0
 zipp==3.17.0
 zstandard==0.22.0
diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh
index 6c160bc6ac9e8..a8ce6df27ae92 100755
--- a/sdks/python/container/run_generate_requirements.sh
+++ b/sdks/python/container/run_generate_requirements.sh
@@ -30,6 +30,16 @@
 # You will need Python interpreters for all versions supported by Beam, see:
 # https://s.apache.org/beam-python-dev-wiki
 
+if ! [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
+  echo This script needs to be executed in a Linux environment.
+  exit 1
+fi
+
+if ! dpkg -s libsnappy-dev >/dev/null ; then
+  echo You must install libsnappy-dev to run this script. Run:
+  echo sudo apt install libsnappy-dev
+fi
+
 if [[ $# -lt 2 ]]; then
   printf "Example usage: \n$> ./sdks/python/container/run_generate_requirements.sh 3.8 <sdk_tarball>"
   printf "\n\where 3.8 is the Python major.minor version."

From 6891007140723c739b2a328c96d0a86ebb1ef96f Mon Sep 17 00:00:00 2001
From: Valentyn Tymofieiev <valentyn@google.com>
Date: Mon, 4 Dec 2023 17:04:38 -0800
Subject: [PATCH 2/3] Also install headers in the python dependency test suite.

---
 .github/workflows/python_dependency_tests.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/python_dependency_tests.yml b/.github/workflows/python_dependency_tests.yml
index 166899df90cbc..329b4ca090d07 100644
--- a/.github/workflows/python_dependency_tests.yml
+++ b/.github/workflows/python_dependency_tests.yml
@@ -38,6 +38,8 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.params.py_ver }}
+      - name: Install headers required by some python packages
+        run: sudo apt-get install -y libsnappy-dev
       - name: Install base_image_requirements.txt
         working-directory: ./sdks/python
         run: pip install --no-deps -r container/${{ matrix.params.py_env }}/base_image_requirements.txt

From e6627d2b03d78c39787141d942a6cacb58488808 Mon Sep 17 00:00:00 2001
From: Valentyn Tymofieiev <valentyn@google.com>
Date: Tue, 5 Dec 2023 11:21:27 -0800
Subject: [PATCH 3/3] Add snappy instructions for MacOS

---
 .../container/run_generate_requirements.sh    | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh
index a8ce6df27ae92..1df7904776411 100755
--- a/sdks/python/container/run_generate_requirements.sh
+++ b/sdks/python/container/run_generate_requirements.sh
@@ -30,14 +30,18 @@
 # You will need Python interpreters for all versions supported by Beam, see:
 # https://s.apache.org/beam-python-dev-wiki
 
-if ! [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
-  echo This script needs to be executed in a Linux environment.
-  exit 1
-fi
-
-if ! dpkg -s libsnappy-dev >/dev/null ; then
-  echo You must install libsnappy-dev to run this script. Run:
-  echo sudo apt install libsnappy-dev
+if [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
+  if ! dpkg -s libsnappy-dev >/dev/null ; then
+    echo You must install libsnappy-dev to run this script. Run:
+    echo sudo apt install libsnappy-dev
+    exit 1
+  fi
+elif [[ "$(expr substr $(uname -s) 1 6)" == "Darwin" ]]; then
+  if ! brew list snappy >/dev/null ; then
+    echo You must install snappy to run this script. Run:
+    echo brew install snappy
+    exit 1
+  fi
 fi
 
 if [[ $# -lt 2 ]]; then