From a69734f87cf1266f0066b195fa0d9bbf31c6f3c0 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 23 Nov 2023 18:20:33 +0000 Subject: [PATCH 01/10] chore(python): Add Python 3.12 Source-Link: https://github.com/googleapis/synthtool/commit/af16e6d4672cc7b400f144de2fc3068b54ff47d2 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:bacc3af03bff793a03add584537b36b5644342931ad989e3ba1171d3bd5399f5 --- .github/.OwlBot.lock.yaml | 4 +- .kokoro/samples/python3.12/common.cfg | 40 ++++++++++++++++++++ .kokoro/samples/python3.12/continuous.cfg | 6 +++ .kokoro/samples/python3.12/periodic-head.cfg | 11 ++++++ .kokoro/samples/python3.12/periodic.cfg | 6 +++ .kokoro/samples/python3.12/presubmit.cfg | 6 +++ 6 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 .kokoro/samples/python3.12/common.cfg create mode 100644 .kokoro/samples/python3.12/continuous.cfg create mode 100644 .kokoro/samples/python3.12/periodic-head.cfg create mode 100644 .kokoro/samples/python3.12/periodic.cfg create mode 100644 .kokoro/samples/python3.12/presubmit.cfg diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 453b540c1e..eb4d9f794d 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:caffe0a9277daeccc4d1de5c9b55ebba0901b57c2f713ec9c876b0d4ec064f61 -# created: 2023-11-08T19:46:45.022803742Z + digest: sha256:bacc3af03bff793a03add584537b36b5644342931ad989e3ba1171d3bd5399f5 +# created: 2023-11-23T18:17:28.105124211Z diff --git a/.kokoro/samples/python3.12/common.cfg b/.kokoro/samples/python3.12/common.cfg new file mode 100644 index 0000000000..abf83e196d --- /dev/null +++ b/.kokoro/samples/python3.12/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.12" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-312" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-dataframes/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.12/continuous.cfg b/.kokoro/samples/python3.12/continuous.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.12/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.12/periodic-head.cfg b/.kokoro/samples/python3.12/periodic-head.cfg new file mode 100644 index 0000000000..123a35fbd3 --- /dev/null +++ b/.kokoro/samples/python3.12/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.12/periodic.cfg b/.kokoro/samples/python3.12/periodic.cfg new file mode 100644 index 0000000000..71cd1e597e --- /dev/null +++ b/.kokoro/samples/python3.12/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.12/presubmit.cfg b/.kokoro/samples/python3.12/presubmit.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.12/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file From 430f05155273b1ce790ddf313c42d08e19d5c479 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 23 Nov 2023 16:36:44 -0500 Subject: [PATCH 02/10] add python 3.12 to noxfile and setup.py --- noxfile.py | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 8d6d641fc1..857672cec4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -35,7 +35,7 @@ DEFAULT_PYTHON_VERSION = "3.10" -UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11"] +UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", "asyncmock", diff --git a/setup.py b/setup.py index 29eacb74a9..1b60489784 100644 --- a/setup.py +++ b/setup.py @@ -102,6 +102,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: OS Independent", "Topic :: Internet", ], From 7978916c5a1472723298ff6620326138bdd630d7 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 23 Nov 2023 16:37:57 -0500 Subject: [PATCH 03/10] update system test to use python 3.12 --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 857672cec4..566d3f4e94 100644 --- a/noxfile.py +++ b/noxfile.py @@ -50,7 +50,7 @@ UNIT_TEST_EXTRAS: List[str] = [] UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} -SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.11"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.12"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "jinja2", "mock", From 95e5dc51f27ab52be34d1b53e5856c42789fd37c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 30 Nov 2023 06:55:17 -0500 Subject: [PATCH 04/10] add constraints file for python 3.12 --- testing/constraints-3.12.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 testing/constraints-3.12.txt diff --git a/testing/constraints-3.12.txt b/testing/constraints-3.12.txt new file mode 100644 index 0000000000..e69de29bb2 From 9e9d9fd30d5080acec9c5f396d990c5c75c78a97 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 30 Nov 2023 06:56:06 -0500 Subject: [PATCH 05/10] add python 3.12 to owlbot.py --- owlbot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/owlbot.py b/owlbot.py index 082970018d..61043a63dd 100644 --- a/owlbot.py +++ b/owlbot.py @@ -29,8 +29,8 @@ # Add templated files # ---------------------------------------------------------------------------- templated_files = common.py_library( - unit_test_python_versions=["3.9", "3.10", "3.11"], - system_test_python_versions=["3.9", "3.11"], + unit_test_python_versions=["3.9", "3.10", "3.11", "3.12"], + system_test_python_versions=["3.9", "3.11", "3.12"], cov_level=35, intersphinx_dependencies={ "pandas": "https://pandas.pydata.org/pandas-docs/stable/", From 4c19ac65be8ac09e2dd5e7207024a51b7ee83bff Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 30 Nov 2023 11:58:05 +0000 Subject: [PATCH 06/10] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .github/workflows/unittest.yml | 2 +- CONTRIBUTING.rst | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 465199fc9a..f95a2cd602 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ['3.9', '3.10', '3.11'] + python: ['3.9', '3.10', '3.11', '3.12'] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f9103bfa72..93bcc94ec5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.9, 3.10 and 3.11 on both UNIX and Windows. + 3.9, 3.10, 3.11 and 3.12 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -72,7 +72,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.11 -- -k + $ nox -s unit-3.12 -- -k .. note:: @@ -143,12 +143,12 @@ Running System Tests $ nox -s system # Run a single system test - $ nox -s system-3.11 -- -k + $ nox -s system-3.12 -- -k .. note:: - System tests are only configured to run under Python 3.9 and 3.11. + System tests are only configured to run under Python 3.9, 3.11 and 3.12. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local @@ -224,10 +224,12 @@ We support: - `Python 3.9`_ - `Python 3.10`_ - `Python 3.11`_ +- `Python 3.12`_ .. _Python 3.9: https://docs.python.org/3.9/ .. _Python 3.10: https://docs.python.org/3.10/ .. _Python 3.11: https://docs.python.org/3.11/ +.. _Python 3.12: https://docs.python.org/3.12/ Supported versions can be found in our ``noxfile.py`` `config`_. From 71fb2df2121a90a3ab74bf250749a6573e32e1c4 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 8 Jan 2024 10:55:11 -0500 Subject: [PATCH 07/10] add python 3.12 as a required check --- .github/sync-repo-settings.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index cfa62f787c..9ffcb635af 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -15,6 +15,7 @@ branchProtectionRules: - 'unit (3.9)' - 'unit (3.10)' - 'unit (3.11)' + - 'unit (3.12)' - 'cover' - 'Kokoro presubmit' permissionRules: From be0871a9f889af8404c65eb2188fa2eb37100573 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 21 Mar 2024 21:56:18 +0000 Subject: [PATCH 08/10] skip recursion test --- bigframes/pandas/__init__.py | 3 +++ tests/system/small/test_dataframe.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/bigframes/pandas/__init__.py b/bigframes/pandas/__init__.py index 77f1af9dd4..067c1b1ebe 100644 --- a/bigframes/pandas/__init__.py +++ b/bigframes/pandas/__init__.py @@ -706,6 +706,9 @@ def to_datetime( # SQL Compilation uses recursive algorithms on deep trees # 10M tree depth should be sufficient to generate any sql that is under bigquery limit +# Note: This limit does not have the desired effect on Python 3.12 in +# which the applicable limit is now hard coded. See: +# https://github.com/python/cpython/issues/112282 sys.setrecursionlimit(max(10000000, sys.getrecursionlimit())) resource.setrlimit( resource.RLIMIT_STACK, (resource.RLIM_INFINITY, resource.RLIM_INFINITY) diff --git a/tests/system/small/test_dataframe.py b/tests/system/small/test_dataframe.py index 3b6cd8c05f..5e034e8e83 100644 --- a/tests/system/small/test_dataframe.py +++ b/tests/system/small/test_dataframe.py @@ -14,6 +14,7 @@ import io import operator +import sys import tempfile import typing from typing import Tuple @@ -4003,6 +4004,13 @@ def test_df_dot_operator_series( ) +# TODO(tswast): We may be able to re-enable this test after we break large +# queries up in https://github.com/googleapis/python-bigquery-dataframes/pull/427 +@pytest.mark.skipif( + sys.version_info >= (3, 12), + # See: https://github.com/python/cpython/issues/112282 + reason="setrecursionlimit has no effect on the Python C stack since Python 3.12.", +) def test_recursion_limit(scalars_df_index): scalars_df_index = scalars_df_index[["int64_too", "int64_col", "float64_col"]] for i in range(400): From 2697119521a6801501199abdef683764b501d3af Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 21 Mar 2024 22:14:22 +0000 Subject: [PATCH 09/10] avoid matplotlib error --- bigframes/operations/_matplotlib/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bigframes/operations/_matplotlib/core.py b/bigframes/operations/_matplotlib/core.py index 7cbeb3df4f..d0c1ee089d 100644 --- a/bigframes/operations/_matplotlib/core.py +++ b/bigframes/operations/_matplotlib/core.py @@ -15,8 +15,6 @@ import abc import typing -import matplotlib.pyplot as plt - class MPLPlot(abc.ABC): @abc.abstractmethod @@ -24,6 +22,11 @@ def generate(self): pass def draw(self) -> None: + # This import can fail with "Matplotlib failed to acquire the + # following lock file" so import here to reduce the chance of + # our parallel test suite from triggering this. + import matplotlib.pyplot as plt + plt.draw_if_interactive() @property From 297c1a73ba3a9435ace64ee69d84d58991035857 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 21 Mar 2024 23:40:02 +0000 Subject: [PATCH 10/10] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- bigframes/operations/_matplotlib/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bigframes/operations/_matplotlib/core.py b/bigframes/operations/_matplotlib/core.py index 1b1bda4184..663e7a789f 100644 --- a/bigframes/operations/_matplotlib/core.py +++ b/bigframes/operations/_matplotlib/core.py @@ -15,7 +15,6 @@ import abc import typing - DEFAULT_SAMPLING_N = 1000 DEFAULT_SAMPLING_STATE = 0