From 907f66d4b5f9cde99d23be11834f961658d3c683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= <raulcumplido@gmail.com> Date: Wed, 27 Sep 2023 13:26:22 +0200 Subject: [PATCH 01/12] GH-37880: [CI][Python][Packaging] Add support for Python 3.12 --- ci/docker/python-wheel-windows-test-vs2017.dockerfile | 7 ++++--- ci/docker/python-wheel-windows-vs2017.dockerfile | 7 ++++--- ci/scripts/install_python.sh | 7 +++++-- dev/release/verify-release-candidate.sh | 6 +++--- dev/tasks/tasks.yml | 7 ++++--- docker-compose.yml | 2 +- python/setup.py | 1 + 7 files changed, 22 insertions(+), 15 deletions(-) diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile index 4aa5363ff7d62..4f5f24549892c 100644 --- a/ci/docker/python-wheel-windows-test-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile @@ -37,7 +37,8 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ ARG python=3.8 RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ - (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") -RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ + (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0-rc3" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") +RUN choco install -r -y --no-progress python --pre --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 531c4e678fafe..b68dea9aea2a9 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -81,9 +81,10 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ ARG python=3.8 RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ - (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") -RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ + (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0-rc3" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") +RUN choco install -r -y --no-progress python --pre --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools COPY python/requirements-wheel-build.txt arrow/python/ diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index 599c238c97803..7a3827fdfa3f7 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -27,8 +27,9 @@ platforms=([windows]=Windows declare -A versions versions=([3.8]=3.8.10 [3.9]=3.9.13 - [3.10]=3.10.8 - [3.11]=3.11.0) + [3.10]=3.10.13 + [3.11]=3.11.5 + [3.12]=3.12.0) if [ "$#" -ne 2 ]; then echo "Usage: $0 <platform> <version>" @@ -47,6 +48,8 @@ if [ $platform = "MacOSX" ]; then if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then fname="python-${full_version}-macos11.pkg" + else if [ "$version" = "3.12" ]; then + fname="python-3.12.0rc3-macos11.pkg" else fname="python-${full_version}-macosx10.9.pkg" fi diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 77b996766f78c..a1ef781b05c59 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -1020,7 +1020,7 @@ test_linux_wheels() { local arch="x86_64" fi - local python_versions="${TEST_PYTHON_VERSIONS:-3.8 3.9 3.10 3.11}" + local python_versions="${TEST_PYTHON_VERSIONS:-3.8 3.9 3.10 3.11 3.12}" local platform_tags="${TEST_WHEEL_PLATFORM_TAGS:-manylinux_2_17_${arch}.manylinux2014_${arch} manylinux_2_28_${arch}}" for python in ${python_versions}; do @@ -1042,11 +1042,11 @@ test_macos_wheels() { # apple silicon processor if [ "$(uname -m)" = "arm64" ]; then - local python_versions="3.8 3.9 3.10 3.11" + local python_versions="3.8 3.9 3.10 3.11 3.12" local platform_tags="macosx_11_0_arm64" local check_flight=OFF else - local python_versions="3.8 3.9 3.10 3.11" + local python_versions="3.8 3.9 3.10 3.11 3.12" local platform_tags="macosx_10_14_x86_64" fi diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 29e038a922412..cc228f155b8d6 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -475,7 +475,8 @@ tasks: {% for python_version, python_tag, abi_tag in [("3.8", "cp38", "cp38"), ("3.9", "cp39", "cp39"), ("3.10", "cp310", "cp310"), - ("3.11", "cp311", "cp311")] %} + ("3.11", "cp311", "cp311"), + ("3.12", "cp312", "cp312")] %} {############################## Wheel Linux ##################################} @@ -541,7 +542,7 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_11_0_arm64.whl -{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311")] %} +{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311"), ("3.12", "cp312")] %} wheel-macos-big-sur-{{ python_tag }}-arm64: ci: github template: python-wheels/github.osx.arm64.yml @@ -1255,7 +1256,7 @@ tasks: flags: "-e ARROW_CSV=ON -e ARROW_PARQUET=ON" image: ubuntu-cpp-minimal -{% for python_version in ["3.8", "3.9", "3.10", "3.11"] %} +{% for python_version in ["3.8", "3.9", "3.10", "3.11", "3.12"] %} test-conda-python-{{ python_version }}: ci: github template: docker-tests/github.linux.yml diff --git a/docker-compose.yml b/docker-compose.yml index 62e5aee0a841c..11390d42be668 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1033,7 +1033,7 @@ services: args: arch: ${ARCH} arch_short: ${ARCH_SHORT} - base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2022-10-25-fbea779 + base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2023-09-24-36b93e4 vcpkg: ${VCPKG} python: ${PYTHON} manylinux: 2014 diff --git a/python/setup.py b/python/setup.py index 062aac307b1e4..0cef947821696 100755 --- a/python/setup.py +++ b/python/setup.py @@ -505,6 +505,7 @@ def has_ext_modules(foo): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], license='Apache License, Version 2.0', maintainer='Apache Arrow Developers', From 5cee4a884084a9d2c7f90fc241b0ce934c89b896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= <raulcumplido@gmail.com> Date: Wed, 27 Sep 2023 14:58:47 +0200 Subject: [PATCH 02/12] Fix elif on bash for installing Python on macOS --- ci/scripts/install_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index 7a3827fdfa3f7..d513d54aeb239 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -48,7 +48,7 @@ if [ $platform = "MacOSX" ]; then if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then fname="python-${full_version}-macos11.pkg" - else if [ "$version" = "3.12" ]; then + elif [ "$version" = "3.12" ]; then fname="python-3.12.0rc3-macos11.pkg" else fname="python-${full_version}-macosx10.9.pkg" From 7b4f2678d5f6794324f3f45243f75840a944fe38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= <raulcumplido@gmail.com> Date: Wed, 27 Sep 2023 15:18:18 +0200 Subject: [PATCH 03/12] Test wheels with 3.12.0rc3 --- dev/tasks/python-wheels/github.linux.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 41b18684cee10..2572177d39902 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -53,10 +53,21 @@ jobs: # TODO(kszucs): auditwheel show - name: Test wheel shell: bash + if: | + '{{ python_version }}' != '3.12' run: | archery docker run python-wheel-manylinux-test-imports archery docker run python-wheel-manylinux-test-unittests + - name: Test wheel fo3 Python 3.12 RC + shell: bash + if: | + '{{ python_version }}' == '3.12' + env: + PYTHON: "3.12.0rc3" + run: | + archery docker run python-wheel-manylinux-test-imports + archery docker run python-wheel-manylinux-test-unittests - name: Test wheel on AlmaLinux 8 shell: bash if: | From 54d5d340852d0c59d05d65f812ddb6a032a0961b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= <raulcumplido@gmail.com> Date: Wed, 27 Sep 2023 15:20:13 +0200 Subject: [PATCH 04/12] Update Pypa image for manylinux 2_28 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 11390d42be668..64c03cfc29a99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1056,7 +1056,7 @@ services: args: arch: ${ARCH} arch_short: ${ARCH_SHORT} - base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2023-01-14-103cb93 + base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2023-09-24-36b93e4 vcpkg: ${VCPKG} python: ${PYTHON} manylinux: 2_28 From 349f3622e575e39a91ffad93eeb40ed8d1bd19c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= <raulcumplido@gmail.com> Date: Wed, 27 Sep 2023 15:37:45 +0200 Subject: [PATCH 05/12] pip install upgrade setuptools too --- dev/tasks/python-wheels/github.osx.amd64.yml | 2 +- dev/tasks/python-wheels/github.osx.arm64.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml index 526412f84214b..e31a3278c8864 100644 --- a/dev/tasks/python-wheels/github.osx.amd64.yml +++ b/dev/tasks/python-wheels/github.osx.amd64.yml @@ -115,7 +115,7 @@ jobs: run: | $PYTHON -m venv test-amd64-env source test-amd64-env/bin/activate - pip install --upgrade pip wheel + pip install --upgrade setuptools pip wheel arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index 2c796d162debf..4cd492f8efd64 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -99,12 +99,12 @@ jobs: run: | $PYTHON -m venv test-arm64-env source test-arm64-env/bin/activate - pip install --upgrade pip wheel + pip install --upgrade setuptools pip wheel # libffi has to be installed on the m1 runner which causes issues with # the cffi wheel. We build cffi with the flags pointing to the correct libffi location. LDFLAGS=-L$(brew --prefix libffi)/lib CFLAGS=-I$(brew --prefix libffi)/include \ pip install cffi --no-binary :all: - pip install -r arrow/python/requirements-wheel-test.txt + pip install -r arrow/python/requirements-wheel-test.txt PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow From 3e3046ccdea515992e42c3977542629862d999ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= <raulcumplido@gmail.com> Date: Wed, 27 Sep 2023 16:49:16 +0200 Subject: [PATCH 06/12] Try with numpy 1.26 for Python 3.12 --- python/requirements-wheel-test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index 9de0acb754079..957b7afcdb72f 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -15,7 +15,8 @@ numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" an numpy==1.23.4; platform_system == "Darwin" and platform_machine == "arm64" and python_version >= "3.11" numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9" numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.11" +numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version == "3.11" +numpy==1.26.0; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.12" numpy==1.19.5; platform_system == "Windows" and python_version < "3.9" numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9" and python_version < "3.11" numpy==1.23.4; platform_system == "Windows" and python_version >= "3.11" From 125ac3496aaa12f41f9ac09bd8c3f61ee3a0ac14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= <raulcumplido@gmail.com> Date: Wed, 27 Sep 2023 17:25:52 +0200 Subject: [PATCH 07/12] Require latest numpy for Python 3.12 --- python/requirements-wheel-test.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index 957b7afcdb72f..ab7e93b273c70 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -7,19 +7,23 @@ pytz tzdata; sys_platform == 'win32' numpy==1.21.3; platform_system == "Linux" and platform_machine == "aarch64" and python_version < "3.11" -numpy==1.23.4; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.11" +numpy==1.23.4; platform_system == "Linux" and platform_machine == "aarch64" and python_version == "3.11" +numpy==1.26.0; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.12" numpy==1.19.5; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.9" numpy==1.21.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" +numpy==1.23.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version == "3.11" +numpy==1.26.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" and python_version < "3.11" -numpy==1.23.4; platform_system == "Darwin" and platform_machine == "arm64" and python_version >= "3.11" +numpy==1.23.4; platform_system == "Darwin" and platform_machine == "arm64" and python_version == "3.11" +numpy==1.26.0; platform_system == "Darwin" and platform_machine == "arm64" and python_version >= "3.12" numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9" numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" and python_version < "3.11" numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version == "3.11" numpy==1.26.0; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.12" numpy==1.19.5; platform_system == "Windows" and python_version < "3.9" numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Windows" and python_version >= "3.11" +numpy==1.23.4; platform_system == "Windows" and python_version == "3.11" +numpy==1.26.0; platform_system == "Windows" and python_version >= "3.12" pandas<1.1.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.8" pandas; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.8" From dbd90ccb64cc7526c8543b780c08d3e7465df64e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= <raulcumplido@gmail.com> Date: Wed, 27 Sep 2023 18:27:32 +0200 Subject: [PATCH 08/12] Use latest GCS to see if it fixes CI --- ci/scripts/install_gcs_testbench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh index 26fca3c946f8a..a968569389b74 100755 --- a/ci/scripts/install_gcs_testbench.sh +++ b/ci/scripts/install_gcs_testbench.sh @@ -36,7 +36,7 @@ esac version=$1 if [[ "${version}" -eq "default" ]]; then - version="v0.36.0" + version="v0.37.0" fi ${PYTHON:-python3} -m pip install \ From e92c96bfac08436bdb2dd6830eaed7c949e49539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= <raulcumplido@gmail.com> Date: Wed, 27 Sep 2023 19:59:11 +0200 Subject: [PATCH 09/12] Temporarily disable install_gcs_testbench --- ci/docker/python-wheel-manylinux-test.dockerfile | 3 ++- ci/scripts/python_wheel_windows_test.bat | 3 ++- dev/tasks/python-wheels/github.osx.amd64.yml | 3 ++- dev/tasks/python-wheels/github.osx.arm64.yml | 5 +++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ci/docker/python-wheel-manylinux-test.dockerfile b/ci/docker/python-wheel-manylinux-test.dockerfile index cdd0ae3ced756..2c64344430c19 100644 --- a/ci/docker/python-wheel-manylinux-test.dockerfile +++ b/ci/docker/python-wheel-manylinux-test.dockerfile @@ -27,4 +27,5 @@ COPY python/requirements-wheel-test.txt /arrow/python/ RUN pip install -r /arrow/python/requirements-wheel-test.txt COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ -RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default +# TODO: Temporarily disable installing GCS testbench +# RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat index c73b0cfd1b9bd..9355a164009f6 100755 --- a/ci/scripts/python_wheel_windows_test.bat +++ b/ci/scripts/python_wheel_windows_test.bat @@ -40,7 +40,8 @@ set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1 @REM Install GCS testbench -call "C:\arrow\ci\scripts\install_gcs_testbench.bat" +@REM TODO: Temporarily disable installing GCS testbench +@REM call "C:\arrow\ci\scripts\install_gcs_testbench.bat" @REM Install the built wheels python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml index e31a3278c8864..02e847eca365b 100644 --- a/dev/tasks/python-wheels/github.osx.amd64.yml +++ b/dev/tasks/python-wheels/github.osx.amd64.yml @@ -117,7 +117,8 @@ jobs: source test-amd64-env/bin/activate pip install --upgrade setuptools pip wheel arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt - PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default + # TODO: Temporarily disable installing GCS testbench + # PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index 4cd492f8efd64..55e59a15fe3e6 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -105,8 +105,9 @@ jobs: LDFLAGS=-L$(brew --prefix libffi)/lib CFLAGS=-I$(brew --prefix libffi)/include \ pip install cffi --no-binary :all: pip install -r arrow/python/requirements-wheel-test.txt - PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default - arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow + # TODO: Temporarily disable installing GCS testbench + # PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default + arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow - name: Upload artifacts shell: bash From 53ea492e81cf27cae43a4097208733cf56315e2b Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" <terence@honles.com> Date: Mon, 2 Oct 2023 21:17:39 +0200 Subject: [PATCH 10/12] update for release version --- ci/docker/python-wheel-windows-test-vs2017.dockerfile | 2 +- ci/docker/python-wheel-windows-vs2017.dockerfile | 2 +- ci/scripts/install_python.sh | 4 +--- dev/tasks/python-wheels/github.linux.yml | 11 ----------- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile index 4f5f24549892c..4ed7d2a9c316b 100644 --- a/ci/docker/python-wheel-windows-test-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile @@ -39,6 +39,6 @@ RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\P (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ - (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0-rc3" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") + (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") RUN choco install -r -y --no-progress python --pre --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index b68dea9aea2a9..8441a2854f8f4 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -83,7 +83,7 @@ RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\P (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ - (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0-rc3" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") + (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") RUN choco install -r -y --no-progress python --pre --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index d513d54aeb239..cd670b9371991 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -46,10 +46,8 @@ full_version=${versions[$2]} if [ $platform = "MacOSX" ]; then echo "Downloading Python installer..." - if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then + if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ] || [ "$version" = "3.12" ]; then fname="python-${full_version}-macos11.pkg" - elif [ "$version" = "3.12" ]; then - fname="python-3.12.0rc3-macos11.pkg" else fname="python-${full_version}-macosx10.9.pkg" fi diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 2572177d39902..41b18684cee10 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -53,21 +53,10 @@ jobs: # TODO(kszucs): auditwheel show - name: Test wheel shell: bash - if: | - '{{ python_version }}' != '3.12' run: | archery docker run python-wheel-manylinux-test-imports archery docker run python-wheel-manylinux-test-unittests - - name: Test wheel fo3 Python 3.12 RC - shell: bash - if: | - '{{ python_version }}' == '3.12' - env: - PYTHON: "3.12.0rc3" - run: | - archery docker run python-wheel-manylinux-test-imports - archery docker run python-wheel-manylinux-test-unittests - name: Test wheel on AlmaLinux 8 shell: bash if: | From d61ef4897bfa432e756825b9f0500b95de8b8ec3 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" <terence@honles.com> Date: Mon, 2 Oct 2023 21:27:06 +0200 Subject: [PATCH 11/12] remove --pre flag to choco --- ci/docker/python-wheel-windows-test-vs2017.dockerfile | 2 +- ci/docker/python-wheel-windows-vs2017.dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile index 4ed7d2a9c316b..e842ede18454b 100644 --- a/ci/docker/python-wheel-windows-test-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile @@ -40,5 +40,5 @@ RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\P (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") -RUN choco install -r -y --no-progress python --pre --version=%PYTHON_VERSION% +RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 8441a2854f8f4..b6a8125a5c4fc 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -84,7 +84,7 @@ RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\P (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") -RUN choco install -r -y --no-progress python --pre --version=%PYTHON_VERSION% +RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools COPY python/requirements-wheel-build.txt arrow/python/ From 58a41e030b82fa50a0a092fa7d897d1919f62cd8 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" <terence@honles.com> Date: Mon, 2 Oct 2023 21:34:47 +0200 Subject: [PATCH 12/12] use same 3.10 version --- ci/docker/python-wheel-windows-test-vs2017.dockerfile | 2 +- ci/docker/python-wheel-windows-vs2017.dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile index e842ede18454b..431325662a581 100644 --- a/ci/docker/python-wheel-windows-test-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile @@ -37,7 +37,7 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ ARG python=3.8 RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.13" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index b6a8125a5c4fc..f493b24d8c4eb 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -81,7 +81,7 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ ARG python=3.8 RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.13" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%