diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 22df4232ea294..669cd76bd72fa 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -366,6 +366,7 @@ jobs: pyspark-pandas-connect-part3 env: MODULES_TO_TEST: ${{ matrix.modules }} + PYTHON_TO_TEST: 'python3.9' HADOOP_PROFILE: ${{ inputs.hadoop }} HIVE_PROFILE: hive2.3 GITHUB_PREV_SHA: ${{ github.event.before }} @@ -447,7 +448,7 @@ jobs: export SKIP_PACKAGING=false echo "Python Packaging Tests Enabled!" fi - ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" + ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --python-executables "$PYTHON_TO_TEST" - name: Upload coverage to Codecov if: fromJSON(inputs.envs).PYSPARK_CODECOV == 'true' uses: codecov/codecov-action@v2 diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml new file mode 100644 index 0000000000000..94d64a825d876 --- /dev/null +++ b/.github/workflows/build_python.yml @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Python - PyPy3.8 (master)" + +on: + schedule: + - cron: '0 15 * * *' + +jobs: + run-build: + permissions: + packages: write + name: Run + uses: ./.github/workflows/build_and_test.yml + if: github.repository == 'apache/spark' + with: + java: 17 + branch: master + hadoop: hadoop3 + envs: >- + { + "PYTHON_TO_TEST": "pypy3" + } + jobs: >- + { + "pyspark": "true" + } diff --git a/dev/run-tests.py b/dev/run-tests.py index cf0db66fba1cc..3fe79af58d717 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -375,7 +375,7 @@ def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, inc run_scala_tests_sbt(test_modules, test_profiles) -def run_python_tests(test_modules, parallelism, with_coverage=False): +def run_python_tests(test_modules, test_pythons, parallelism, with_coverage=False): set_title_and_block("Running PySpark tests", "BLOCK_PYSPARK_UNIT_TESTS") if with_coverage: @@ -390,6 +390,7 @@ def run_python_tests(test_modules, parallelism, with_coverage=False): if test_modules != [modules.root]: command.append("--modules=%s" % ",".join(m.name for m in test_modules)) command.append("--parallelism=%i" % parallelism) + command.append("--python-executables=%s" % test_pythons) run_cmd(command) @@ -423,6 +424,12 @@ def parse_opts(): default=8, help="The number of suites to test in parallel (default %(default)d)", ) + parser.add_argument( + "--python-executables", + type=str, + default="python3.9", + help="A comma-separated list of Python executables to test against (default: %(default)s)", + ) parser.add_argument( "-m", "--modules", @@ -651,6 +658,7 @@ def main(): if modules_with_python_tests and not os.environ.get("SKIP_PYTHON"): run_python_tests( modules_with_python_tests, + opts.python_executables, opts.parallelism, with_coverage=os.environ.get("PYSPARK_CODECOV", "false") == "true", )