Skip to content

Commit

Permalink
Remove Python 3.8 GitHub Actions Runs (#32429)
Browse files Browse the repository at this point in the history
* [WIP] Remove Python 3.8 targets from Actions

* revert changes to 3.9 build.gradle

* More gradle cleanup

* Add posargs

* Update local-env-setup.sh

* Fix default env python version

* More places to update python version refs

* mypy/linting fixes

* yapf

* Update other locations

* update more files

* update standard_external_transforms.yaml

* update test case that fails on 3.8 -> 3.9
  • Loading branch information
jrmccluskey authored Sep 16, 2024
1 parent 6514136 commit bbd09d6
Show file tree
Hide file tree
Showing 119 changed files with 453 additions and 269 deletions.
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
{
"PythonTestProperties": {
"ALL_SUPPORTED_VERSIONS": ["3.8", "3.9", "3.10", "3.11"],
"LOWEST_SUPPORTED": ["3.8"],
"HIGHEST_SUPPORTED": ["3.11"],
"ESSENTIAL_VERSIONS": ["3.8", "3.11"],
"CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.8", "3.11"],
"ALL_SUPPORTED_VERSIONS": ["3.9", "3.10", "3.11", "3.12"],
"LOWEST_SUPPORTED": ["3.9"],
"HIGHEST_SUPPORTED": ["3.12"],
"ESSENTIAL_VERSIONS": ["3.9", "3.12"],
"CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.9", "3.12"],
"CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.11"],
"VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.8", "3.9", "3.10", "3.11" ],
"LOAD_TEST_PYTHON_VERSION": "3.8",
"CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.8",
"DEFAULT_INTERPRETER": "python3.8",
"VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.9", "3.10", "3.11", "3.12"],
"LOAD_TEST_PYTHON_VERSION": "3.9",
"CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.9",
"DEFAULT_INTERPRETER": "python3.9",
"TOX_ENV": ["Cloud", "Cython"]
},
"JavaTestProperties": {
Expand Down
2 changes: 1 addition & 1 deletion .github/actions/setup-environment-action/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ runs:
if: ${{ inputs.python-version != '' }}
uses: actions/setup-python@v4
with:
python-version: ${{ inputs.python-version == 'default' && '3.8' || inputs.python-version }}
python-version: ${{ inputs.python-version == 'default' && '3.9' || inputs.python-version }}
- name: Install Java
if: ${{ inputs.java-version != '' }}
uses: actions/setup-java@v3
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/beam_CloudML_Benchmarks_Dataflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@ jobs:
uses: ./.github/actions/setup-environment-action
with:
python-version: |
3.8
3.9
3.10
- name: Prepare test arguments
uses: ./.github/actions/test-arguments-action
with:
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-pytorch-imagenet-python-101-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet101-${{env.NOW_UTC}}.txt' \
- name: run Pytorch Imagenet Classification with Resnet 152
Expand All @@ -106,7 +106,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_2 }} --job_name=benchmark-tests-pytorch-imagenet-python-152-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152-${{env.NOW_UTC}}.txt' \
- name: run Pytorch Language Modeling using Hugging face bert-base-uncased model
Expand All @@ -117,7 +117,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_3 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-base-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_base_uncased-${{env.NOW_UTC}}.txt' \
- name: run Pytorch Langauge Modeling using Hugging Face bert-large-uncased model
Expand All @@ -128,7 +128,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_4 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-large-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_large_uncased-${{env.NOW_UTC}}.txt' \
- name: run Pytorch Imagenet Classification with Resnet 152 with Tesla T4 GPU
Expand All @@ -139,6 +139,6 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_5 }} --job_name=benchmark-tests-pytorch-imagenet-python-gpu-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152_gpu-${{env.NOW_UTC}}.txt'
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ jobs:
--info \
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \
- name: run CoGBK 2GB of 100B records with multiple keys
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -105,7 +105,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \
- name: run CoGBK reiterate 4 times 10kB values
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -114,7 +114,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \
- name: run CoGBK reiterate 4 times 2MB values
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -123,5 +123,5 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-cogbk-4-${{ steps.datetime.outputs.datetime }}'
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-cogbk-1-${{ steps.datetime.outputs.datetime }}' \
- name: run CoGBK 2GB of 100B records with multiple keys
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -104,7 +104,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-cogbk-2-${{ steps.datetime.outputs.datetime }}' \
- name: run CoGBK reiterate 4 times 10kB values
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -113,7 +113,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-cogbk-3-${{ steps.datetime.outputs.datetime }}' \
- name: run CoGBK reiterate 4 times 2MB values
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -122,5 +122,5 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_4 }} --job_name=load-tests-python-dataflow-streaming-cogbk-4-${{ steps.datetime.outputs.datetime }}'
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-combine-1-${{env.NOW_UTC}}' \
- name: run Combine Dataflow Batch Python Load Test 2 (fanout 4)
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -101,7 +101,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-combine-2-${{env.NOW_UTC}}' \
- name: run Combine Dataflow Batch Python Load Test 3 (fanout 8)
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -110,5 +110,5 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-combine-3-${{env.NOW_UTC}}'
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-combine-1-${{env.NOW_UTC}}' \
- name: run 2GB Fanout 4 test
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -101,7 +101,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-combine-4-${{env.NOW_UTC}}' \
- name: run 2GB Fanout 8 test
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -110,5 +110,5 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-combine-5-${{env.NOW_UTC}}'
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ jobs:
with:
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
arguments: |
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=PortableRunner \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-combine-1-${{env.NOW_UTC}}' \
Expand All @@ -121,7 +121,7 @@ jobs:
with:
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
arguments: |
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=PortableRunner \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-combine-4-${{env.NOW_UTC}}' \
Expand All @@ -130,7 +130,7 @@ jobs:
with:
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
arguments: |
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=PortableRunner \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-combine-5-${{env.NOW_UTC}}' \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ jobs:
with:
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
arguments: |
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=PortableRunner \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-combine-4-${{env.NOW_UTC}}' \
Expand All @@ -113,7 +113,7 @@ jobs:
with:
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
arguments: |
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=PortableRunner \
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-combine-5-${{env.NOW_UTC}}' \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,5 +87,5 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.microbenchmarks_test \
-Prunner=DirectRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_FnApiRunner_Microbenchmark_test_arguments_1 }}'
10 changes: 5 additions & 5 deletions .github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-gbk-1-${{env.NOW_UTC}}' \
- name: run 2GB of 100B records test
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -103,7 +103,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-2-${{env.NOW_UTC}}' \
- name: run 2GB of 100kB records test
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -112,7 +112,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-gbk-3-${{env.NOW_UTC}}' \
- name: run fanout 4 times with 2GB 10-byte records test
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -121,7 +121,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-gbk-4-${{env.NOW_UTC}}' \
- name: run fanout 8 times with 2GB 10-byte records total test
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -130,5 +130,5 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_5 }} --job_name=load-tests-python-dataflow-batch-gbk-5-${{env.NOW_UTC}}'
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-gbk-3-${{env.NOW_UTC}}' \
# // TODO(https://github.com/apache/beam/issues/20403). Skipping some cases because they are too slow:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-gbk-6-${{env.NOW_UTC}}' \
- name: run reiterate 4 times 2MB values test
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -100,5 +100,5 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \
-Prunner=DataflowRunner \
-PpythonVersion=3.8 \
-PpythonVersion=3.9 \
'-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-7-${{env.NOW_UTC}}'
Loading

0 comments on commit bbd09d6

Please sign in to comment.