From c4143315571e6481a0b3e976d6feeecffec904d4 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Tue, 12 Nov 2024 10:10:33 -0500 Subject: [PATCH 1/2] Revert "Distroless python sdk (#32960)" This reverts commit 81f35ab62298a2ec9fadeded82461b363b6401db. --- sdks/python/container/Dockerfile | 26 +---------- sdks/python/container/common.gradle | 9 +--- sdks/python/test-suites/dataflow/build.gradle | 6 --- .../python/test-suites/dataflow/common.gradle | 45 ------------------- sdks/python/test-suites/gradle.properties | 3 -- 5 files changed, 2 insertions(+), 87 deletions(-) diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile index f3d22a4b5bc6..7bea6229668f 100644 --- a/sdks/python/container/Dockerfile +++ b/sdks/python/container/Dockerfile @@ -103,33 +103,9 @@ RUN if [ "$pull_licenses" = "true" ] ; then \ python /tmp/license_scripts/pull_licenses_py.py ; \ fi -FROM beam as base +FROM beam ARG pull_licenses COPY --from=third_party_licenses /opt/apache/beam/third_party_licenses /opt/apache/beam/third_party_licenses RUN if [ "$pull_licenses" != "true" ] ; then \ rm -rf /opt/apache/beam/third_party_licenses ; \ fi - -ARG TARGETARCH -FROM gcr.io/distroless/python3-debian12:latest-${TARGETARCH} as distroless -ARG py_version - -# Contains header files needed by the Python interpreter. -COPY --from=base /usr/local/include /usr/local/include - -# Contains the Python interpreter executables. -COPY --from=base /usr/local/bin /usr/local/bin - -# Contains the Python library dependencies. -COPY --from=base /usr/local/lib /usr/local/lib - -# Python standard library modules. -COPY --from=base /usr/lib/python${py_version} /usr/lib/python${py_version} - -# Contains the boot entrypoint and related files such as licenses. -COPY --from=base /opt /opt - -ENV PATH "$PATH:/usr/local/bin" - -# Despite the ENTRYPOINT set above, need to reset since deriving the layer derives from a different image. -ENTRYPOINT ["/opt/apache/beam/boot"] diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 885662362894..0175778a6301 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -71,16 +71,10 @@ def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy) } def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") -def baseBuildTarget = 'base' -def buildTarget = project.findProperty('container-build-target') ?: 'base' -var imageName = project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk" -if (buildTarget != baseBuildTarget) { - imageName += "_${buildTarget}" -} docker { name containerImageName( - name: imageName, + name: project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk", root: project.rootProject.hasProperty(["docker-repository-root"]) ? project.rootProject["docker-repository-root"] : project.docker_image_default_repo_root, @@ -96,7 +90,6 @@ docker { platform(*project.containerPlatforms()) load project.useBuildx() && !pushContainers push pushContainers - target buildTarget } dockerPrepare.dependsOn copyLauncherDependencies diff --git a/sdks/python/test-suites/dataflow/build.gradle b/sdks/python/test-suites/dataflow/build.gradle index 4500b395b0a6..04a79683fd36 100644 --- a/sdks/python/test-suites/dataflow/build.gradle +++ b/sdks/python/test-suites/dataflow/build.gradle @@ -60,12 +60,6 @@ task validatesContainerTests { } } -task validatesDistrolessContainerTests { - getVersionsAsList('distroless_python_versions').each { - dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:validatesDistrolessContainer") - } -} - task examplesPostCommit { getVersionsAsList('dataflow_examples_postcommit_py_versions').each { dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:examples") diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index cd0db4a62f77..71d44652bc7e 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -380,51 +380,6 @@ task validatesContainer() { } } -/** - * Validates the distroless (https://github.com/GoogleContainerTools/distroless) variant of the Python SDK container - * image (sdks/python/container/Dockerfile). - * To test a single version of Python: - * ./gradlew :sdks:python:test-suites:dataflow:py311:validatesDistrolessContainer - * See https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-VirtualEnvironmentSetup - * for more information on setting up different Python versions. - */ -task validatesDistrolessContainer() { - def pyversion = "${project.ext.pythonVersion.replace('.', '')}" - def buildTarget = 'distroless' - def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}" - def tag = java.time.Instant.now().getEpochSecond() - def imageURL = "${repository}/beam_python${project.ext.pythonVersion}_sdk_${buildTarget}:${tag}" - project.rootProject.ext['docker-repository-root'] = repository - project.rootProject.ext['container-build-target'] = buildTarget - project.rootProject.ext['docker-tag'] = tag - if (project.rootProject.hasProperty('dry-run')) { - println "Running in dry run mode: imageURL: ${imageURL}, pyversion: ${pyversion}, buildTarget: ${buildTarget}, repository: ${repository}, tag: ${tag}, envdir: ${envdir}" - return - } - dependsOn 'initializeForDataflowJob' - dependsOn ":sdks:python:container:py${pyversion}:docker" - dependsOn ":sdks:python:container:py${pyversion}:dockerPush" - def testTarget = "apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it" - def argMap = [ - "output": "gs://temp-storage-for-end-to-end-tests/py-it-cloud/output", - "project": "apache-beam-testing", - "region": "us-central1", - "runner": "TestDataflowRunner", - "sdk_container_image": "${imageURL}", - "sdk_location": "container", - "staging_location": "gs://temp-storage-for-end-to-end-tests/staging-it", - "temp_location": "gs://temp-storage-for-end-to-end-tests/temp-it", - ] - def cmdArgs = mapToArgString(argMap) - doLast { - exec { - workingDir = "${rootDir}/sdks/python" - executable 'sh' - args '-c', ". ${envdir}/bin/activate && pytest ${testTarget} --test-pipeline-options=\"${cmdArgs}\"" - } - } -} - task validatesContainerARM() { def pyversion = "${project.ext.pythonVersion.replace('.', '')}" dependsOn 'initializeForDataflowJob' diff --git a/sdks/python/test-suites/gradle.properties b/sdks/python/test-suites/gradle.properties index 08266c4b0dd5..d027cd3144d3 100644 --- a/sdks/python/test-suites/gradle.properties +++ b/sdks/python/test-suites/gradle.properties @@ -54,6 +54,3 @@ prism_examples_postcommit_py_versions=3.9,3.12 # cross language postcommit python test suites cross_language_validates_py_versions=3.9,3.12 - -# Python versions to support distroless variants -distroless_python_versions=3.9,3.10,3.11,3.12 From 93a3dea312a38ed37872562fe91b997a3d87990b Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Tue, 12 Nov 2024 10:11:33 -0500 Subject: [PATCH 2/2] Trigger post commit python test. --- .github/trigger_files/beam_PostCommit_Python.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json index 1eb60f6e4959..9e1d1e1b80dd 100644 --- a/.github/trigger_files/beam_PostCommit_Python.json +++ b/.github/trigger_files/beam_PostCommit_Python.json @@ -1,5 +1,5 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 3 + "modification": 4 }