Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert PR #32960 #33088

Merged
merged 2 commits into from
Nov 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/trigger_files/beam_PostCommit_Python.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run.",
"modification": 3
"modification": 4
}

26 changes: 1 addition & 25 deletions sdks/python/container/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -103,33 +103,9 @@ RUN if [ "$pull_licenses" = "true" ] ; then \
python /tmp/license_scripts/pull_licenses_py.py ; \
fi

FROM beam as base
FROM beam
ARG pull_licenses
COPY --from=third_party_licenses /opt/apache/beam/third_party_licenses /opt/apache/beam/third_party_licenses
RUN if [ "$pull_licenses" != "true" ] ; then \
rm -rf /opt/apache/beam/third_party_licenses ; \
fi

ARG TARGETARCH
FROM gcr.io/distroless/python3-debian12:latest-${TARGETARCH} as distroless
ARG py_version

# Contains header files needed by the Python interpreter.
COPY --from=base /usr/local/include /usr/local/include

# Contains the Python interpreter executables.
COPY --from=base /usr/local/bin /usr/local/bin

# Contains the Python library dependencies.
COPY --from=base /usr/local/lib /usr/local/lib

# Python standard library modules.
COPY --from=base /usr/lib/python${py_version} /usr/lib/python${py_version}

# Contains the boot entrypoint and related files such as licenses.
COPY --from=base /opt /opt

ENV PATH "$PATH:/usr/local/bin"

# Despite the ENTRYPOINT set above, need to reset since deriving the layer derives from a different image.
ENTRYPOINT ["/opt/apache/beam/boot"]
9 changes: 1 addition & 8 deletions sdks/python/container/common.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -71,16 +71,10 @@ def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy)
}

def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers")
def baseBuildTarget = 'base'
def buildTarget = project.findProperty('container-build-target') ?: 'base'
var imageName = project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk"
if (buildTarget != baseBuildTarget) {
imageName += "_${buildTarget}"
}

docker {
name containerImageName(
name: imageName,
name: project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk",
root: project.rootProject.hasProperty(["docker-repository-root"]) ?
project.rootProject["docker-repository-root"] :
project.docker_image_default_repo_root,
Expand All @@ -96,7 +90,6 @@ docker {
platform(*project.containerPlatforms())
load project.useBuildx() && !pushContainers
push pushContainers
target buildTarget
}

dockerPrepare.dependsOn copyLauncherDependencies
Expand Down
6 changes: 0 additions & 6 deletions sdks/python/test-suites/dataflow/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,6 @@ task validatesContainerTests {
}
}

task validatesDistrolessContainerTests {
getVersionsAsList('distroless_python_versions').each {
dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:validatesDistrolessContainer")
}
}

task examplesPostCommit {
getVersionsAsList('dataflow_examples_postcommit_py_versions').each {
dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:examples")
Expand Down
45 changes: 0 additions & 45 deletions sdks/python/test-suites/dataflow/common.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -380,51 +380,6 @@ task validatesContainer() {
}
}

/**
* Validates the distroless (https://github.com/GoogleContainerTools/distroless) variant of the Python SDK container
* image (sdks/python/container/Dockerfile).
* To test a single version of Python:
* ./gradlew :sdks:python:test-suites:dataflow:py311:validatesDistrolessContainer
* See https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-VirtualEnvironmentSetup
* for more information on setting up different Python versions.
*/
task validatesDistrolessContainer() {
def pyversion = "${project.ext.pythonVersion.replace('.', '')}"
def buildTarget = 'distroless'
def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
def tag = java.time.Instant.now().getEpochSecond()
def imageURL = "${repository}/beam_python${project.ext.pythonVersion}_sdk_${buildTarget}:${tag}"
project.rootProject.ext['docker-repository-root'] = repository
project.rootProject.ext['container-build-target'] = buildTarget
project.rootProject.ext['docker-tag'] = tag
if (project.rootProject.hasProperty('dry-run')) {
println "Running in dry run mode: imageURL: ${imageURL}, pyversion: ${pyversion}, buildTarget: ${buildTarget}, repository: ${repository}, tag: ${tag}, envdir: ${envdir}"
return
}
dependsOn 'initializeForDataflowJob'
dependsOn ":sdks:python:container:py${pyversion}:docker"
dependsOn ":sdks:python:container:py${pyversion}:dockerPush"
def testTarget = "apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it"
def argMap = [
"output": "gs://temp-storage-for-end-to-end-tests/py-it-cloud/output",
"project": "apache-beam-testing",
"region": "us-central1",
"runner": "TestDataflowRunner",
"sdk_container_image": "${imageURL}",
"sdk_location": "container",
"staging_location": "gs://temp-storage-for-end-to-end-tests/staging-it",
"temp_location": "gs://temp-storage-for-end-to-end-tests/temp-it",
]
def cmdArgs = mapToArgString(argMap)
doLast {
exec {
workingDir = "${rootDir}/sdks/python"
executable 'sh'
args '-c', ". ${envdir}/bin/activate && pytest ${testTarget} --test-pipeline-options=\"${cmdArgs}\""
}
}
}

task validatesContainerARM() {
def pyversion = "${project.ext.pythonVersion.replace('.', '')}"
dependsOn 'initializeForDataflowJob'
Expand Down
3 changes: 0 additions & 3 deletions sdks/python/test-suites/gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,3 @@ prism_examples_postcommit_py_versions=3.9,3.12

# cross language postcommit python test suites
cross_language_validates_py_versions=3.9,3.12

# Python versions to support distroless variants
distroless_python_versions=3.9,3.10,3.11,3.12
Loading