From bc06581e11085d02a52356681226ee8caac5a825 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Tue, 24 Oct 2023 11:05:32 -0400 Subject: [PATCH] Enable run test on Java 21 (#28969) * upgrade gradle 8,3 -> 8.4 * Use temurin 21 base image as its available now * Setup Java21 JPMS postcommits * Setup Java21 Example PostCommit on DataflowV2 * Fix kotlin test --- .github/workflows/README.md | 1 + ...tCommit_Java_Examples_Dataflow_V2_Java.yml | 7 +- ...eam_PostCommit_Java_Jpms_Direct_Java21.yml | 97 +++++++++ .../beam/gradle/BeamModulePlugin.groovy | 190 ++++++++---------- examples/java/build.gradle | 4 + examples/kotlin/build.gradle | 9 +- gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 14 +- sdks/java/container/agent/build.gradle | 9 +- sdks/java/container/java21/build.gradle | 2 - sdks/java/testing/jpms-tests/build.gradle | 4 +- 11 files changed, 209 insertions(+), 130 deletions(-) create mode 100644 .github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5777a84b2a3f..18333212487c 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -283,6 +283,7 @@ Please note that jobs with matrix need to have matrix element in the comment. Ex | [ PostCommit Java Jpms Dataflow Java17 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml) | N/A |`Run Jpms Dataflow Java 17 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml?query=event%3Aschedule) | | [ PostCommit Java Jpms Direct Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml) | N/A |`Run Jpms Direct Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml?query=event%3Aschedule) | | [ PostCommit Java Jpms Direct Java17 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml) | N/A |`Run Jpms Direct Java 17 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml?query=event%3Aschedule) | +| [ PostCommit Java Jpms Direct Java21 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml) | N/A |`Run Jpms Direct Java21 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml?query=event%3Aschedule) | | [ PostCommit Java Jpms Flink Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml) | N/A |`Run Jpms Flink Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml?query=event%3Aschedule) | | [ PostCommit Java Jpms Spark Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml) | N/A |`Run Jpms Spark Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml?query=event%3Aschedule) | | [ PostCommit Java Nexmark Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml) | N/A |`Run Dataflow Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml?query=event%3Aschedule) | diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml index 1b0065901046..fe8de2d2abe0 100644 --- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml @@ -59,7 +59,7 @@ jobs: job_name: [beam_PostCommit_Java_Examples_Dataflow_V2_Java] job_phrase_1: [Run Java ] job_phrase_2: [Examples on Dataflow Runner V2] - java_version: ['11', '17'] + java_version: ['11', '17', '21'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || @@ -76,7 +76,10 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: ${{ matrix.java_version }} + distribution: 'temurin' + java-version: | + ${{ matrix.java_version }} + 8 - name: run PostCommit Java Examples Dataflow V2 Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml new file mode 100644 index 000000000000..fb93a1c72953 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Jpms Direct Java21 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Jpms_Direct_Java21: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Jpms Direct Java 21 PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_Jpms_Direct_Java21"] + job_phrase: ["Run Jpms Direct Java 21 PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: | + 21 + 8 + - name: run PostCommit Java Jpms Direct Java21 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:jpms-tests:directRunnerIntegrationTest + arguments: + -PskipCheckerFramework + -PtestJavaVersion=21 + -Pjava21Home=$JAVA_HOME_21_X64 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index b32a41f80279..3d7dd5d5aaf9 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -449,6 +449,21 @@ class BeamModulePlugin implements Plugin { return 'beam' + p.path.replace(':', '-') } + static def getSupportedJavaVersion() { + if (JavaVersion.current() == JavaVersion.VERSION_1_8) { + return 'java8' + } else if (JavaVersion.current() == JavaVersion.VERSION_11) { + return 'java11' + } else if (JavaVersion.current() == JavaVersion.VERSION_17) { + return 'java17' + } else if (JavaVersion.current() == JavaVersion.VERSION_21) { + return 'java21' + } else { + String exceptionMessage = "Your Java version is unsupported. You need Java version of 8, 11, 17 or 21 to get started, but your Java version is: " + JavaVersion.current(); + throw new GradleException(exceptionMessage) + } + } + /* * Set compile args for compiling and running in different java version by modifying the compiler args in place. * @@ -922,51 +937,63 @@ class BeamModulePlugin implements Plugin { + suffix) } - project.ext.setJava17Options = { CompileOptions options -> - def java17Home = project.findProperty("java17Home") - options.fork = true - options.forkOptions.javaHome = java17Home as File - options.compilerArgs += ['-Xlint:-path'] - // Error prone requires some packages to be exported/opened for Java 17 - // Disabling checks since this property is only used for Jenkins tests - // https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support - options.errorprone.errorproneArgs.add("-XepDisableAllChecks") - // The -J prefix is needed to workaround https://github.com/gradle/gradle/issues/22747 - options.forkOptions.jvmArgs += [ - "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", - "-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", - "-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" - ] - } - - project.ext.setJava21Options = { CompileOptions options -> - def java21Home = project.findProperty("java21Home") - options.fork = true - options.forkOptions.javaHome = java21Home as File - options.compilerArgs += ['-Xlint:-path'] - // Error prone requires some packages to be exported/opened for Java 17+ - // Disabling checks since this property is only used for Jenkins tests - // https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support - options.errorprone.errorproneArgs.add("-XepDisableAllChecks") - options.forkOptions.jvmArgs += [ - "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", - "-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", - "-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" - ] + // set compiler options for java version overrides to compile with a different java version + project.ext.setJavaVerOptions = { CompileOptions options, String ver -> + if (ver == '11') { + def java11Home = project.findProperty("java11Home") + options.fork = true + options.forkOptions.javaHome = java11Home as File + options.compilerArgs += ['-Xlint:-path'] + } else if (ver == '17') { + def java17Home = project.findProperty("java17Home") + options.fork = true + options.forkOptions.javaHome = java17Home as File + options.compilerArgs += ['-Xlint:-path'] + // Error prone requires some packages to be exported/opened for Java 17 + // Disabling checks since this property is only used for tests + // https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support + options.errorprone.errorproneArgs.add("-XepDisableAllChecks") + // The -J prefix is needed to workaround https://github.com/gradle/gradle/issues/22747 + options.forkOptions.jvmArgs += [ + "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", + "-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", + "-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" + ] + } else if (ver == '21') { + def java21Home = project.findProperty("java21Home") + options.fork = true + options.forkOptions.javaHome = java21Home as File + options.compilerArgs += ['-Xlint:-path'] + // Error prone requires some packages to be exported/opened for Java 17+ + // Disabling checks since this property is only used for tests + options.errorprone.errorproneArgs.add("-XepDisableAllChecks") + options.forkOptions.jvmArgs += [ + "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", + "-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", + "-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" + ] + // TODO(https://github.com/apache/beam/issues/28963) + // upgrade checkerFramework to enable it in Java 21 + project.checkerFramework { + skipCheckerFramework = true + } + } else { + throw new GradleException("Unknown Java Version ${ver} for setting additional java options") + } } project.ext.repositories = { @@ -1515,37 +1542,20 @@ class BeamModulePlugin implements Plugin { options.errorprone.errorproneArgs.add("-Xep:Slf4jLoggerShouldBeNonStatic:OFF") } - if (project.findProperty('testJavaVersion') == "11") { - def java11Home = project.findProperty("java11Home") - project.tasks.compileTestJava { - options.fork = true - options.forkOptions.javaHome = java11Home as File - options.compilerArgs += ['-Xlint:-path'] - setCompileAndRuntimeJavaVersion(options.compilerArgs, '11') - } - project.tasks.withType(Test).configureEach { - useJUnit() - executable = "${java11Home}/bin/java" - } - } else if (project.findProperty('testJavaVersion') == "17") { - def java17Home = project.findProperty("java17Home") - project.tasks.compileTestJava { - setCompileAndRuntimeJavaVersion(options.compilerArgs, '17') - project.ext.setJava17Options(options) - } - project.tasks.withType(Test).configureEach { - useJUnit() - executable = "${java17Home}/bin/java" - } - } else if (project.findProperty('testJavaVersion') == "21") { - def java21Home = project.findProperty("java21Home") + // if specified test java version, modify the compile and runtime versions accordingly + if (project.hasProperty('testJavaVersion')) { + String ver = project.getProperty('testJavaVersion') + def testJavaHome = project.getProperty("java${ver}Home") + + // redirect java compiler to specified version for compileTestJava only project.tasks.compileTestJava { - setCompileAndRuntimeJavaVersion(options.compilerArgs, '21') - project.ext.setJava17Options(options) + setCompileAndRuntimeJavaVersion(options.compilerArgs, ver) + project.ext.setJavaVerOptions(options, ver) } + // redirect java runtime to specified version for running tests project.tasks.withType(Test).configureEach { useJUnit() - executable = "${java21Home}/bin/java" + executable = "${testJavaHome}/bin/java" } } @@ -2579,17 +2589,7 @@ class BeamModulePlugin implements Plugin { "java_expansion_service_allowlist_file": javaClassLookupAllowlistFile, ] def usesDataflowRunner = config.pythonPipelineOptions.contains("--runner=TestDataflowRunner") || config.pythonPipelineOptions.contains("--runner=DataflowRunner") - def javaContainerSuffix - if (JavaVersion.current() == JavaVersion.VERSION_1_8) { - javaContainerSuffix = 'java8' - } else if (JavaVersion.current() == JavaVersion.VERSION_11) { - javaContainerSuffix = 'java11' - } else if (JavaVersion.current() == JavaVersion.VERSION_17) { - javaContainerSuffix = 'java17' - } else { - String exceptionMessage = "Your Java version is unsupported. You need Java version of 8 or 11 or 17 to get started, but your Java version is: " + JavaVersion.current(); - throw new GradleException(exceptionMessage) - } + def javaContainerSuffix = getSupportedJavaVersion() // 1. Builds the chosen expansion service jar and launches it def setupTask = project.tasks.register(config.name+"Setup") { @@ -2692,17 +2692,7 @@ class BeamModulePlugin implements Plugin { ] def serviceArgs = project.project(':sdks:python').mapToArgString(expansionServiceOpts) def pythonContainerSuffix = project.project(':sdks:python').pythonVersion.replace('.', '') - def javaContainerSuffix - if (JavaVersion.current() == JavaVersion.VERSION_1_8) { - javaContainerSuffix = 'java8' - } else if (JavaVersion.current() == JavaVersion.VERSION_11) { - javaContainerSuffix = 'java11' - } else if (JavaVersion.current() == JavaVersion.VERSION_17) { - javaContainerSuffix = 'java17' - } else { - String exceptionMessage = "Your Java version is unsupported. You need Java version of 8 or 11 or 17 to get started, but your Java version is: " + JavaVersion.current(); - throw new GradleException(exceptionMessage) - } + def javaContainerSuffix = getSupportedJavaVersion() def setupTask = project.tasks.register(config.name+"Setup", Exec) { dependsOn ':sdks:java:container:'+javaContainerSuffix+':docker' dependsOn ':sdks:python:container:py'+pythonContainerSuffix+':docker' @@ -2875,17 +2865,7 @@ class BeamModulePlugin implements Plugin { ] def serviceArgs = project.project(':sdks:python').mapToArgString(transformServiceOpts) def pythonContainerSuffix = project.project(':sdks:python').pythonVersion.replace('.', '') - def javaContainerSuffix - if (JavaVersion.current() == JavaVersion.VERSION_1_8) { - javaContainerSuffix = 'java8' - } else if (JavaVersion.current() == JavaVersion.VERSION_11) { - javaContainerSuffix = 'java11' - } else if (JavaVersion.current() == JavaVersion.VERSION_17) { - javaContainerSuffix = 'java17' - } else { - String exceptionMessage = "Your Java version is unsupported. You need Java version of 8 or 11 or 17 to get started, but your Java version is: " + JavaVersion.current(); - throw new GradleException(exceptionMessage) - } + def javaContainerSuffix = getSupportedJavaVersion() // Transform service delivers transforms that refer to SDK harness containers with following sufixes. def transformServiceJavaContainerSuffix = 'java11' diff --git a/examples/java/build.gradle b/examples/java/build.gradle index 2e262e8de795..a43862ae801d 100644 --- a/examples/java/build.gradle +++ b/examples/java/build.gradle @@ -54,6 +54,10 @@ configurations.sparkRunnerPreCommit { dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) implementation library.java.vendored_guava_32_1_2_jre + if (project.findProperty('testJavaVersion') == '21' || JavaVersion.current().equals(JavaVersion.VERSION_21)) { + // this dependency is somehow needed for compile only under Java21 + compileOnly library.java.kafka + } implementation library.java.kafka_clients implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":sdks:java:extensions:avro") diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle index 98258401d588..829aefd44745 100644 --- a/examples/kotlin/build.gradle +++ b/examples/kotlin/build.gradle @@ -80,6 +80,9 @@ dependencies { for (String runner : preCommitRunners) { delegate.add(runner + "PreCommit", project(path: ":examples:kotlin", configuration: "testRuntimeMigration")) } + directRunnerPreCommit project(project.path) + flinkRunnerPreCommit project(project.path) + sparkRunnerPreCommit project(project.path) directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow") flinkRunnerPreCommit project(":runners:flink:${project.ext.latestFlinkVersion}") sparkRunnerPreCommit project(":runners:spark:3") @@ -106,10 +109,7 @@ for (String runner : preCommitRunners) { "--runner=" + preCommitRunnerClass[runner], ] classpath = configurations."${runner}PreCommit" - include "**/WordCountIT.class" - if (!"sparkRunner".equals(runner)) { - include "**/WindowedWordCountIT.class" - } + include "**/kotlin/**/*Test.class" forkEvery 1 maxParallelForks 4 systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions) @@ -136,4 +136,3 @@ compileTestKotlin { repositories { mavenCentral() } - diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ac72c34e8acc..3fa8f862f753 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 0adc8e1a5321..1aa94a426907 100755 --- a/gradlew +++ b/gradlew @@ -145,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -153,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -202,11 +202,11 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/sdks/java/container/agent/build.gradle b/sdks/java/container/agent/build.gradle index 044d175257cc..44338915bf28 100644 --- a/sdks/java/container/agent/build.gradle +++ b/sdks/java/container/agent/build.gradle @@ -47,15 +47,12 @@ jar { // the order is intended here if (project.hasProperty('java11Home')) { - def java11Home = project.findProperty('java11Home') project.tasks.withType(JavaCompile) { - options.fork = true - options.forkOptions.javaHome = java11Home as File - options.compilerArgs += ['-Xlint:-path'] + setJavaVerOptions(options, '11') } } else if (project.hasProperty('java17Home')) { project.tasks.withType(JavaCompile) { - setJava17Options(options) + setJavaVerOptions(options, '17') checkerFramework { skipCheckerFramework = true @@ -63,7 +60,7 @@ if (project.hasProperty('java11Home')) { } } else if (project.hasProperty('java21Home')) { project.tasks.withType(JavaCompile) { - setJava21Options(options) + setJavaVerOptions(options, '21') checkerFramework { skipCheckerFramework = true diff --git a/sdks/java/container/java21/build.gradle b/sdks/java/container/java21/build.gradle index 038064102dcb..75c84c0c7cdf 100644 --- a/sdks/java/container/java21/build.gradle +++ b/sdks/java/container/java21/build.gradle @@ -17,8 +17,6 @@ */ project.ext { - // TODO(https://github.com/apache/beam/issues/28120) switch to temurin once available - javaBaseImage = 'openjdk' imageJavaVersion = '21' } diff --git a/sdks/java/testing/jpms-tests/build.gradle b/sdks/java/testing/jpms-tests/build.gradle index 429fc063027f..9b881d7a2a9e 100644 --- a/sdks/java/testing/jpms-tests/build.gradle +++ b/sdks/java/testing/jpms-tests/build.gradle @@ -46,9 +46,9 @@ project.tasks.compileJava { options.fork = true options.forkOptions.javaHome = project.findProperty("java${project.getProperty('testJavaVersion')}Home") as File if (project.getProperty('testJavaVersion') == '17') { - setJava17Options(options) + setJavaVerOptions(options, '17') } else if (project.getProperty('testJavaVersion') == '21') { - setJava21Options(options) + setJavaVerOptions(options, '21') } } }