Skip to content

Commit

Permalink
Enable run test on Java 21
Browse files Browse the repository at this point in the history
* upgrade gradle 8,3 -> 8.4

* Use temurin 21 base image as its available now

* Setup Java21 JPMS postcommits

* Setup Java21 Example PostCommit on DataflowV2

* Fix kotlin test
  • Loading branch information
Abacn committed Oct 23, 2023
1 parent 683fbd6 commit f226b33
Show file tree
Hide file tree
Showing 10 changed files with 208 additions and 130 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
job_name: [beam_PostCommit_Java_Examples_Dataflow_V2_Java]
job_phrase_1: [Run Java ]
job_phrase_2: [Examples on Dataflow Runner V2]
java_version: ['11', '17']
java_version: ['11', '17', '21']
if: |
github.event_name == 'workflow_dispatch' ||
github.event_name == 'schedule' ||
Expand All @@ -76,7 +76,10 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: ${{ matrix.java_version }}
distribution: 'temurin'
java-version: |
${{ matrix.java_version }}
8
- name: run PostCommit Java Examples Dataflow V2 Java${{ matrix.java_version }} script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
Expand Down
97 changes: 97 additions & 0 deletions .github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: PostCommit Java Jpms Direct Java21

on:
schedule:
- cron: '0 */6 * * *'
workflow_dispatch:

#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
permissions:
actions: write
pull-requests: write
checks: write
contents: read
deployments: read
id-token: none
issues: write
discussions: read
packages: read
pages: read
repository-projects: read
security-events: read
statuses: read

# This allows a subsequently queued workflow run to interrupt previous runs
concurrency:
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}'
cancel-in-progress: true

env:
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}

jobs:
beam_PostCommit_Java_Jpms_Direct_Java21:
if: |
github.event_name == 'workflow_dispatch' ||
github.event_name == 'schedule' ||
github.event.comment.body == 'Run Jpms Direct Java 21 PostCommit'
runs-on: [self-hosted, ubuntu-20.04, main]
timeout-minutes: 240
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
strategy:
matrix:
job_name: ["beam_PostCommit_Java_Jpms_Direct_Java21"]
job_phrase: ["Run Jpms Direct Java 21 PostCommit"]
steps:
- uses: actions/checkout@v4
- name: Setup repository
uses: ./.github/actions/setup-action
with:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- name: Set up Java
uses: actions/setup-java@v3.11.0
with:
distribution: 'temurin'
java-version: |
21
8
- name: run PostCommit Java Jpms Direct Java21 script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:java:testing:jpms-tests:directRunnerIntegrationTest
arguments:
-PskipCheckerFramework
-PtestJavaVersion=21
-Pjava21Home=$JAVA_HOME_21_X64
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v3
if: failure()
with:
name: JUnit Test Results
path: "**/build/reports/tests/"
- name: Publish JUnit Test Results
uses: EnricoMi/publish-unit-test-result-action@v2
if: always()
with:
commit: '${{ env.prsha || env.GITHUB_SHA }}'
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
files: '**/build/test-results/**/*.xml'
190 changes: 85 additions & 105 deletions buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,21 @@ class BeamModulePlugin implements Plugin<Project> {
return 'beam' + p.path.replace(':', '-')
}

static def getSupportedJavaVersion() {
if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
return 'java8'
} else if (JavaVersion.current() == JavaVersion.VERSION_11) {
return 'java11'
} else if (JavaVersion.current() == JavaVersion.VERSION_17) {
return 'java17'
} else if (JavaVersion.current() == JavaVersion.VERSION_21) {
return 'java21'
} else {
String exceptionMessage = "Your Java version is unsupported. You need Java version of 8, 11, 17 or 21 to get started, but your Java version is: " + JavaVersion.current();
throw new GradleException(exceptionMessage)
}
}

/*
* Set compile args for compiling and running in different java version by modifying the compiler args in place.
*
Expand Down Expand Up @@ -922,51 +937,63 @@ class BeamModulePlugin implements Plugin<Project> {
+ suffix)
}

project.ext.setJava17Options = { CompileOptions options ->
def java17Home = project.findProperty("java17Home")
options.fork = true
options.forkOptions.javaHome = java17Home as File
options.compilerArgs += ['-Xlint:-path']
// Error prone requires some packages to be exported/opened for Java 17
// Disabling checks since this property is only used for Jenkins tests
// https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support
options.errorprone.errorproneArgs.add("-XepDisableAllChecks")
// The -J prefix is needed to workaround https://github.com/gradle/gradle/issues/22747
options.forkOptions.jvmArgs += [
"-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
"-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
"-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED"
]
}

project.ext.setJava21Options = { CompileOptions options ->
def java21Home = project.findProperty("java21Home")
options.fork = true
options.forkOptions.javaHome = java21Home as File
options.compilerArgs += ['-Xlint:-path']
// Error prone requires some packages to be exported/opened for Java 17+
// Disabling checks since this property is only used for Jenkins tests
// https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support
options.errorprone.errorproneArgs.add("-XepDisableAllChecks")
options.forkOptions.jvmArgs += [
"-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
"-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
"-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED"
]
// set compiler options for java version overrides to compile with a different java version
project.ext.setJavaVerOptions = { CompileOptions options, String ver ->
if (ver == '11') {
def java11Home = project.findProperty("java11Home")
options.fork = true
options.forkOptions.javaHome = java11Home as File
options.compilerArgs += ['-Xlint:-path']
} else if (ver == '17') {
def java17Home = project.findProperty("java17Home")
options.fork = true
options.forkOptions.javaHome = java17Home as File
options.compilerArgs += ['-Xlint:-path']
// Error prone requires some packages to be exported/opened for Java 17
// Disabling checks since this property is only used for tests
// https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support
options.errorprone.errorproneArgs.add("-XepDisableAllChecks")
// The -J prefix is needed to workaround https://github.com/gradle/gradle/issues/22747
options.forkOptions.jvmArgs += [
"-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
"-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
"-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED"
]
} else if (ver == '21') {
def java21Home = project.findProperty("java21Home")
options.fork = true
options.forkOptions.javaHome = java21Home as File
options.compilerArgs += ['-Xlint:-path']
// Error prone requires some packages to be exported/opened for Java 17+
// Disabling checks since this property is only used for tests
options.errorprone.errorproneArgs.add("-XepDisableAllChecks")
options.forkOptions.jvmArgs += [
"-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
"-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
"-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED"
]
// TODO(https://github.com/apache/beam/issues/28963)
// upgrade checkerFramework to enable it in Java 21
project.checkerFramework {
skipCheckerFramework = true
}
} else {
throw new GradleException("Unknown Java Version ${ver} for setting additional java options")
}
}

project.ext.repositories = {
Expand Down Expand Up @@ -1515,37 +1542,20 @@ class BeamModulePlugin implements Plugin<Project> {
options.errorprone.errorproneArgs.add("-Xep:Slf4jLoggerShouldBeNonStatic:OFF")
}

if (project.findProperty('testJavaVersion') == "11") {
def java11Home = project.findProperty("java11Home")
project.tasks.compileTestJava {
options.fork = true
options.forkOptions.javaHome = java11Home as File
options.compilerArgs += ['-Xlint:-path']
setCompileAndRuntimeJavaVersion(options.compilerArgs, '11')
}
project.tasks.withType(Test).configureEach {
useJUnit()
executable = "${java11Home}/bin/java"
}
} else if (project.findProperty('testJavaVersion') == "17") {
def java17Home = project.findProperty("java17Home")
project.tasks.compileTestJava {
setCompileAndRuntimeJavaVersion(options.compilerArgs, '17')
project.ext.setJava17Options(options)
}
project.tasks.withType(Test).configureEach {
useJUnit()
executable = "${java17Home}/bin/java"
}
} else if (project.findProperty('testJavaVersion') == "21") {
def java21Home = project.findProperty("java21Home")
// if specified test java version, modify the compile and runtime versions accordingly
if (project.hasProperty('testJavaVersion')) {
String ver = project.getProperty('testJavaVersion')
def testJavaHome = project.getProperty("java${ver}Home")

// redirect java compiler to specified version for compileTestJava only
project.tasks.compileTestJava {
setCompileAndRuntimeJavaVersion(options.compilerArgs, '21')
project.ext.setJava17Options(options)
setCompileAndRuntimeJavaVersion(options.compilerArgs, ver)
project.ext.setJavaVerOptions(options, ver)
}
// redirect java runtime to specified version for running tests
project.tasks.withType(Test).configureEach {
useJUnit()
executable = "${java21Home}/bin/java"
executable = "${testJavaHome}/bin/java"
}
}

Expand Down Expand Up @@ -2579,17 +2589,7 @@ class BeamModulePlugin implements Plugin<Project> {
"java_expansion_service_allowlist_file": javaClassLookupAllowlistFile,
]
def usesDataflowRunner = config.pythonPipelineOptions.contains("--runner=TestDataflowRunner") || config.pythonPipelineOptions.contains("--runner=DataflowRunner")
def javaContainerSuffix
if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
javaContainerSuffix = 'java8'
} else if (JavaVersion.current() == JavaVersion.VERSION_11) {
javaContainerSuffix = 'java11'
} else if (JavaVersion.current() == JavaVersion.VERSION_17) {
javaContainerSuffix = 'java17'
} else {
String exceptionMessage = "Your Java version is unsupported. You need Java version of 8 or 11 or 17 to get started, but your Java version is: " + JavaVersion.current();
throw new GradleException(exceptionMessage)
}
def javaContainerSuffix = getSupportedJavaVersion()

// 1. Builds the chosen expansion service jar and launches it
def setupTask = project.tasks.register(config.name+"Setup") {
Expand Down Expand Up @@ -2692,17 +2692,7 @@ class BeamModulePlugin implements Plugin<Project> {
]
def serviceArgs = project.project(':sdks:python').mapToArgString(expansionServiceOpts)
def pythonContainerSuffix = project.project(':sdks:python').pythonVersion.replace('.', '')
def javaContainerSuffix
if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
javaContainerSuffix = 'java8'
} else if (JavaVersion.current() == JavaVersion.VERSION_11) {
javaContainerSuffix = 'java11'
} else if (JavaVersion.current() == JavaVersion.VERSION_17) {
javaContainerSuffix = 'java17'
} else {
String exceptionMessage = "Your Java version is unsupported. You need Java version of 8 or 11 or 17 to get started, but your Java version is: " + JavaVersion.current();
throw new GradleException(exceptionMessage)
}
def javaContainerSuffix = getSupportedJavaVersion()
def setupTask = project.tasks.register(config.name+"Setup", Exec) {
dependsOn ':sdks:java:container:'+javaContainerSuffix+':docker'
dependsOn ':sdks:python:container:py'+pythonContainerSuffix+':docker'
Expand Down Expand Up @@ -2875,17 +2865,7 @@ class BeamModulePlugin implements Plugin<Project> {
]
def serviceArgs = project.project(':sdks:python').mapToArgString(transformServiceOpts)
def pythonContainerSuffix = project.project(':sdks:python').pythonVersion.replace('.', '')
def javaContainerSuffix
if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
javaContainerSuffix = 'java8'
} else if (JavaVersion.current() == JavaVersion.VERSION_11) {
javaContainerSuffix = 'java11'
} else if (JavaVersion.current() == JavaVersion.VERSION_17) {
javaContainerSuffix = 'java17'
} else {
String exceptionMessage = "Your Java version is unsupported. You need Java version of 8 or 11 or 17 to get started, but your Java version is: " + JavaVersion.current();
throw new GradleException(exceptionMessage)
}
def javaContainerSuffix = getSupportedJavaVersion()

// Transform service delivers transforms that refer to SDK harness containers with following sufixes.
def transformServiceJavaContainerSuffix = 'java11'
Expand Down
4 changes: 4 additions & 0 deletions examples/java/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ configurations.sparkRunnerPreCommit {
dependencies {
implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
implementation library.java.vendored_guava_32_1_2_jre
if (project.findProperty('testJavaVersion') == '21' || JavaVersion.current().equals(JavaVersion.VERSION_21)) {
// this dependency is somehow needed for compile only under Java21
compileOnly library.java.kafka
}
implementation library.java.kafka_clients
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation project(":sdks:java:extensions:avro")
Expand Down
9 changes: 4 additions & 5 deletions examples/kotlin/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ dependencies {
for (String runner : preCommitRunners) {
delegate.add(runner + "PreCommit", project(path: ":examples:kotlin", configuration: "testRuntimeMigration"))
}
directRunnerPreCommit project(project.path)
flinkRunnerPreCommit project(project.path)
sparkRunnerPreCommit project(project.path)
directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow")
flinkRunnerPreCommit project(":runners:flink:${project.ext.latestFlinkVersion}")
sparkRunnerPreCommit project(":runners:spark:3")
Expand All @@ -106,10 +109,7 @@ for (String runner : preCommitRunners) {
"--runner=" + preCommitRunnerClass[runner],
]
classpath = configurations."${runner}PreCommit"
include "**/WordCountIT.class"
if (!"sparkRunner".equals(runner)) {
include "**/WindowedWordCountIT.class"
}
include "**/kotlin/**/*Test.class"
forkEvery 1
maxParallelForks 4
systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions)
Expand All @@ -136,4 +136,3 @@ compileTestKotlin {
repositories {
mavenCentral()
}

Loading

0 comments on commit f226b33

Please sign in to comment.