From aff736fb7cdf8f3bff03de421111a8e756f02d8e Mon Sep 17 00:00:00 2001 From: Luke Cwik Date: Fri, 17 Nov 2017 10:49:44 -0800 Subject: [PATCH] Update build.gradle files to correspond with pom.xml changes within master branch up to f10399d7c374d0342fdac524c8a2ddf4d7ec5cd3 --- build.gradle | 6 ++- runners/core-construction-java/build.gradle | 1 + runners/core-java/build.gradle | 1 + runners/direct-java/build.gradle | 1 + .../google-cloud-dataflow-java/build.gradle | 2 +- runners/java-fn-execution/build.gradle | 12 ++++- sdks/java/core/build.gradle | 2 +- sdks/java/extensions/sql/build.gradle | 5 ++ sdks/java/fn-execution/build.gradle | 12 ++++- sdks/java/io/file-based-io-tests/build.gradle | 48 +++++++++++++++++++ settings.gradle | 2 + 11 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 sdks/java/io/file-based-io-tests/build.gradle diff --git a/build.gradle b/build.gradle index a35455ec51cb5..8e2b4a5a4b257 100644 --- a/build.gradle +++ b/build.gradle @@ -74,6 +74,7 @@ ext.library = [ datastore_v1_proto_client: "com.google.cloud.datastore:datastore-v1-proto-client:1.4.0", datastore_v1_protos: "com.google.cloud.datastore:datastore-v1-protos:1.3.0", error_prone_annotations: "com.google.errorprone:error_prone_annotations:2.0.15", + findbugs_annotations: "com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1", findbugs_jsr305: "com.google.code.findbugs:jsr305:3.0.1", gax_grpc: "com.google.api:gax-grpc:0.20.0", google_api_client: "com.google.api-client:google-api-client:$google_clients_version", @@ -83,7 +84,7 @@ ext.library = [ google_api_services_bigquery: "com.google.apis:google-api-services-bigquery:v2-rev355-$google_clients_version", google_api_services_clouddebugger: "com.google.apis:google-api-services-clouddebugger:v2-rev8-$google_clients_version", google_api_services_cloudresourcemanager: "com.google.apis:google-api-services-cloudresourcemanager:v1-rev6-$google_clients_version", - google_api_services_dataflow: "com.google.apis:google-api-services-dataflow:v1b3-rev214-$google_clients_version", + google_api_services_dataflow: "com.google.apis:google-api-services-dataflow:v1b3-rev218-$google_clients_version", google_api_services_pubsub: "com.google.apis:google-api-services-pubsub:v1-rev10-$google_clients_version", google_api_services_storage: "com.google.apis:google-api-services-storage:v1-rev71-$google_clients_version", google_auth_library_credentials: "com.google.auth:google-auth-library-credentials:$google_auth_version", @@ -254,6 +255,9 @@ rat { // Proto/grpc generated wrappers "**/apache_beam/portability/api/*_pb2*.py", "**/go/pkg/beam/model/**/*.pb.go", + + // VCF test files + "**/apache_beam/testing/data/vcf/*", ] } check.dependsOn rat diff --git a/runners/core-construction-java/build.gradle b/runners/core-construction-java/build.gradle index a330426a37c91..8572372a21ff1 100644 --- a/runners/core-construction-java/build.gradle +++ b/runners/core-construction-java/build.gradle @@ -31,6 +31,7 @@ test { dependencies { compile library.java.guava + compileOnly librarj.java.findbugs_annotations shadow project(path: ":beam-model-parent:beam-model-pipeline", configuration: "shadow") shadow project(path: ":beam-model-parent:beam-model-job-management", configuration: "shadow") shadow project(path: ":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core", configuration: "shadow") diff --git a/runners/core-java/build.gradle b/runners/core-java/build.gradle index 0c0450781ea6a..52102b47f0109 100644 --- a/runners/core-java/build.gradle +++ b/runners/core-java/build.gradle @@ -31,6 +31,7 @@ test { dependencies { compile library.java.guava + compileOnly library.java.findbugs_annotations shadow project(path: ":beam-model-parent:beam-model-pipeline", configuration: "shadow") shadow project(path: ":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core", configuration: "shadow") shadow project(path: ":beam-model-parent:beam-model-fn-execution", configuration: "shadow") diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle index 1a770f9a6cd9b..335e552056df6 100644 --- a/runners/direct-java/build.gradle +++ b/runners/direct-java/build.gradle @@ -64,6 +64,7 @@ shadowJar { relocate "org.apache.beam.runners.core", "org.apache.beam." + project.name.replace("-", ".") + ".repackaged.runners.core" relocate "org.apache.beam.model", "org.apache.beam." + project.name.replace("-", ".") + ".repackaged.model" relocate "com.google.protobuf", "org.apache.beam." + project.name.replace("-", ".") + ".repackaged.com.google.protobuf" + relocate "javax.annotation", "org.apache.beam." + project.name.replace("-", ".") + ".repackaged.javax.annotation" } task packageTests(type: Jar) { diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 41c7c2af5360d..47a59b62fba41 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -36,7 +36,7 @@ processResources { filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ 'dataflow.legacy_environment_major_version' : '6', 'dataflow.fnapi_environment_major_version' : '1', - 'dataflow.container_version' : 'beam-master-20170926' + 'dataflow.container_version' : 'beam-master-20171113' ] } diff --git a/runners/java-fn-execution/build.gradle b/runners/java-fn-execution/build.gradle index d4646c65ece7a..dd4eaaed47d42 100644 --- a/runners/java-fn-execution/build.gradle +++ b/runners/java-fn-execution/build.gradle @@ -17,7 +17,7 @@ */ apply from: project(":").file("build_rules.gradle") -applyJavaNature(javaVersion: 1.8) +applyJavaNature() description = "Apache Beam :: Runners :: Java Fn Execution" @@ -30,6 +30,14 @@ description = "Apache Beam :: Runners :: Java Fn Execution" */ evaluationDependsOn(":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution") +configurations.all { + // Fn Execution contains shared utilities for Runners and Harnesses which use + // the Portability framework. Runner-side interactions must not require a + // dependency on any particular SDK, so this library must not introduce such an + // edge. + exclude group: "org.apache.beam", module: "beam-sdks-java-core" +} + dependencies { compile library.java.guava shadow project(path: ":beam-model-parent:beam-model-pipeline", configuration: "shadow") @@ -38,10 +46,12 @@ dependencies { shadow library.java.grpc_core shadow library.java.grpc_stub shadow library.java.grpc_netty + shadow library.java.slf4j_api testCompile project(":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution").sourceSets.test.output testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library + testCompile library.java.mockito_core } task packageTests(type: Jar) { diff --git a/sdks/java/core/build.gradle b/sdks/java/core/build.gradle index b1f35ba7335d6..732eb05c3b0b7 100644 --- a/sdks/java/core/build.gradle +++ b/sdks/java/core/build.gradle @@ -53,7 +53,7 @@ dependencies { compile library.java.byte_buddy compile library.java.commons_compress compile library.java.commons_lang3 - compileOnly "com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1" + compileOnly library.java.findbugs_annotations shadow library.java.jackson_core shadow library.java.jackson_annotations shadow library.java.jackson_databind diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 90189aaeb4c30..38335c0c5899d 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -33,6 +33,8 @@ dependencies { compile "org.apache.calcite:calcite-core:$calcite_version" compile "org.apache.calcite:calcite-linq4j:$calcite_version" compile "org.apache.calcite.avatica:avatica-core:$avatica_version" + compile "com.alibaba:fastjson:1.2.12" + compileOnly library.java.findbugs_jsr305 shadow project(path: ":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core", configuration: "shadow") shadow project(path: ":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-extensions-parent:beam-sdks-java-extensions-join-library", configuration: "shadow") shadow library.java.slf4j_api @@ -44,8 +46,11 @@ dependencies { testCompile library.java.slf4j_jdk14 testCompile library.java.junit testCompile library.java.hamcrest_core + testCompile library.java.mockito_core } +// TODO: Add support for generating fmpp + shadowJar { dependencies { include(dependency("org.apache.calcite:.*")) diff --git a/sdks/java/fn-execution/build.gradle b/sdks/java/fn-execution/build.gradle index 5451154e385e5..1ffe428385a22 100644 --- a/sdks/java/fn-execution/build.gradle +++ b/sdks/java/fn-execution/build.gradle @@ -17,9 +17,17 @@ */ apply from: project(":").file("build_rules.gradle") -applyJavaNature(javaVersion: 1.8) +applyJavaNature() -description = "Apache Beam :: SDKs :: Java :: Harness Core" +description = "Apache Beam :: SDKs :: Java :: Fn Execution" + +configurations.all { + // Fn Execution contains shared utilities for Runners and Harnesses which use + // the Portability framework. Runner-side interactions must not require a + // dependency on any particular SDK, so this library must not introduce such an + // edge. + exclude group: "org.apache.beam", module: "beam-sdks-java-core" +} dependencies { compile library.java.guava diff --git a/sdks/java/io/file-based-io-tests/build.gradle b/sdks/java/io/file-based-io-tests/build.gradle new file mode 100644 index 0000000000000..75e942e94b324 --- /dev/null +++ b/sdks/java/io/file-based-io-tests/build.gradle @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply from: project(":").file("build_rules.gradle") +applyJavaNature() + +description = "Apache Beam :: SDKs :: Java :: File-based-io-tests" + +/* + * We need to rely on manually specifying these evaluationDependsOn to ensure that + * the following projects are evaluated before we evaluate this project. This is because + * we are attempting to reference the "sourceSets.test.output" directly. + * TODO: Swap to generating test artifacts which we can then rely on instead of + * the test outputs directly. + */ +evaluationDependsOn(":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-common") + +dependencies { + shadowTest project(":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core") + shadowTest project(":beam-runners-parent:beam-runners-direct-java").sourceSets.test.output + shadowTest project(":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-common") + shadowTest project(":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-common").sourceSets.test.output + shadowTest library.java.guava + shadowTest library.java.junit + shadowTest library.java.hamcrest_core +} + +task packageTests(type: Jar) { + from sourceSets.test.output + classifier = "tests" +} + +artifacts.archives packageTests diff --git a/settings.gradle b/settings.gradle index 4a1888d578fbd..cd5a9f7db7d4f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -50,6 +50,7 @@ include ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-s include ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-redis' include ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-solr' include ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-tika' +include ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-file-based-io-tests' include ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-xml' include ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent' include ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-maven-archetypes-parent:beam-sdks-java-maven-archetypes-starter' @@ -125,6 +126,7 @@ project(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-s project(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-redis').projectDir = "$rootDir/sdks/java/io/redis" as File project(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-solr').projectDir = "$rootDir/sdks/java/io/solr" as File project(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-tika').projectDir = "$rootDir/sdks/java/io/tika" as File +project(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-file-based-io-tests').projectDir = "$rootDir/sdks/java/io/file-based-io-tests" as File project(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-xml').projectDir = "$rootDir/sdks/java/io/xml" as File project(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent').projectDir = "$rootDir/sdks/java/io" as File project(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-maven-archetypes-parent:beam-sdks-java-maven-archetypes-starter').projectDir = "$rootDir/sdks/java/maven-archetypes/starter" as File