From 98cfe65bf9842564c43d6816d04bcee5ec74b846 Mon Sep 17 00:00:00 2001 From: Kenneth Knowles Date: Thu, 31 Jan 2019 18:36:37 -0800 Subject: [PATCH 1/2] [BEAM-6566] Add plugin to check IWYU dependencies, inactivated --- build.gradle | 1 + .../beam/gradle/BeamModulePlugin.groovy | 22 +++++++++++++++++++ sdks/java/extensions/sql/shell/build.gradle | 5 ++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ca311655f07f0..e526580becc24 100644 --- a/build.gradle +++ b/build.gradle @@ -65,6 +65,7 @@ buildscript { classpath "com.github.ben-manes:gradle-versions-plugin:0.17.0" // Enable dependency checks classpath "org.ajoberstar.grgit:grgit-gradle:3.0.0" // Enable website git publish to asf-site branch classpath "com.avast.gradle:gradle-docker-compose-plugin:0.8.8" // Enable docker compose tasks + classpath "ca.cutterslade.gradle:gradle-dependency-analyze:1.3.0" // Enable dep analysis // Plugins which require online access should not be enabled when running in offline mode. if (!gradle.startParameter.isOffline()) { diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 396b7aaccb60a..3c49a6ee7e3f8 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -82,6 +82,9 @@ class BeamModulePlugin implements Plugin { /** Controls whether the findbugs plugin is enabled and configured. */ boolean enableFindbugs = true + /** Controls whether the dependency analysis plugin is enabled. */ + boolean enableStrictDependencies = false + /** * List of additional lint warnings to disable. * In addition, defaultLintSuppressions defined below @@ -310,6 +313,15 @@ class BeamModulePlugin implements Plugin { // when attempting to resolve dependency issues. project.apply plugin: "project-report" + // Apply a plugin which fails the build if there is a dependency on a transitive + // non-declared dependency, since these can break users (as in BEAM-6558) + // + // Though this is Java-specific, it is required to be applied to the root + // project due to implemeentation-details of the plugin. It can be enabled/disabled + // via JavaNatureConfiguration per project. It is disabled by default until we can + // make all of our deps good. + project.apply plugin: "ca.cutterslade.analyze" + // Adds a taskTree task that prints task dependency tree report to the console. // Useful for investigating build issues. // See: https://github.com/dorongold/gradle-task-tree @@ -769,6 +781,16 @@ class BeamModulePlugin implements Plugin { } } + if (configuration.enableStrictDependencies) { + project.tasks.analyzeClassesDependencies.enabled = true + project.tasks.analyzeDependencies.enabled = true + project.tasks.analyzeTestClassesDependencies.enabled = false + } else { + project.tasks.analyzeClassesDependencies.enabled = false + project.tasks.analyzeTestClassesDependencies.enabled = false + project.tasks.analyzeDependencies.enabled = false + } + // Enable errorprone static analysis project.apply plugin: 'net.ltgt.errorprone' diff --git a/sdks/java/extensions/sql/shell/build.gradle b/sdks/java/extensions/sql/shell/build.gradle index 825a19ec2aac7..7a1aa94f7ed1e 100644 --- a/sdks/java/extensions/sql/shell/build.gradle +++ b/sdks/java/extensions/sql/shell/build.gradle @@ -22,10 +22,13 @@ apply plugin: "application" dependencies { compile project(path: ":beam-sdks-java-extensions-sql-jdbc", configuration: "shadow") + permitUnusedDeclared project(path: ":beam-sdks-java-extensions-sql-jdbc", configuration: "shadow") if (project.hasProperty("beam.sql.shell.bundled")) { project.getProperty("beam.sql.shell.bundled").tokenize(",").each { - subproject -> compile project(path: subproject, configuration: "shadow") + subproject -> + compile project(path: subproject, configuration: "shadow") + permitUnusedDeclared project(path: subproject, configuration: "shadow") } } } From 297bc1dfc852497c52fadcc8a3c52b7e382e53ab Mon Sep 17 00:00:00 2001 From: Kenneth Knowles Date: Thu, 31 Jan 2019 18:37:05 -0800 Subject: [PATCH 2/2] [BEAM-6566] Activate IWYU for Beam SQL and fix errors --- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 1 + sdks/java/extensions/sql/build.gradle | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 3c49a6ee7e3f8..0223ad0a315af 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -392,6 +392,7 @@ class BeamModulePlugin implements Plugin { byte_buddy : "net.bytebuddy:byte-buddy:1.9.3", cassandra_driver_core : "com.datastax.cassandra:cassandra-driver-core:$cassandra_driver_version", cassandra_driver_mapping : "com.datastax.cassandra:cassandra-driver-mapping:$cassandra_driver_version", + commons_codec : "commons-codec:commons-codec:1.10", commons_compress : "org.apache.commons:commons-compress:1.16.1", commons_csv : "org.apache.commons:commons-csv:1.4", commons_io_1x : "commons-io:commons-io:1.3.2", diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 407366ae1d79b..0f792769c9f34 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -23,6 +23,7 @@ applyJavaNature( // javacc generated code produces lint warnings disableLintWarnings: ['dep-ann'], testShadowJar: true, + enableStrictDependencies: true, shadowClosure: DEFAULT_SHADOW_CLOSURE << { dependencies { include(dependency(library.java.protobuf_java)) @@ -71,6 +72,7 @@ dependencies { shadow project(path: ":beam-sdks-java-core", configuration: "shadow") shadow project(path: ":beam-sdks-java-extensions-join-library", configuration: "shadow") shadow library.java.slf4j_api + shadow library.java.commons_codec shadow library.java.commons_csv shadow library.java.commons_lang3 shadow library.java.jackson_databind @@ -88,6 +90,14 @@ dependencies { shadowTest library.java.hamcrest_library shadowTest library.java.mockito_core shadowTest library.java.quickcheck_core + + // Dependencies that we don't directly reference + permitUnusedDeclared "com.jayway.jsonpath:json-path:2.4.0" + permitUnusedDeclared library.java.jackson_dataformat_yaml + + // Dependencies that are bundled in when we bundle Calcite + permitUsedUndeclared "org.codehaus.janino:janino:3.0.9" + permitUsedUndeclared "org.codehaus.janino:commons-compiler:3.0.9" } // Copy Caclcite templates and our own template into the build directory