[BEAM-6558] Add IWYU plugin, activate for Beam SQL, fix errors

build.gradle

@@ -65,6 +65,7 @@ buildscript {
     classpath "com.github.ben-manes:gradle-versions-plugin:0.17.0"                                      // Enable dependency checks
     classpath "org.ajoberstar.grgit:grgit-gradle:3.0.0"                                                 // Enable website git publish to asf-site branch
     classpath "com.avast.gradle:gradle-docker-compose-plugin:0.8.8"                                     // Enable docker compose tasks
+    classpath "ca.cutterslade.gradle:gradle-dependency-analyze:1.3.0"                                   // Enable dep analysis
 
     // Plugins which require online access should not be enabled when running in offline mode.
     if (!gradle.startParameter.isOffline()) {

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -82,6 +82,9 @@ class BeamModulePlugin implements Plugin<Project> {
     /** Controls whether the findbugs plugin is enabled and configured. */
     boolean enableFindbugs = true
 
+    /** Controls whether the dependency analysis plugin is enabled. */
+    boolean enableStrictDependencies = false
+
     /**
      * List of additional lint warnings to disable.
      * In addition, defaultLintSuppressions defined below
@@ -310,6 +313,15 @@ class BeamModulePlugin implements Plugin<Project> {
     // when attempting to resolve dependency issues.
     project.apply plugin: "project-report"
 
+    // Apply a plugin which fails the build if there is a dependency on a transitive
+    // non-declared dependency, since these can break users (as in BEAM-6558)
+    //
+    // Though this is Java-specific, it is required to be applied to the root
+    // project due to implemeentation-details of the plugin. It can be enabled/disabled
+    // via JavaNatureConfiguration per project. It is disabled by default until we can
+    // make all of our deps good.
+    project.apply plugin: "ca.cutterslade.analyze"
+
     // Adds a taskTree task that prints task dependency tree report to the console.
     // Useful for investigating build issues.
     // See: https://github.com/dorongold/gradle-task-tree
@@ -380,6 +392,7 @@ class BeamModulePlugin implements Plugin<Project> {
         byte_buddy                                  : "net.bytebuddy:byte-buddy:1.9.3",
         cassandra_driver_core                       : "com.datastax.cassandra:cassandra-driver-core:$cassandra_driver_version",
         cassandra_driver_mapping                    : "com.datastax.cassandra:cassandra-driver-mapping:$cassandra_driver_version",
+        commons_codec                               : "commons-codec:commons-codec:1.10",
         commons_compress                            : "org.apache.commons:commons-compress:1.16.1",
         commons_csv                                 : "org.apache.commons:commons-csv:1.4",
         commons_io_1x                               : "commons-io:commons-io:1.3.2",
@@ -769,6 +782,16 @@ class BeamModulePlugin implements Plugin<Project> {
         }
       }
 
+      if (configuration.enableStrictDependencies) {
+        project.tasks.analyzeClassesDependencies.enabled = true
+        project.tasks.analyzeDependencies.enabled = true
+        project.tasks.analyzeTestClassesDependencies.enabled = false
+      } else {
+        project.tasks.analyzeClassesDependencies.enabled = false
+        project.tasks.analyzeTestClassesDependencies.enabled = false
+        project.tasks.analyzeDependencies.enabled = false
+      }
+
       // Enable errorprone static analysis
       project.apply plugin: 'net.ltgt.errorprone'
 

sdks/java/extensions/sql/build.gradle

@@ -23,6 +23,7 @@ applyJavaNature(
   // javacc generated code produces lint warnings
   disableLintWarnings: ['dep-ann'],
   testShadowJar: true,
+  enableStrictDependencies: true,
   shadowClosure: DEFAULT_SHADOW_CLOSURE << {
     dependencies {
       include(dependency(library.java.protobuf_java))
@@ -71,6 +72,7 @@ dependencies {
   shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
   shadow project(path: ":beam-sdks-java-extensions-join-library", configuration: "shadow")
   shadow library.java.slf4j_api
+  shadow library.java.commons_codec
   shadow library.java.commons_csv
   shadow library.java.commons_lang3
   shadow library.java.jackson_databind
@@ -88,6 +90,14 @@ dependencies {
   shadowTest library.java.hamcrest_library
   shadowTest library.java.mockito_core
   shadowTest library.java.quickcheck_core
+
+  // Dependencies that we don't directly reference
+  permitUnusedDeclared "com.jayway.jsonpath:json-path:2.4.0"
+  permitUnusedDeclared library.java.jackson_dataformat_yaml
+
+  // Dependencies that are bundled in when we bundle Calcite
+  permitUsedUndeclared "org.codehaus.janino:janino:3.0.9"
+  permitUsedUndeclared "org.codehaus.janino:commons-compiler:3.0.9"
 }
 
 // Copy Caclcite templates and our own template into the build directory

sdks/java/extensions/sql/shell/build.gradle

@@ -22,10 +22,13 @@ apply plugin: "application"
 
 dependencies {
   compile project(path: ":beam-sdks-java-extensions-sql-jdbc", configuration: "shadow")
+  permitUnusedDeclared project(path: ":beam-sdks-java-extensions-sql-jdbc", configuration: "shadow")
 
   if (project.hasProperty("beam.sql.shell.bundled")) {
     project.getProperty("beam.sql.shell.bundled").tokenize(",").each {
-      subproject -> compile project(path: subproject, configuration: "shadow")
+      subproject ->
+          compile project(path: subproject, configuration: "shadow")
+          permitUnusedDeclared project(path: subproject, configuration: "shadow")
     }
   }
 }