Change scio-smb to depend on provided scio io modules

spotify · Jan 4, 2024 · bb6894e · bb6894e
1 parent 1671db7
commit bb6894e
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 15 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1002,8 +1002,8 @@ lazy val `scio-parquet` = project
   .in(file("scio-parquet"))
   .dependsOn(
     `scio-core`,
-    `scio-avro`,
     `scio-tensorflow` % "provided",
+    `scio-avro` % Test,
     `scio-test` % "test->test"
   )
   .settings(commonSettings)
@@ -1054,7 +1054,6 @@ lazy val `scio-parquet` = project
 lazy val `scio-tensorflow` = project
   .in(file("scio-tensorflow"))
   .dependsOn(
-    `scio-avro`,
     `scio-core`,
     `scio-test` % "test->test"
   )
@@ -1324,6 +1323,10 @@ lazy val `scio-smb` = project
   .in(file("scio-smb"))
   .dependsOn(
     `scio-core`,
+    `scio-avro` % Provided,
+    `scio-google-cloud-platform` % Provided,
+    `scio-parquet` % Provided,
+    `scio-tensorflow` % Provided,
     `scio-test` % "test->test"
   )
   .settings(commonSettings)
@@ -1342,7 +1345,6 @@ lazy val `scio-smb` = project
       "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion,
       "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion,
       "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
-      "com.google.apis" % "google-api-services-bigquery" % googleApiServicesBigQueryVersion,
       "com.google.auto.service" % "auto-service-annotations" % autoServiceVersion,
       "com.google.auto.value" % "auto-value-annotations" % autoValueVersion,
       "com.google.code.findbugs" % "jsr305" % jsr305Version,
@@ -1351,27 +1353,28 @@ lazy val `scio-smb` = project
       "com.spotify" %% "magnolify-parquet" % magnolifyVersion,
       "joda-time" % "joda-time" % jodaTimeVersion,
       "org.apache.beam" % "beam-sdks-java-core" % beamVersion,
-      "org.apache.beam" % "beam-sdks-java-extensions-avro" % beamVersion,
-      "org.apache.beam" % "beam-sdks-java-extensions-protobuf" % beamVersion,
       // #3260 work around for sorter memory limit until we patch upstream
       // "org.apache.beam" % "beam-sdks-java-extensions-sorter" % beamVersion,
-      "org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
-      "org.apache.beam" % "beam-sdks-java-io-hadoop-common" % beamVersion,
       "org.apache.beam" % "beam-vendor-guava-26_0-jre" % beamVendorVersion,
       "org.apache.commons" % "commons-lang3" % commonsLang3Version,
       "org.checkerframework" % "checker-qual" % checkerQualVersion,
       "org.slf4j" % "log4j-over-slf4j" % slf4jVersion, // log4j is excluded from hadoop
       "org.slf4j" % "slf4j-api" % slf4jVersion,
       // provided
+      "com.google.apis" % "google-api-services-bigquery" % googleApiServicesBigQueryVersion % Provided, // scio-gcp
       "com.github.ben-manes.caffeine" % "caffeine" % caffeineVersion % Provided,
-      "org.apache.avro" % "avro" % avroVersion % Provided,
-      "org.apache.hadoop" % "hadoop-common" % hadoopVersion % Provided,
+      "org.apache.avro" % "avro" % avroVersion % Provided,  // scio-avro
+      "org.apache.beam" % "beam-sdks-java-extensions-avro" % beamVersion % Provided, // scio-avro
+      "org.apache.beam" % "beam-sdks-java-extensions-protobuf" % beamVersion % Provided, // scio-tensorflow
+      "org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion % Provided, // scio-gcp
+      "org.apache.beam" % "beam-sdks-java-io-hadoop-common" % beamVersion % Provided, // scio-parquet
+      "org.apache.hadoop" % "hadoop-common" % hadoopVersion % Provided, // scio-parquet
       "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVersion % Provided,
-      "org.apache.parquet" % "parquet-avro" % parquetVersion % Provided excludeAll (Exclude.avro),
-      "org.apache.parquet" % "parquet-column" % parquetVersion % Provided,
-      "org.apache.parquet" % "parquet-common" % parquetVersion % Provided,
-      "org.apache.parquet" % "parquet-hadoop" % parquetVersion % Provided,
-      "org.tensorflow" % "tensorflow-core-api" % tensorFlowVersion % Provided,
+      "org.apache.parquet" % "parquet-avro" % parquetVersion % Provided excludeAll (Exclude.avro), // scio-parquet
+      "org.apache.parquet" % "parquet-column" % parquetVersion % Provided, // scio-parquet
+      "org.apache.parquet" % "parquet-common" % parquetVersion % Provided, // scio-parquet
+      "org.apache.parquet" % "parquet-hadoop" % parquetVersion % Provided, // scio-parquet
+      "org.tensorflow" % "tensorflow-core-api" % tensorFlowVersion % Provided,  // scio-tensorflow
       // runtime
       "org.apache.beam" % "beam-sdks-java-io-hadoop-format" % beamVersion % Runtime,
       "org.apache.hadoop" % "hadoop-client" % hadoopVersion % Runtime excludeAll (Exclude.metricsCore),

diff --git a/scio-parquet/src/main/scala/com/spotify/scio/parquet/types/dynamic/dynamic.scala b/scio-parquet/src/main/scala/com/spotify/scio/parquet/types/dynamic/dynamic.scala
@@ -19,7 +19,7 @@ package com.spotify.scio.parquet.types
 import com.spotify.scio.parquet.types.dynamic.syntax.AllSyntax
 
 /**
- * Parquet avro package for dynamic destinations. Import All.
+ * Parquet type-safe package for dynamic destinations. Import All.
  *
  * {{{
  * import com.spotify.scio.parquet.types.dynamic._