🐛 Destination S3-Glue: fix parsing empty object in schema (#19907)

* fix empty object in schema * treat blank object as string * refactor removed class * bump connector version * auto-bump connector version Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Dec 14, 2022 · b417937 · b417937
1 parent c637aed
commit b417937
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 11 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -312,7 +312,7 @@
 - name: S3 Glue
   destinationDefinitionId: 471e5cab-8ed1-49f3-ba11-79c687784737
   dockerRepository: airbyte/destination-s3-glue
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.1
   documentationUrl: https://docs.airbyte.com/integrations/destinations/s3-glue
   releaseStage: alpha
 - name: SFTP-JSON

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -5674,7 +5674,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-s3-glue:0.1.0"
+- dockerImage: "airbyte/destination-s3-glue:0.1.1"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/s3"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/destination-s3-glue/Dockerfile b/airbyte-integrations/connectors/destination-s3-glue/Dockerfile
@@ -14,5 +14,5 @@ ENV APPLICATION destination-s3-glue
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/destination-s3-glue
diff --git a/...ion-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/GlueOperations.java b/...ion-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/GlueOperations.java
@@ -146,13 +146,17 @@ private String transformSchemaRecursive(JsonNode jsonNode) {
         yield arrayType;
       }
       case "object" -> {
-        String objectType = "struct<";
-        Map<String, JsonNode> properties = objectMapper.convertValue(jsonNode.get("properties"), new TypeReference<>() {});
-        String columnTypes = properties.entrySet().stream()
-            .map(p -> p.getKey() + " : " + transformSchemaRecursive(p.getValue()))
-            .collect(Collectors.joining(","));
-        objectType += (columnTypes + ">");
-        yield objectType;
+        if (jsonNode.has("properties")) {
+          String objectType = "struct<";
+          Map<String, JsonNode> properties = objectMapper.convertValue(jsonNode.get("properties"), new TypeReference<>() {});
+          String columnTypes = properties.entrySet().stream()
+              .map(p -> p.getKey() + " : " + transformSchemaRecursive(p.getValue()))
+              .collect(Collectors.joining(","));
+          objectType += (columnTypes + ">");
+          yield objectType;
+        } else {
+          yield "string";
+        }
       }
       default -> type;
     };

diff --git a/...glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java b/...glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java
@@ -6,11 +6,12 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Preconditions;
+
 import io.airbyte.commons.functional.CheckedBiConsumer;
 import io.airbyte.commons.functional.CheckedBiFunction;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.integrations.base.AirbyteMessageConsumer;
-import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair;
+
 import io.airbyte.integrations.destination.NamingConventionTransformer;
 import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer;
 import io.airbyte.integrations.destination.buffered_stream_consumer.OnCloseFunction;
@@ -22,6 +23,7 @@
 import io.airbyte.integrations.destination.s3.WriteConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteStream;
+import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
 import io.airbyte.protocol.models.ConfiguredAirbyteStream;
 import io.airbyte.protocol.models.DestinationSyncMode;

diff --git a/docs/integrations/destinations/s3-glue.md b/docs/integrations/destinations/s3-glue.md
@@ -245,4 +245,5 @@ Output files can be compressed. The default option is GZIP compression. If compr
 
 | Version | Date       | Pull Request                                             | Subject        |
 | :------ | :--------- | :------------------------------------------------------- | :------------- |
+| 0.1.1   | 2022-12-13 | [19907](https://github.com/airbytehq/airbyte/pull/19907) | Fix parsing empty object in schema |
 | 0.1.0   | 2022-11-17 | [18695](https://github.com/airbytehq/airbyte/pull/18695) | Initial Commit |