Yaml plugin addition

docs/modules/ROOT/pages/dataformats.adoc

@@ -840,3 +840,35 @@ The following parameters are supported:
 |
 
 |===
+
+## YAML Format
+### MIME types and identifiers
+* `application/x-yaml`
+
+### `read`
+
+Reads input YAML structure and converts it to the internal DataSonnet representation.
+
+No additional `read` parameters are supported.
+
+### `write`
+
+Creates YAML structure from the provided input.
+
+The following write parameters are supported:
+
+[%header, cols=3*a]
+|===
+|Parameter
+|Description
+|Default value
+
+|`MarkerLine`
+|If set to `false`, the resulting YAML will not contain the three-dashes boundary markers (`---`)
+|`true`
+
+|`DisableQuotes`
+|If set to `true`, output values will be unquoted, i.e. will not be wrapped in quote characters
+|`false`
+
+|===

pom.xml

@@ -140,6 +140,11 @@
             <artifactId>jackson-dataformat-csv</artifactId>
             <version>${jackson.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.dataformat</groupId>
             <artifactId>jackson-dataformat-properties</artifactId>

src/main/java/com/datasonnet/document/MediaTypes.java

@@ -1,7 +1,7 @@
 package com.datasonnet.document;
 
 /*-
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2019-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -348,6 +348,16 @@ public class MediaTypes {
 
     public static final String APPLICATION_CSV_VALUE = "application/csv";
 
+    /**
+     * Public constant media type for {@code application/yaml}.
+     */
+    public static final MediaType APPLICATION_YAML;
+
+    /**
+     * A String equivalent of {@link MediaTypes#APPLICATION_YAML}.
+     */
+    public static final String APPLICATION_YAML_VALUE = "application/x-yaml";
+
     // See Null Object pattern
     /**
      * Public constant media type for representing an unknown content type. This is meant to used to signal to Datasonnet
@@ -391,6 +401,7 @@ public class MediaTypes {
         APPLICATION_JAVA = new MediaType("application", "x-java-object");
         APPLICATION_CSV = new MediaType("application", "csv");
         UNKNOWN = new MediaType("unknown", "unknown");
+        APPLICATION_YAML = new MediaType("application", "x-yaml");
     }
 
     // TODO: 8/11/20 add explicit file extension support to MediaType class
@@ -404,6 +415,9 @@ public static Optional<MediaType> forExtension(String ext) {
                 return Optional.of(APPLICATION_CSV);
             case "txt":
                 return Optional.of(TEXT_PLAIN);
+            case "yml":
+            case "yaml":
+                return Optional.of(APPLICATION_YAML);
             default:
                 return Optional.empty();
         }

src/main/java/com/datasonnet/plugins/DefaultYamlFormatPlugin.java

@@ -0,0 +1,148 @@
+package com.datasonnet.plugins;
+
+/*-
+ * Copyright 2019-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.datasonnet.document.DefaultDocument;
+import com.datasonnet.document.Document;
+import com.datasonnet.document.MediaType;
+import com.datasonnet.document.MediaTypes;
+import com.datasonnet.spi.PluginException;
+import com.datasonnet.spi.ujsonUtils;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.fasterxml.jackson.dataformat.yaml.YAMLParser;
+import ujson.Value;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Optional;
+
+public class DefaultYamlFormatPlugin extends BaseJacksonDataFormatPlugin {
+
+    private static final ObjectMapper DEFAULT_OBJECT_MAPPER = new ObjectMapper();
+    private static final YAMLFactory DEFAULT_YAML_FACTORY = new YAMLFactory();
+
+    public static final String DS_PARAM_MARKER_LINE = "markerline";
+    // this may break some things like reading 3.0.0 as a number,
+    // would need to specify this in docs
+    public static final String DS_PARAM_DISABLE_QUOTES = "disablequotes";
+
+    public DefaultYamlFormatPlugin() {
+        supportedTypes.add(MediaTypes.APPLICATION_YAML);
+
+        readerSupportedClasses.add(java.lang.String.class);
+        readerSupportedClasses.add(java.lang.CharSequence.class);
+        readerSupportedClasses.add(java.nio.ByteBuffer.class);
+        readerSupportedClasses.add(byte[].class);
+
+        writerSupportedClasses.add(java.lang.String.class);
+        writerSupportedClasses.add(java.lang.CharSequence.class);
+        writerSupportedClasses.add(java.nio.ByteBuffer.class);
+        writerSupportedClasses.add(byte[].class);
+
+        readerParams.add(DS_PARAM_MARKER_LINE);
+        writerParams.addAll(readerParams);
+        writerParams.add(DS_PARAM_DISABLE_QUOTES);
+    }
+
+    @Override
+    public Value read(Document<?> doc) throws PluginException {
+        if (doc.getContent() == null) {
+            return ujson.Null$.MODULE$;
+        }
+
+        try {
+            YAMLParser yamlParser = DEFAULT_YAML_FACTORY.createParser((String) doc.getContent());
+            List<JsonNode> docs = DEFAULT_OBJECT_MAPPER.readValues(yamlParser, new TypeReference<JsonNode>() {
+            }).readAll();
+
+            if (docs.size() <= 1) { //if only one node, only one object so dont return the list
+                return ujsonFrom(DEFAULT_OBJECT_MAPPER.valueToTree(docs.get(0)));
+            }
+            return ujsonFrom(DEFAULT_OBJECT_MAPPER.valueToTree(docs));
+        } catch (IOException e) {
+            e.printStackTrace();
+            throw new PluginException("Failed to read yaml data");
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T> Document<T> write(Value input, MediaType mediaType, Class<T> targetType) throws PluginException {
+        Charset charset = mediaType.getCharset();
+        if (charset == null) {
+            charset = Charset.defaultCharset();
+        }
+
+        try {
+            Object inputAsJava = ujsonUtils.javaObjectFrom(input);
+            ObjectMapper yamlMapper = new ObjectMapper(DEFAULT_YAML_FACTORY);
+            StringBuilder value = null;
+
+            //if instance of list, it is multiple docs in one.
+            if (inputAsJava instanceof List) {
+                List<Object> listInputAsJava = (List<Object>) inputAsJava;
+                value = new StringBuilder();
+                for (Object obj : listInputAsJava) {
+                    value.append(yamlMapper.writeValueAsString(obj));
+                }
+            } else { //single document
+                //remove the beginning '---' if specified
+                //only available for single docs
+                String yaml = yamlMapper.writeValueAsString(inputAsJava);
+                if (mediaType.getParameters().containsKey(DS_PARAM_MARKER_LINE)) {
+                    String paramStr = mediaType.getParameters().get(DS_PARAM_MARKER_LINE);
+                    boolean disableMarkerLines = Boolean.parseBoolean(Optional.ofNullable(paramStr).orElse("true"));
+                    if (!disableMarkerLines) {
+                        yaml = yaml.replaceFirst("---(\\n| )", "");
+                    }
+                }
+                value = new StringBuilder(yaml);
+            }
+
+            String output = value.toString();
+            if (mediaType.getParameters().containsKey(DS_PARAM_DISABLE_QUOTES)) {
+                output = output.replaceAll("\"", "");
+            }
+
+            if (targetType.isAssignableFrom(String.class)) {
+                return new DefaultDocument<>((T) output, MediaTypes.APPLICATION_YAML);
+            }
+
+            if (targetType.isAssignableFrom(CharSequence.class)) {
+                return new DefaultDocument<>((T) output, MediaTypes.APPLICATION_YAML);
+            }
+
+            if (targetType.isAssignableFrom(ByteBuffer.class)) {
+                return new DefaultDocument<>((T) ByteBuffer.wrap(output.getBytes(charset)), MediaTypes.APPLICATION_YAML);
+            }
+
+            if (targetType.isAssignableFrom(byte[].class)) {
+                return new DefaultDocument<>((T) output.getBytes(charset), MediaTypes.APPLICATION_YAML);
+            }
+
+            throw new PluginException("Unable to parse to target type.");
+        } catch (JsonProcessingException e) {
+            throw new PluginException("Failed to write yaml data");
+        }
+    }
+
+}

src/main/java/com/datasonnet/spi/DataFormatService.java

@@ -35,6 +35,7 @@ public class DataFormatService {
                     DefaultXMLFormatPlugin$.MODULE$,
                     new DefaultCSVFormatPlugin(),
                     new DefaultPlainTextFormatPlugin(),
+                    new DefaultYamlFormatPlugin(),
                     new MimeMultipartPlugin()));
 
     public DataFormatService(List<DataFormatPlugin> plugins) {

src/test/java/com/datasonnet/YamlReaderTest.java

@@ -0,0 +1,99 @@
+package com.datasonnet;
+
+/*-
+ * Copyright 2019-2021 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.datasonnet.document.DefaultDocument;
+import com.datasonnet.document.MediaTypes;
+import org.junit.jupiter.api.Test;
+import org.skyscreamer.jsonassert.JSONAssert;
+
+import java.util.HashMap;
+
+public class YamlReaderTest {
+
+    @Test
+    void testYamlReader() throws Exception {
+        String data =   "message: \"Hello World\"\n" +
+                        "object: \n" +
+                        "  num: 3.14159\n" +
+                        "  bool: true\n" +
+                        "  array: \n" +
+                        "    - 1\n" +
+                        "    - 2";
+        DefaultDocument<?> doc = new DefaultDocument<>(data, MediaTypes.APPLICATION_YAML);
+
+        String mapping = "/** DataSonnet\n" +
+                "version=2.0\n" +
+                "output application/json\n" +
+                "input payload application/x-yaml\n" +
+                "*/\n" +
+                "payload";
+
+        Mapper mapper = new Mapper(mapping);
+        String mapped = mapper.transform(doc, new HashMap<>(), MediaTypes.APPLICATION_JSON).getContent();
+
+        String expectedJson = "{\"message\":\"Hello World\",\"object\":{\"num\":3.14159,\"bool\":true,\"array\":[1,2]}}";
+        JSONAssert.assertEquals(expectedJson, mapped, true);
+    }
+
+    @Test
+    void testYamlReaderWithLine() throws Exception {
+        String data =   "---\nmessage: \"Hello World\"\n" +
+                "object: \n" +
+                "  num: 3.14159\n" +
+                "  bool: true\n" +
+                "  array: \n" +
+                "    - 1\n" +
+                "    - 2";
+        DefaultDocument<?> doc = new DefaultDocument<>(data, MediaTypes.APPLICATION_YAML);
+
+        String mapping = "/** DataSonnet\n" +
+                "version=2.0\n" +
+                "output application/json\n" +
+                "input payload application/x-yaml\n" +
+                "*/\n" +
+                "payload";
+
+        Mapper mapper = new Mapper(mapping);
+        String mapped = mapper.transform(doc, new HashMap<>(), MediaTypes.APPLICATION_JSON).getContent();
+
+        String expectedJson = "{\"message\":\"Hello World\",\"object\":{\"num\":3.14159,\"bool\":true,\"array\":[1,2]}}";
+        JSONAssert.assertEquals(expectedJson, mapped, true);
+    }
+
+    @Test
+    void testYamlReaderMultiple() throws Exception {
+        String data =   "---\n" +
+                "message: \"Hello World\"\n" +
+                "---\n" +
+                "test: \"Value\"\n";
+        DefaultDocument<?> doc = new DefaultDocument<>(data, MediaTypes.APPLICATION_YAML);
+
+        String mapping = "/** DataSonnet\n" +
+                "version=2.0\n" +
+                "output application/json\n" +
+                "input payload application/x-yaml\n" +
+                "*/\n" +
+                "payload";
+
+        Mapper mapper = new Mapper(mapping);
+        String mapped = mapper.transform(doc, new HashMap<>(), MediaTypes.APPLICATION_JSON).getContent();
+
+        String expectedJson = "[{\"message\":\"Hello World\"},{\"test\":\"Value\"} ]";
+        JSONAssert.assertEquals(expectedJson, mapped, true);
+    }
+}