Pull request #133: [ITB-1769] Extend the JSON validator to also suppo…

…rt YAML as input

Merge in ITB/json-validator from development to master

* commit '303fc99863e791d79cf5c4ab177cbaf2361bb999':
  [ITB-1769] Extend the JSON validator to also support YAML as input

README.md

@@ -10,7 +10,7 @@
 
 # JSON validator
 
-The **JSON validator** is a web application to validate JSON data against [JSON Schema](https://json-schema.org/) (Core Draft v4, v6, v7, v2019-09 and v2020-12).
+The **JSON validator** is a web application to validate JSON and YAML content data against [JSON Schema](https://json-schema.org/) (Core Draft v4, v6, v7, v2019-09 and v2020-12).
 The application provides a fully reusable core that requires only configuration to determine the supported specifications,
 configured validation types and other validator customisations. The web application allows validation via:
 
@@ -137,6 +137,7 @@ The JSON validator calls plugins in sequence passing in the following input:
 | `tempFolder`        | `String`  | The absolute and full path to a temporary folder for plugins. This will be automatically deleted after all plugins complete validation. |
 | `locale`            | `String`  | The locale (language code) to use for reporting of results (e.g. "fr", "fr_FR").                                                        |
 | `locationAsPointer` | `Boolean` | Whether report item locations should be JSON pointers or not.                                                                           |
+| `yaml`              | `Boolean` | Whether content should be treated as YAML (false - the default - meaning JSON).                                                         |
 
 ## Output from plugins
 

jsonvalidator-common/src/main/java/eu/europa/ec/itb/json/DomainConfig.java

@@ -1,11 +1,14 @@
 package eu.europa.ec.itb.json;
 
+import eu.europa.ec.itb.json.validation.YamlSupportEnum;
 import eu.europa.ec.itb.validation.commons.artifact.ExternalArtifactSupport;
 import eu.europa.ec.itb.validation.commons.artifact.TypedValidationArtifactInfo;
 import eu.europa.ec.itb.validation.commons.artifact.ValidationArtifactInfo;
 import eu.europa.ec.itb.validation.commons.config.WebDomainConfig;
 
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -15,6 +18,8 @@ public class DomainConfig extends WebDomainConfig {
 
     private final Set<String> sharedSchemas = new HashSet<>();
     private boolean reportItemCount = false;
+    private Map<String, YamlSupportEnum> yamlSupport = new HashMap<>();
+    private YamlSupportEnum defaultYamlSupport = YamlSupportEnum.NONE;
 
     /**
      * Whether the result report should list the number of parsed items in case an array was provided.
@@ -51,6 +56,28 @@ public ValidationArtifactInfo getSchemaInfo(String validationType) {
         return getArtifactInfo().get(validationType).get();
     }
 
+    /**
+     * @param yamlSupport Set the map of validation types to YAML support.
+     */
+    public void setYamlSupport(Map<String, YamlSupportEnum> yamlSupport, YamlSupportEnum defaultSupport) {
+        if (yamlSupport != null) {
+            this.yamlSupport = yamlSupport;
+        }
+        if (defaultSupport != null) {
+            this.defaultYamlSupport = defaultSupport;
+        }
+    }
+
+    /**
+     * Check whether the provided (full) validation type support's YAML.
+     *
+     * @param validationType The full validation to check its YAML support.
+     * @return The mapping of full validation type to their level of YAML support (NONE being the default).
+     */
+    public YamlSupportEnum getYamlSupportForType(String validationType) {
+        return yamlSupport.getOrDefault(validationType, defaultYamlSupport);
+    }
+
     /**
      * Check to see if there is a validation type that supports user-provided JSON schemas.
      *

jsonvalidator-common/src/main/java/eu/europa/ec/itb/json/DomainConfigCache.java

@@ -1,13 +1,15 @@
 package eu.europa.ec.itb.json;
 
+import eu.europa.ec.itb.json.validation.YamlSupportEnum;
 import eu.europa.ec.itb.validation.commons.ValidatorChannel;
+import eu.europa.ec.itb.validation.commons.config.ParseUtils;
 import eu.europa.ec.itb.validation.commons.config.WebDomainConfigCache;
+import jakarta.annotation.PostConstruct;
 import org.apache.commons.configuration2.Configuration;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import jakarta.annotation.PostConstruct;
 import java.util.Arrays;
 
 import static eu.europa.ec.itb.validation.commons.config.ParseUtils.addMissingDefaultValues;
@@ -62,6 +64,8 @@ protected void addDomainConfiguration(DomainConfig domainConfig, Configuration c
         addValidationArtifactInfo("validator.schemaFile", "validator.externalSchemas", "validator.externalSchemaCombinationApproach", domainConfig, config);
         domainConfig.getSharedSchemas().addAll(Arrays.asList(StringUtils.split(StringUtils.defaultIfBlank(config.getString("validator.referencedSchemas"), ""), ',')));
         domainConfig.setReportItemCount(config.getBoolean("validator.reportContentArrayItemCount", false));
+        var defaultYamlSupport = YamlSupportEnum.fromValue(config.getString("validator.yamlSupport", "none"));
+        domainConfig.setYamlSupport(ParseUtils.parseEnumMap("validator.yamlSupport", defaultYamlSupport, config, domainConfig.getType(), YamlSupportEnum::fromValue), defaultYamlSupport);
         addMissingDefaultValues(domainConfig.getWebServiceDescription(), appConfig.getDefaultLabels());
     }
 

jsonvalidator-common/src/main/java/eu/europa/ec/itb/json/validation/JSONValidator.java

@@ -217,6 +217,7 @@ private ValidateRequest preparePluginInput(File pluginTmpFolder) {
         request.getInput().add(Utils.createInputItem("tempFolder", pluginTmpFolder.getAbsolutePath()));
         request.getInput().add(Utils.createInputItem("locale", specs.getLocalisationHelper().getLocale().toString()));
         request.getInput().add(Utils.createInputItem("locationAsPointer", String.valueOf(specs.isLocationAsPointer())));
+        request.getInput().add(Utils.createInputItem("yaml", String.valueOf(specs.isYaml())));
         return request;
     }
 
@@ -371,12 +372,11 @@ private List<Message> validateAgainstSchema(File schemaFile) {
      */
     private JsonNode getContentNode(JsonNodeReader reader) {
         if (contentNode == null) {
+            if (reader == null) {
+                reader = JsonNodeReader.builder().build();
+            }
             try (var input = Files.newInputStream(specs.getInputFileToUse().toPath())) {
-                if (reader != null) {
-                    contentNode = reader.readTree(input, InputFormat.JSON);
-                } else {
-                    contentNode = objectMapper.readTree(input);
-                }
+                contentNode = reader.readTree(input, specs.isYaml()?InputFormat.YAML:InputFormat.JSON);
             } catch (IOException e) {
                 throw new ValidatorException("validator.label.exception.failedToParseJSON", e);
             }

jsonvalidator-common/src/main/java/eu/europa/ec/itb/json/validation/ValidationSpecs.java

@@ -1,5 +1,9 @@
 package eu.europa.ec.itb.json.validation;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonElement;
@@ -23,6 +27,11 @@
  */
 public class ValidationSpecs {
 
+    private final static ObjectReader YAML_READER;
+    private final static ObjectWriter YAML_WRITER;
+    private final static ObjectReader JSON_READER;
+    private final static ObjectWriter JSON_WRITER;
+
     private File input;
     private File inputToUse;
     private LocalisationHelper localisationHelper;
@@ -33,6 +42,17 @@ public class ValidationSpecs {
     private boolean locationAsPointer;
     private boolean addInputToReport;
     private boolean produceAggregateReport;
+    private Boolean isYamlInternal;
+
+    static {
+        // Construct immutable (thread-safe) readers and writers for JSON and YAML.
+        var jsonMapper = new ObjectMapper();
+        JSON_READER = jsonMapper.reader();
+        JSON_WRITER = jsonMapper.writer();
+        var yamlMapper = new YAMLMapper();
+        YAML_READER = yamlMapper.reader();
+        YAML_WRITER = yamlMapper.writer();
+    }
 
     /**
      * Private constructor to prevent direct initialisation.
@@ -56,15 +76,33 @@ public File getInputFileToUse() {
                 // No preprocessing needed.
                 inputToUse = prettyPrint(input);
             } else {
-                inputToUse = new File(input.getParent(), UUID.randomUUID() + ".json");
                 // A preprocessing JSONPath expression has been provided for the given validation type.
-                try (InputStream inputStream = new FileInputStream(input)) {
+                File inputToParse;
+                if (isYaml()) {
+                    // First convert the YAML input to JSON.
+                    inputToParse = new File(input.getParent(), UUID.randomUUID() + ".json");
+                    try {
+                        JSON_WRITER.writeValue(inputToParse, YAML_READER.readValue(input));
+                    } catch (Exception e) {
+                        throw new ValidatorException("validator.label.exception.errorInputForPreprocessing", e);
+                    }
+                } else {
+                    inputToParse = input;
+                }
+                inputToUse = new File(inputToParse.getParent(), UUID.randomUUID() + ".json");
+                try (InputStream inputStream = new FileInputStream(inputToParse)) {
                     Object preprocessedJsonObject = JsonPath.parse(inputStream).read(expression);
                     Gson gson = new Gson();
                     try (var writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(inputToUse.getAbsolutePath()), StandardCharsets.UTF_8))) {
                         gson.toJson(preprocessedJsonObject, writer);
                         writer.flush();
-                        inputToUse = prettyPrint(inputToUse);
+                        if (isYaml()) {
+                            // Convert the JSON to YAML.
+                            YAML_WRITER.writeValue(inputToUse, JSON_READER.readValue(inputToUse));
+                        } else {
+                            // Pretty print the JSON.
+                            inputToUse = prettyPrint(inputToUse);
+                        }
                     }
                 } catch (JsonPathException e) {
                     throw new ValidatorException("validator.label.exception.jsonPathError", e, expression);
@@ -142,21 +180,51 @@ public boolean isProduceAggregateReport() {
      * @return The pretty-printed result.
      */
     private File prettyPrint(File input) {
-        try (FileReader in = new FileReader(input)) {
-            JsonElement json = com.google.gson.JsonParser.parseReader(in);
-            Gson gson = new GsonBuilder()
-                    .setPrettyPrinting()
-                    .serializeNulls()
-                    .create();
-            String jsonOutput = gson.toJson(json);
-            File output = new File(input.getParent(), input.getName() + ".pretty");
-            FileUtils.writeStringToFile(output, jsonOutput, StandardCharsets.UTF_8);
-            return output;
-        } catch (JsonSyntaxException e) {
-            throw new ValidatorException("validator.label.exception.providedInputNotJSON", e);
-        } catch (IOException e) {
-            throw new ValidatorException("validator.label.exception.failedToParseJSON", e);
+        if (isYaml()) {
+            // No need to pretty print YAML
+            return input;
+        } else {
+            try (FileReader in = new FileReader(input)) {
+                JsonElement json = com.google.gson.JsonParser.parseReader(in);
+                Gson gson = new GsonBuilder()
+                        .setPrettyPrinting()
+                        .serializeNulls()
+                        .create();
+                String jsonOutput = gson.toJson(json);
+                File output = new File(input.getParent(), input.getName() + ".pretty");
+                FileUtils.writeStringToFile(output, jsonOutput, StandardCharsets.UTF_8);
+                return output;
+            } catch (JsonSyntaxException e) {
+                throw new ValidatorException("validator.label.exception.providedInputNotJSON", e);
+            } catch (IOException e) {
+                throw new ValidatorException("validator.label.exception.failedToParseJSON", e);
+            }
+        }
+    }
+
+    /**
+     * Check to see if the input file should be treated as YAML.
+     *
+     * @return The check result.
+     */
+    public boolean isYaml() {
+        if (isYamlInternal == null) {
+            YamlSupportEnum yamlSupport = domainConfig.getYamlSupportForType(getValidationType());
+            if (yamlSupport == YamlSupportEnum.FORCE) {
+                isYamlInternal = Boolean.TRUE;
+            } else if (yamlSupport == YamlSupportEnum.NONE) {
+                isYamlInternal = Boolean.FALSE;
+            } else {
+                // We could either have YAML or JSON - we'll check the input file's contents.
+                try (var reader = new BufferedReader(new FileReader(input))) {
+                    String firstLine = reader.readLine();
+                    isYamlInternal = firstLine != null && !(firstLine.startsWith("{") || firstLine.startsWith("["));
+                } catch (IOException e) {
+                    throw new ValidatorException("validator.label.exception.failedToParseJSON", e);
+                }
+            }
         }
+        return isYamlInternal;
     }
 
     /**

jsonvalidator-common/src/main/java/eu/europa/ec/itb/json/validation/YamlSupportEnum.java

@@ -0,0 +1,41 @@
+package eu.europa.ec.itb.json.validation;
+
+public enum YamlSupportEnum {
+
+    /** No support (only JSON is supported). */
+    NONE("none"),
+    /** Forced (only YAML is supported). */
+    FORCE("force"),
+    /** Supported (both YAML and JSON are supported). */
+    SUPPORT("support");
+
+    private final String value;
+
+    /**
+     * Constructor.
+     *
+     * @param value The enum's underlying value.
+     */
+    YamlSupportEnum(String value) {
+        this.value = value;
+    }
+
+    /**
+     * Get the enum type that corresponds to the provided value.
+     *
+     * @param value The value to process.
+     * @return The resulting enum.
+     * @throws IllegalArgumentException If the provided value is unknown.
+     */
+    public static YamlSupportEnum fromValue(String value) {
+        if (NONE.value.equals(value)) {
+            return NONE;
+        } else if (FORCE.value.equals(value)) {
+            return FORCE;
+        } else if (SUPPORT.value.equals(value)) {
+            return SUPPORT;
+        }
+        throw new IllegalArgumentException("Unknown YAML support type ["+value+"]");
+    }
+
+}