From 1e71543fda27c6fcf54ac6af98e49d635e3633c5 Mon Sep 17 00:00:00 2001 From: Jose Montoya Date: Tue, 1 Dec 2020 17:29:55 -0600 Subject: [PATCH 1/2] header default input/output through quality param removes the explicit default input and output header lines to instead rely on quality media type params --- .../java/com/datasonnet/header/Header.java | 108 ++++++++---------- src/main/scala/com/datasonnet/Mapper.scala | 4 +- src/test/java/com/datasonnet/HeaderTest.java | 33 +++++- 3 files changed, 81 insertions(+), 64 deletions(-) diff --git a/src/main/java/com/datasonnet/header/Header.java b/src/main/java/com/datasonnet/header/Header.java index d077518a..8a6209dc 100644 --- a/src/main/java/com/datasonnet/header/Header.java +++ b/src/main/java/com/datasonnet/header/Header.java @@ -19,6 +19,7 @@ import com.datasonnet.document.Document; import com.datasonnet.document.InvalidMediaTypeException; import com.datasonnet.document.MediaType; +import com.datasonnet.document.MediaTypes; import com.fasterxml.jackson.dataformat.javaprop.JavaPropsMapper; import com.fasterxml.jackson.dataformat.javaprop.JavaPropsSchema; import com.fasterxml.jackson.dataformat.javaprop.util.JPropNode; @@ -33,22 +34,20 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; - public class Header { public static final String DATASONNET_HEADER = "/** DataSonnet"; public static final String COMMENT_PREFIX = "//"; public static final Pattern VERSION_LINE = Pattern.compile("^version *= *(?[a-zA-Z0-9.+-]+) *(\\r?\\n|$)"); public static final String DATASONNET_DEFAULT_PREFIX = "default "; public static final String DATASONNET_INPUT = "input"; - public static final String DATASONNET_DEFAULT_INPUT = DATASONNET_DEFAULT_PREFIX + DATASONNET_INPUT; - public static final Pattern INPUT_LINE = Pattern.compile("^(?:(?default )?input (?\\w+)|input (?\\*)) (?\\S.*)$"); + public static final Pattern INPUT_LINE = Pattern.compile("^(?:input (?\\w+)|input (?\\*)) (?\\S.*)$"); public static final String DATASONNET_OUTPUT = "output"; - public static final String DATASONNET_DEFAULT_OUTPUT = DATASONNET_DEFAULT_PREFIX + DATASONNET_OUTPUT; - public static final Pattern OUTPUT_LINE = Pattern.compile("^(?default )?output (?\\S.*)$"); + public static final Pattern OUTPUT_LINE = Pattern.compile("^output (?\\S.*)$"); public static final String DATASONNET_PRESERVE_ORDER = "preserveOrder"; public static final String DATAFORMAT_PREFIX = "dataformat"; public static final String DATAFORMAT_ALL = "*"; @@ -57,7 +56,7 @@ public class Header { private final String versionMinor; private final boolean preserveOrder; private final Map> namedInputs; - private final Map output; + private final Map outputs; // using maps to facilitate only one per super/sub type private final Map allInputs; private final Map dataFormats; @@ -68,7 +67,7 @@ public Header(String version, boolean preserveOrder, Map> namedInputs, Map defaultInputs, - Collection output, + List outputs, MediaType defaultOutput, Iterable allInputs, Iterable dataFormats) { @@ -78,24 +77,28 @@ public Header(String version, this.preserveOrder = preserveOrder; this.defaultInputs = new HashMap<>(defaultInputs); this.namedInputs = new HashMap<>(); - for(Map.Entry> entry : namedInputs.entrySet()) { - this.namedInputs.put(entry.getKey(), indexMediaTypes(entry.getValue())); - if(!this.defaultInputs.containsKey(entry.getKey())) { - if(entry.getValue().size() == 1) { - this.defaultInputs.put(entry.getKey(), entry.getValue().iterator().next()); - } + + for (Map.Entry> entry : namedInputs.entrySet()) { + Collection types = entry.getValue(); + if (types.size() > 0) { + Map indexed = indexMediaTypes(types); + this.namedInputs.put(entry.getKey(), indexed); + + List sorted = new ArrayList<>(indexed.values()); + MediaType.sortByQualityValue(sorted); + this.defaultInputs.put(entry.getKey(), sorted.get(0)); } } - this.output = indexMediaTypes(output); - if(defaultOutput == null) { - if(output.size() == 1) { - this.defaultOutput = output.iterator().next(); - } else { - this.defaultOutput = null; - } + + this.outputs = indexMediaTypes(outputs); + if (defaultOutput == null && outputs.size() > 0) { + List sorted = new ArrayList<>(outputs); + MediaType.sortByQualityValue(sorted); + this.defaultOutput = sorted.get(0); } else { this.defaultOutput = defaultOutput; } + this.allInputs = indexMediaTypes(allInputs); this.dataFormats = indexMediaTypes(dataFormats); } @@ -112,9 +115,8 @@ private Integer calculateIndex(MediaType mediaType) { return mediaType.getType().hashCode() + mediaType.getSubtype().hashCode(); } - private static final Header EMPTY = - new Header(LATEST_RELEASE_VERSION, true, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), null, Collections.emptyList(), Collections.emptyList()); + new Header(LATEST_RELEASE_VERSION, true, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), MediaTypes.ANY, Collections.emptyList(), Collections.emptyList()); public static Header parseHeader(String script) throws HeaderParseException { if (!script.trim().startsWith(DATASONNET_HEADER)) { @@ -242,12 +244,12 @@ private static List extractMediaTypes(Map return types; } - private static Map getOrEmpty(Map map, String key) { - return (Map)map.getOrDefault(key, Collections.emptyMap()); + private static Map getOrEmpty(Map> map, String key) { + return map.getOrDefault(key, Collections.emptyMap()); } - private static boolean getBoolean(Map propsMap, String key, boolean defaultTo) { - if(propsMap.containsKey(key)) { + private static boolean getBoolean(Map propsMap, String key, boolean defaultTo) { + if (propsMap.containsKey(key)) { return Boolean.parseBoolean(propsMap.get(key).toString()); } else { return defaultTo; @@ -257,10 +259,8 @@ private static boolean getBoolean(Map propsMap, String key, boolean defaultTo) { @NotNull private static Header parseHeader20(String headerSection, String version) throws HeaderParseException { boolean preserve = true; - List output = new ArrayList<>(4); - MediaType defaultOutput = null; + List outputs = new ArrayList<>(4); Map> inputs = new HashMap<>(4); - Map defaultInputs = new HashMap<>(4); List allInputs = new ArrayList<>(4); List dataformat = new ArrayList<>(4); @@ -270,13 +270,14 @@ private static Header parseHeader20(String headerSection, String version) throws if (line.startsWith(DATASONNET_PRESERVE_ORDER)) { String[] tokens = line.split("=", 2); preserve = Boolean.parseBoolean(tokens[1]); - } else if (line.startsWith(DATASONNET_INPUT) || line.startsWith(DATASONNET_DEFAULT_INPUT)) { + } else if (line.startsWith(DATASONNET_INPUT)) { Matcher matcher = INPUT_LINE.matcher(line); if(!matcher.matches()) { throw new HeaderParseException("Unable to parse header line " + line + ", it must follow the input line format"); } String name = matcher.group("name"); MediaType mediaType = MediaType.valueOf(matcher.group("mediatype")); + if (matcher.group("all") != null) { // there's a *. This also means it can't be a default. allInputs.add(mediaType); } else { @@ -285,25 +286,16 @@ private static Header parseHeader20(String headerSection, String version) throws } inputs.get(name).add(mediaType); } - if(matcher.group("default") != null) { - if(defaultInputs.containsKey(name)) { - throw new HeaderParseException("There cannot be two default media types for the input " + name); - } - defaultInputs.put(name, mediaType); - } - } else if (line.startsWith(DATASONNET_OUTPUT) || line.startsWith(DATASONNET_DEFAULT_OUTPUT)) { + } else if (line.startsWith(DATASONNET_OUTPUT)) { Matcher matcher = OUTPUT_LINE.matcher(line); - if(!matcher.matches()) { + + if (!matcher.matches()) { throw new HeaderParseException("Unable to parse header line " + line + ", it must follow the output line format"); } + MediaType mediaType = MediaType.valueOf(matcher.group("mediatype")); - output.add(mediaType); - if(matcher.group("default") != null) { - if(defaultOutput != null) { - throw new HeaderParseException("There cannot be two default output media types"); - } - defaultOutput = mediaType; - } + outputs.add(mediaType); + } else if (line.startsWith(DATAFORMAT_PREFIX)) { String[] tokens = line.split(" ", 2); MediaType toAdd = MediaType.valueOf(tokens[1]); @@ -320,7 +312,7 @@ private static Header parseHeader20(String headerSection, String version) throws } } - return new Header(version, preserve, Collections.unmodifiableMap(inputs), Collections.emptyMap(), output, null, allInputs, dataformat); + return new Header(version, preserve, Collections.unmodifiableMap(inputs), Collections.emptyMap(), outputs, null, allInputs, dataformat); } @NotNull @@ -348,20 +340,20 @@ public Map> getNamedInputs() { return Collections.unmodifiableMap(namedInputs); } - public MediaType getDefaultNamedInput(String name) { - return defaultInputs.get(name); + public Optional getDefaultNamedInput(String name) { + return Optional.ofNullable(defaultInputs.get(name)); } - public Collection getOutput() { - return Collections.unmodifiableCollection(output.values()); + public Collection getOutputs() { + return Collections.unmodifiableCollection(outputs.values()); } - public MediaType getDefaultOutput() { - return defaultOutput; + public Optional getDefaultOutput() { + return Optional.ofNullable(defaultOutput); } - public Collection getPayload() { - return Collections.unmodifiableCollection(namedInputs.getOrDefault("payload", Collections.emptyMap()).values()); + public Optional getDefaultPayload() { + return Optional.ofNullable(defaultInputs.get("payload")); } public Collection getAllInputs() { @@ -390,8 +382,8 @@ public Document combineInputParams(String inputName, Document doc) { } if (namedInputs.containsKey(inputName)) { - Map inputTypes = namedInputs.getOrDefault(inputName, Collections.emptyMap()); - if(inputTypes.containsKey(key)) { + Map inputTypes = namedInputs.get(inputName); + if (inputTypes != null && inputTypes.containsKey(key)) { params.putAll(inputTypes.get(key).getParameters()); } } @@ -414,8 +406,8 @@ public MediaType combineOutputParams(MediaType mediaType) { params.putAll(dataFormats.get(key).getParameters()); } - if (output.containsKey(key)) { - params.putAll(output.get(key).getParameters()); + if (outputs.containsKey(key)) { + params.putAll(outputs.get(key).getParameters()); } params.putAll(mediaType.getParameters()); diff --git a/src/main/scala/com/datasonnet/Mapper.scala b/src/main/scala/com/datasonnet/Mapper.scala index 0de8ec00..571538f3 100644 --- a/src/main/scala/com/datasonnet/Mapper.scala +++ b/src/main/scala/com/datasonnet/Mapper.scala @@ -183,7 +183,7 @@ class Mapper(var script: String, private def effectiveOutput(output: MediaType): MediaType = { if (output.equalsTypeAndSubtype(MediaTypes.ANY)) { val fromHeader = header.getDefaultOutput - if (fromHeader != null && !fromHeader.equalsTypeAndSubtype(MediaTypes.ANY)) header.combineOutputParams(fromHeader) + if (fromHeader.isPresent && !fromHeader.get.equalsTypeAndSubtype(MediaTypes.ANY)) header.combineOutputParams(fromHeader.get) else header.combineOutputParams(defaultOutput) } else { header.combineOutputParams(output) @@ -194,7 +194,7 @@ class Mapper(var script: String, private def effectiveInput[T](name: String, input: Document[T]): Document[T] = { if (input.getMediaType.equalsTypeAndSubtype(MediaTypes.UNKNOWN)){ val fromHeader = header.getDefaultNamedInput(name) - if (fromHeader != null) header.combineInputParams(name, input.withMediaType(fromHeader)) + if (fromHeader.isPresent) header.combineInputParams(name, input.withMediaType(fromHeader.get)) else header.combineInputParams(name, input.withMediaType(MediaTypes.APPLICATION_JAVA)) } else { header.combineInputParams(name, input) diff --git a/src/test/java/com/datasonnet/HeaderTest.java b/src/test/java/com/datasonnet/HeaderTest.java index 1abb6f5b..0a3147cd 100644 --- a/src/test/java/com/datasonnet/HeaderTest.java +++ b/src/test/java/com/datasonnet/HeaderTest.java @@ -16,9 +16,10 @@ * limitations under the License. */ +import com.datasonnet.document.MediaTypes; import com.datasonnet.header.Header; -import org.junit.jupiter.api.BeforeAll; import com.datasonnet.header.HeaderParseException; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -26,8 +27,8 @@ import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -107,7 +108,7 @@ void testHeaderNamedInputs() { @Test void testHeaderNamedInputCommaSeparated() { - Map parameters = header.getDefaultNamedInput("payload").getParameters(); + Map parameters = header.getDefaultNamedInput("payload").orElseThrow(AssertionError::new).getParameters(); assertTrue(parameters.containsKey("namespace-separator")); assertTrue(parameters.containsKey("text-value-key")); } @@ -120,7 +121,7 @@ void testHeaderDataformat() { @Test void testHeaderOutput() { - Set keys = header.getDefaultOutput().getParameters().keySet(); + Set keys = header.getDefaultOutput().orElseThrow(AssertionError::new).getParameters().keySet(); assertTrue(keys.contains("ds.csv.quote")); } @@ -141,4 +142,28 @@ void testUnterminatedHeaderFailsNicely() { "version=2.0\n"); }); } + + @Test + public void test_default_output() throws HeaderParseException { + Header header1 = Header.parseHeader("/** DataSonnet\n" + + "version=2.0\n" + + "output application/x-java-object;q=0.9\n" + + "output application/json;q=1.0\n" + + "*/"); + + assertTrue(header1.getDefaultOutput().isPresent()); + assertTrue(MediaTypes.APPLICATION_JSON.equalsTypeAndSubtype(header1.getDefaultOutput().get())); + } + + @Test + public void test_default_input() throws HeaderParseException { + Header header1 = Header.parseHeader("/** DataSonnet\n" + + "version=2.0\n" + + "input payload application/json;q=0.9\n" + + "input payload application/x-java-object;q=1.0\n" + + "*/"); + + assertTrue(header1.getDefaultPayload().isPresent()); + assertTrue(MediaTypes.APPLICATION_JAVA.equalsTypeAndSubtype(header1.getDefaultPayload().get())); + } } From 4e4e685661ed45a593f7300cf7a31f1a627d8209 Mon Sep 17 00:00:00 2001 From: Jose Montoya Date: Tue, 1 Dec 2020 18:12:34 -0600 Subject: [PATCH 2/2] remove default input/output from header constructor --- src/main/java/com/datasonnet/header/Header.java | 14 +++++--------- src/test/java/com/datasonnet/HeaderTest.java | 6 +++--- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/datasonnet/header/Header.java b/src/main/java/com/datasonnet/header/Header.java index 8a6209dc..084a0067 100644 --- a/src/main/java/com/datasonnet/header/Header.java +++ b/src/main/java/com/datasonnet/header/Header.java @@ -66,16 +66,14 @@ public class Header { public Header(String version, boolean preserveOrder, Map> namedInputs, - Map defaultInputs, List outputs, - MediaType defaultOutput, Iterable allInputs, Iterable dataFormats) { String[] versions = version.split("\\.",2); //[0] = major [1] = minor + remainder if exists this.versionMajor = versions[0]; this.versionMinor = versions[1]; this.preserveOrder = preserveOrder; - this.defaultInputs = new HashMap<>(defaultInputs); + this.defaultInputs = new HashMap<>(); this.namedInputs = new HashMap<>(); for (Map.Entry> entry : namedInputs.entrySet()) { @@ -91,12 +89,12 @@ public Header(String version, } this.outputs = indexMediaTypes(outputs); - if (defaultOutput == null && outputs.size() > 0) { + if (outputs.size() > 0) { List sorted = new ArrayList<>(outputs); MediaType.sortByQualityValue(sorted); this.defaultOutput = sorted.get(0); } else { - this.defaultOutput = defaultOutput; + this.defaultOutput = MediaTypes.ANY; } this.allInputs = indexMediaTypes(allInputs); @@ -116,7 +114,7 @@ private Integer calculateIndex(MediaType mediaType) { } private static final Header EMPTY = - new Header(LATEST_RELEASE_VERSION, true, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), MediaTypes.ANY, Collections.emptyList(), Collections.emptyList()); + new Header(LATEST_RELEASE_VERSION, true, Collections.emptyMap(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); public static Header parseHeader(String script) throws HeaderParseException { if (!script.trim().startsWith(DATASONNET_HEADER)) { @@ -220,9 +218,7 @@ public JPropPathSplitter pathSplitter() { return new Header("1.0", getBoolean(propsMap,DATASONNET_PRESERVE_ORDER, true), inputs, - Collections.emptyMap(), output, - null, allInputs, dataFormat); } catch (IOException|IllegalArgumentException exc) { @@ -312,7 +308,7 @@ private static Header parseHeader20(String headerSection, String version) throws } } - return new Header(version, preserve, Collections.unmodifiableMap(inputs), Collections.emptyMap(), outputs, null, allInputs, dataformat); + return new Header(version, preserve, Collections.unmodifiableMap(inputs), outputs, allInputs, dataformat); } @NotNull diff --git a/src/test/java/com/datasonnet/HeaderTest.java b/src/test/java/com/datasonnet/HeaderTest.java index 0a3147cd..7d90caf3 100644 --- a/src/test/java/com/datasonnet/HeaderTest.java +++ b/src/test/java/com/datasonnet/HeaderTest.java @@ -144,7 +144,7 @@ void testUnterminatedHeaderFailsNicely() { } @Test - public void test_default_output() throws HeaderParseException { + public void testDefaultOutput() throws HeaderParseException { Header header1 = Header.parseHeader("/** DataSonnet\n" + "version=2.0\n" + "output application/x-java-object;q=0.9\n" + @@ -156,11 +156,11 @@ public void test_default_output() throws HeaderParseException { } @Test - public void test_default_input() throws HeaderParseException { + public void testDefaultInput() throws HeaderParseException { Header header1 = Header.parseHeader("/** DataSonnet\n" + "version=2.0\n" + - "input payload application/json;q=0.9\n" + "input payload application/x-java-object;q=1.0\n" + + "input payload application/json;q=0.9\n" + "*/"); assertTrue(header1.getDefaultPayload().isPresent());