Skip to content

Commit

Permalink
Merge pull request #76 from datasonnet/qparam
Browse files Browse the repository at this point in the history 
header default input/output through quality param
  • Loading branch information
@jam01
jam01 authored Dec 2, 2020
2 parents 12270ae + 4e4e685 commit fd1273f
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 70 deletions.
116 changes: 52 additions & 64 deletions src/main/java/com/datasonnet/header/Header.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.datasonnet.document.Document;
import com.datasonnet.document.InvalidMediaTypeException;
import com.datasonnet.document.MediaType;
import com.datasonnet.document.MediaTypes;
import com.fasterxml.jackson.dataformat.javaprop.JavaPropsMapper;
import com.fasterxml.jackson.dataformat.javaprop.JavaPropsSchema;
import com.fasterxml.jackson.dataformat.javaprop.util.JPropNode;
Expand All @@ -33,22 +34,20 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class Header {
public static final String DATASONNET_HEADER = "/** DataSonnet";
public static final String COMMENT_PREFIX = "//";
public static final Pattern VERSION_LINE = Pattern.compile("^version *= *(?<version>[a-zA-Z0-9.+-]+) *(\\r?\\n|$)");
public static final String DATASONNET_DEFAULT_PREFIX = "default ";
public static final String DATASONNET_INPUT = "input";
public static final String DATASONNET_DEFAULT_INPUT = DATASONNET_DEFAULT_PREFIX + DATASONNET_INPUT;
public static final Pattern INPUT_LINE = Pattern.compile("^(?:(?<default>default )?input (?<name>\\w+)|input (?<all>\\*)) (?<mediatype>\\S.*)$");
public static final Pattern INPUT_LINE = Pattern.compile("^(?:input (?<name>\\w+)|input (?<all>\\*)) (?<mediatype>\\S.*)$");
public static final String DATASONNET_OUTPUT = "output";
public static final String DATASONNET_DEFAULT_OUTPUT = DATASONNET_DEFAULT_PREFIX + DATASONNET_OUTPUT;
public static final Pattern OUTPUT_LINE = Pattern.compile("^(?<default>default )?output (?<mediatype>\\S.*)$");
public static final Pattern OUTPUT_LINE = Pattern.compile("^output (?<mediatype>\\S.*)$");
public static final String DATASONNET_PRESERVE_ORDER = "preserveOrder";
public static final String DATAFORMAT_PREFIX = "dataformat";
public static final String DATAFORMAT_ALL = "*";
Expand All @@ -57,7 +56,7 @@ public class Header {
private final String versionMinor;
private final boolean preserveOrder;
private final Map<String, Map<Integer, MediaType>> namedInputs;
private final Map<Integer, MediaType> output;
private final Map<Integer, MediaType> outputs;
// using maps to facilitate only one per super/sub type
private final Map<Integer, MediaType> allInputs;
private final Map<Integer, MediaType> dataFormats;
Expand All @@ -67,35 +66,37 @@ public class Header {
public Header(String version,
boolean preserveOrder,
Map<String, Collection<MediaType>> namedInputs,
Map<String, MediaType> defaultInputs,
Collection<MediaType> output,
MediaType defaultOutput,
List<MediaType> outputs,
Iterable<MediaType> allInputs,
Iterable<MediaType> dataFormats) {
String[] versions = version.split("\\.",2); //[0] = major [1] = minor + remainder if exists
this.versionMajor = versions[0];
this.versionMinor = versions[1];
this.preserveOrder = preserveOrder;
this.defaultInputs = new HashMap<>(defaultInputs);
this.defaultInputs = new HashMap<>();
this.namedInputs = new HashMap<>();
for(Map.Entry<String, Collection<MediaType>> entry : namedInputs.entrySet()) {
this.namedInputs.put(entry.getKey(), indexMediaTypes(entry.getValue()));
if(!this.defaultInputs.containsKey(entry.getKey())) {
if(entry.getValue().size() == 1) {
this.defaultInputs.put(entry.getKey(), entry.getValue().iterator().next());
}

for (Map.Entry<String, Collection<MediaType>> entry : namedInputs.entrySet()) {
Collection<MediaType> types = entry.getValue();
if (types.size() > 0) {
Map<Integer, MediaType> indexed = indexMediaTypes(types);
this.namedInputs.put(entry.getKey(), indexed);

List<MediaType> sorted = new ArrayList<>(indexed.values());
MediaType.sortByQualityValue(sorted);
this.defaultInputs.put(entry.getKey(), sorted.get(0));
}
}
this.output = indexMediaTypes(output);
if(defaultOutput == null) {
if(output.size() == 1) {
this.defaultOutput = output.iterator().next();
} else {
this.defaultOutput = null;
}

this.outputs = indexMediaTypes(outputs);
if (outputs.size() > 0) {
List<MediaType> sorted = new ArrayList<>(outputs);
MediaType.sortByQualityValue(sorted);
this.defaultOutput = sorted.get(0);
} else {
this.defaultOutput = defaultOutput;
this.defaultOutput = MediaTypes.ANY;
}

this.allInputs = indexMediaTypes(allInputs);
this.dataFormats = indexMediaTypes(dataFormats);
}
Expand All @@ -112,9 +113,8 @@ private Integer calculateIndex(MediaType mediaType) {
return mediaType.getType().hashCode() + mediaType.getSubtype().hashCode();
}


private static final Header EMPTY =
new Header(LATEST_RELEASE_VERSION, true, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), null, Collections.emptyList(), Collections.emptyList());
new Header(LATEST_RELEASE_VERSION, true, Collections.emptyMap(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList());

public static Header parseHeader(String script) throws HeaderParseException {
if (!script.trim().startsWith(DATASONNET_HEADER)) {
Expand Down Expand Up @@ -218,9 +218,7 @@ public JPropPathSplitter pathSplitter() {
return new Header("1.0",
getBoolean(propsMap,DATASONNET_PRESERVE_ORDER, true),
inputs,
Collections.emptyMap(),
output,
null,
allInputs,
dataFormat);
} catch (IOException|IllegalArgumentException exc) {
Expand All @@ -242,12 +240,12 @@ private static List<MediaType> extractMediaTypes(Map<String, Map<String, String>
return types;
}

private static <T> Map<String, T> getOrEmpty(Map map, String key) {
return (Map)map.getOrDefault(key, Collections.emptyMap());
private static <T> Map<String, T> getOrEmpty(Map<String, Map<String, T>> map, String key) {
return map.getOrDefault(key, Collections.emptyMap());
}

private static boolean getBoolean(Map propsMap, String key, boolean defaultTo) {
if(propsMap.containsKey(key)) {
private static boolean getBoolean(Map<String, ?> propsMap, String key, boolean defaultTo) {
if (propsMap.containsKey(key)) {
return Boolean.parseBoolean(propsMap.get(key).toString());
} else {
return defaultTo;
Expand All @@ -257,10 +255,8 @@ private static boolean getBoolean(Map propsMap, String key, boolean defaultTo) {
@NotNull
private static Header parseHeader20(String headerSection, String version) throws HeaderParseException {
boolean preserve = true;
List<MediaType> output = new ArrayList<>(4);
MediaType defaultOutput = null;
List<MediaType> outputs = new ArrayList<>(4);
Map<String, List<MediaType>> inputs = new HashMap<>(4);
Map<String, MediaType> defaultInputs = new HashMap<>(4);
List<MediaType> allInputs = new ArrayList<>(4);
List<MediaType> dataformat = new ArrayList<>(4);

Expand All @@ -270,13 +266,14 @@ private static Header parseHeader20(String headerSection, String version) throws
if (line.startsWith(DATASONNET_PRESERVE_ORDER)) {
String[] tokens = line.split("=", 2);
preserve = Boolean.parseBoolean(tokens[1]);
} else if (line.startsWith(DATASONNET_INPUT) || line.startsWith(DATASONNET_DEFAULT_INPUT)) {
} else if (line.startsWith(DATASONNET_INPUT)) {
Matcher matcher = INPUT_LINE.matcher(line);
if(!matcher.matches()) {
throw new HeaderParseException("Unable to parse header line " + line + ", it must follow the input line format");
}
String name = matcher.group("name");
MediaType mediaType = MediaType.valueOf(matcher.group("mediatype"));

if (matcher.group("all") != null) { // there's a *. This also means it can't be a default.
allInputs.add(mediaType);
} else {
Expand All @@ -285,25 +282,16 @@ private static Header parseHeader20(String headerSection, String version) throws
}
inputs.get(name).add(mediaType);
}
if(matcher.group("default") != null) {
if(defaultInputs.containsKey(name)) {
throw new HeaderParseException("There cannot be two default media types for the input " + name);
}
defaultInputs.put(name, mediaType);
}
} else if (line.startsWith(DATASONNET_OUTPUT) || line.startsWith(DATASONNET_DEFAULT_OUTPUT)) {
} else if (line.startsWith(DATASONNET_OUTPUT)) {
Matcher matcher = OUTPUT_LINE.matcher(line);
if(!matcher.matches()) {

if (!matcher.matches()) {
throw new HeaderParseException("Unable to parse header line " + line + ", it must follow the output line format");
}

MediaType mediaType = MediaType.valueOf(matcher.group("mediatype"));
output.add(mediaType);
if(matcher.group("default") != null) {
if(defaultOutput != null) {
throw new HeaderParseException("There cannot be two default output media types");
}
defaultOutput = mediaType;
}
outputs.add(mediaType);

} else if (line.startsWith(DATAFORMAT_PREFIX)) {
String[] tokens = line.split(" ", 2);
MediaType toAdd = MediaType.valueOf(tokens[1]);
Expand All @@ -320,7 +308,7 @@ private static Header parseHeader20(String headerSection, String version) throws
}
}

return new Header(version, preserve, Collections.unmodifiableMap(inputs), Collections.emptyMap(), output, null, allInputs, dataformat);
return new Header(version, preserve, Collections.unmodifiableMap(inputs), outputs, allInputs, dataformat);
}

@NotNull
Expand Down Expand Up @@ -348,20 +336,20 @@ public Map<String, Iterable<MediaType>> getNamedInputs() {
return Collections.unmodifiableMap(namedInputs);
}

public MediaType getDefaultNamedInput(String name) {
return defaultInputs.get(name);
public Optional<MediaType> getDefaultNamedInput(String name) {
return Optional.ofNullable(defaultInputs.get(name));
}

public Collection<MediaType> getOutput() {
return Collections.unmodifiableCollection(output.values());
public Collection<MediaType> getOutputs() {
return Collections.unmodifiableCollection(outputs.values());
}

public MediaType getDefaultOutput() {
return defaultOutput;
public Optional<MediaType> getDefaultOutput() {
return Optional.ofNullable(defaultOutput);
}

public Collection<MediaType> getPayload() {
return Collections.unmodifiableCollection(namedInputs.getOrDefault("payload", Collections.emptyMap()).values());
public Optional<MediaType> getDefaultPayload() {
return Optional.ofNullable(defaultInputs.get("payload"));
}

public Collection<MediaType> getAllInputs() {
Expand Down Expand Up @@ -390,8 +378,8 @@ public <T> Document<T> combineInputParams(String inputName, Document<T> doc) {
}

if (namedInputs.containsKey(inputName)) {
Map<Integer, MediaType> inputTypes = namedInputs.getOrDefault(inputName, Collections.emptyMap());
if(inputTypes.containsKey(key)) {
Map<Integer, MediaType> inputTypes = namedInputs.get(inputName);
if (inputTypes != null && inputTypes.containsKey(key)) {
params.putAll(inputTypes.get(key).getParameters());
}
}
Expand All @@ -414,8 +402,8 @@ public MediaType combineOutputParams(MediaType mediaType) {
params.putAll(dataFormats.get(key).getParameters());
}

if (output.containsKey(key)) {
params.putAll(output.get(key).getParameters());
if (outputs.containsKey(key)) {
params.putAll(outputs.get(key).getParameters());
}

params.putAll(mediaType.getParameters());
Expand Down
4 changes: 2 additions & 2 deletions src/main/scala/com/datasonnet/Mapper.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ class Mapper(var script: String,
private def effectiveOutput(output: MediaType): MediaType = {
if (output.equalsTypeAndSubtype(MediaTypes.ANY)) {
val fromHeader = header.getDefaultOutput
if (fromHeader != null && !fromHeader.equalsTypeAndSubtype(MediaTypes.ANY)) header.combineOutputParams(fromHeader)
if (fromHeader.isPresent && !fromHeader.get.equalsTypeAndSubtype(MediaTypes.ANY)) header.combineOutputParams(fromHeader.get)
else header.combineOutputParams(defaultOutput)
} else {
header.combineOutputParams(output)
Expand All @@ -194,7 +194,7 @@ class Mapper(var script: String,
private def effectiveInput[T](name: String, input: Document[T]): Document[T] = {
if (input.getMediaType.equalsTypeAndSubtype(MediaTypes.UNKNOWN)){
val fromHeader = header.getDefaultNamedInput(name)
if (fromHeader != null) header.combineInputParams(name, input.withMediaType(fromHeader))
if (fromHeader.isPresent) header.combineInputParams(name, input.withMediaType(fromHeader.get))
else header.combineInputParams(name, input.withMediaType(MediaTypes.APPLICATION_JAVA))
} else {
header.combineInputParams(name, input)
Expand Down
33 changes: 29 additions & 4 deletions src/test/java/com/datasonnet/HeaderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,19 @@
* limitations under the License.
*/

import com.datasonnet.document.MediaTypes;
import com.datasonnet.header.Header;
import org.junit.jupiter.api.BeforeAll;
import com.datasonnet.header.HeaderParseException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;

import java.util.Map;
import java.util.Set;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;


@TestInstance(TestInstance.Lifecycle.PER_CLASS)
Expand Down Expand Up @@ -107,7 +108,7 @@ void testHeaderNamedInputs() {

@Test
void testHeaderNamedInputCommaSeparated() {
Map<String, String> parameters = header.getDefaultNamedInput("payload").getParameters();
Map<String, String> parameters = header.getDefaultNamedInput("payload").orElseThrow(AssertionError::new).getParameters();
assertTrue(parameters.containsKey("namespace-separator"));
assertTrue(parameters.containsKey("text-value-key"));
}
Expand All @@ -120,7 +121,7 @@ void testHeaderDataformat() {

@Test
void testHeaderOutput() {
Set<String> keys = header.getDefaultOutput().getParameters().keySet();
Set<String> keys = header.getDefaultOutput().orElseThrow(AssertionError::new).getParameters().keySet();
assertTrue(keys.contains("ds.csv.quote"));
}

Expand All @@ -141,4 +142,28 @@ void testUnterminatedHeaderFailsNicely() {
"version=2.0\n");
});
}

@Test
public void testDefaultOutput() throws HeaderParseException {
Header header1 = Header.parseHeader("/** DataSonnet\n" +
"version=2.0\n" +
"output application/x-java-object;q=0.9\n" +
"output application/json;q=1.0\n" +
"*/");

assertTrue(header1.getDefaultOutput().isPresent());
assertTrue(MediaTypes.APPLICATION_JSON.equalsTypeAndSubtype(header1.getDefaultOutput().get()));
}

@Test
public void testDefaultInput() throws HeaderParseException {
Header header1 = Header.parseHeader("/** DataSonnet\n" +
"version=2.0\n" +
"input payload application/x-java-object;q=1.0\n" +
"input payload application/json;q=0.9\n" +
"*/");

assertTrue(header1.getDefaultPayload().isPresent());
assertTrue(MediaTypes.APPLICATION_JAVA.equalsTypeAndSubtype(header1.getDefaultPayload().get()));
}
}

0 comments on commit fd1273f

Please sign in to comment.