From 449fbba2b16e78f1e647a8cf403602ba0dfad36d Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Mon, 6 Dec 2021 10:10:55 +0100 Subject: [PATCH 1/2] Move maps test files to resources directory. (283075c) --- .../test/java/org/metafacture/metafix/MetafixLookupTest.java | 5 +++-- .../org/metafacture/metafix/maps/test.csv | 0 .../org/metafacture/metafix/maps/test.tsv | 0 3 files changed, 3 insertions(+), 2 deletions(-) rename metafix/src/test/{java => resources}/org/metafacture/metafix/maps/test.csv (100%) rename metafix/src/test/{java => resources}/org/metafacture/metafix/maps/test.tsv (100%) diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java index 25f9ad5d..132ce77a 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java @@ -127,7 +127,7 @@ public void inlineDotNotationNested() { @Test public void csv() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', 'src/test/java/org/metafacture/metafix/maps/test.csv')"), + "lookup('title', 'src/test/resources/org/metafacture/metafix/maps/test.csv')"), i -> { i.startRecord("1"); i.endRecord(); @@ -161,7 +161,7 @@ public void csv() { @Test public void tsv() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', 'src/test/java/org/metafacture/metafix/maps/test.tsv', sep_char:'\t')"), + "lookup('title', 'src/test/resources/org/metafacture/metafix/maps/test.tsv', sep_char:'\t')"), i -> { i.startRecord("1"); i.endRecord(); @@ -190,4 +190,5 @@ public void tsv() { o.get().endRecord(); }); } + } diff --git a/metafix/src/test/java/org/metafacture/metafix/maps/test.csv b/metafix/src/test/resources/org/metafacture/metafix/maps/test.csv similarity index 100% rename from metafix/src/test/java/org/metafacture/metafix/maps/test.csv rename to metafix/src/test/resources/org/metafacture/metafix/maps/test.csv diff --git a/metafix/src/test/java/org/metafacture/metafix/maps/test.tsv b/metafix/src/test/resources/org/metafacture/metafix/maps/test.tsv similarity index 100% rename from metafix/src/test/java/org/metafacture/metafix/maps/test.tsv rename to metafix/src/test/resources/org/metafacture/metafix/maps/test.tsv From a5039f18745f1dbfd2a031a701d9f301ca4a2fe9 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Fri, 3 Dec 2021 17:02:38 +0100 Subject: [PATCH 2/2] Extend Maps support. (#85) --- .../org/metafacture/metafix/FixMethod.java | 99 +++++--- .../java/org/metafacture/metafix/Metafix.java | 65 ++++- .../metafix/RecordTransformer.java | 18 +- .../metafix/MetafixLookupTest.java | 237 ++++++++++++------ .../metafix/MetafixScriptTest.java | 111 ++++++++ .../org/metafacture/metafix/MetafixTest.java | 127 ++++++++++ 6 files changed, 521 insertions(+), 136 deletions(-) create mode 100644 metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java create mode 100644 metafix/src/test/java/org/metafacture/metafix/MetafixTest.java diff --git a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java index 4f7e964c..8e6113e4 100644 --- a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -16,6 +16,7 @@ package org.metafacture.metafix; +import org.metafacture.metamorph.api.Maps; import org.metafacture.metamorph.maps.FileMap; import java.util.Arrays; @@ -27,10 +28,29 @@ enum FixMethod { + // SCRIPT-LEVEL METHODS: + + put_map { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + metafix.putMap(params.get(0), options); + } + }, + put_filemap { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String fileName = params.get(0); + final FileMap fileMap = new FileMap(); + + fileMap.setSeparator(options.getOrDefault(FILEMAP_SEPARATOR_OPTION, FILEMAP_DEFAULT_SEPARATOR)); + fileMap.setFile(fileName); + + metafix.putMap(params.size() <= 1 ? fileName : params.get(1), fileMap); + } + }, + // RECORD-LEVEL METHODS: set_field { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final String field = params.get(0); record.remove(field); @@ -38,7 +58,7 @@ public void apply(final Record record, final List params, final Map params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final String field = params.get(0); final List toAdd = params.subList(1, params.size()); if (field.endsWith(DOT_APPEND)) { @@ -50,7 +70,7 @@ public void apply(final Record record, final List params, final Map params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final String field = params.get(0); final Value value = record.get(field.replace(DOT_APPEND, EMPTY)); @@ -65,7 +85,7 @@ public void apply(final Record record, final List params, final Map params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final String field = params.get(0); record.getList(field, a -> a.forEach(recordEntry -> { @@ -81,7 +101,7 @@ public void apply(final Record record, final List params, final Map params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final String field = params.get(0); record.getList(field, a -> record.put(field, Value.newHash(h -> { @@ -92,28 +112,28 @@ public void apply(final Record record, final List params, final Map params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.append(params.get(0), params.get(1)); } }, move_field { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.copy(params); record.removeNested(params.get(0)); } }, copy_field { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.copy(params); } }, remove_field { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { params.forEach(record::removeNested); } }, format { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final String field = params.get(0); record.getList(field, oldValues -> { @@ -123,7 +143,7 @@ public void apply(final Record record, final List params, final Map params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final String field = params.get(0); record.getList(field, a -> a.forEach(v -> { @@ -160,7 +180,7 @@ public void apply(final Record record, final List params, final Map params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final String joinChar = options.get("join_char"); record.replace(params.get(0), params.subList(1, params.size()).stream() .filter(f -> literalString(f) || record.find(f) != null) @@ -173,71 +193,69 @@ private boolean literalString(final String s) { } }, reject { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.setReject(true); } }, retain { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.retainFields(params); } }, vacuum { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.removeEmptyValues(); } }, + // FIELD-LEVEL METHODS: // TODO SPEC: switch to morph-style named params in general? substring { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.transformFields(params, s -> s.substring(Integer.parseInt(params.get(1)), Integer.parseInt(params.get(2)) - 1)); } }, trim { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.transformFields(params, String::trim); } }, upcase { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.transformFields(params, String::toUpperCase); } }, downcase { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.transformFields(params, String::toLowerCase); } }, capitalize { - public void apply(final Record record, final List params, final Map options) { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.transformFields(params, s -> s.substring(0, 1).toUpperCase() + s.substring(1)); } }, lookup { - public void apply(final Record record, final List params, final Map options) { - record.transformFields(params, s -> { - final Map map = buildMap(options, params.size() <= 1 ? null : params.get(1)); - return map.getOrDefault(s, map.get("__default")); // TODO Catmandu uses 'default' - }); - } + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final Map map; - private Map buildMap(final Map options, final String fileLocation) { - final String sep = "sep_char"; - final Map map = fileLocation != null ? fileMap(fileLocation, options.get(sep)) : options; - return map; - } + if (params.size() <= 1) { + map = options; + } + else { + final String mapName = params.get(1); - private Map fileMap(final String location, final String separator) { - final FileMap fileMap = new FileMap(); - fileMap.setSeparator(","); // CSV as default - if (separator != null) { // override with option - fileMap.setSeparator(separator); + if (!metafix.getMapNames().contains(mapName)) { + put_filemap.apply(metafix, record, Arrays.asList(mapName), options); + } + + map = metafix.getMap(mapName); } - fileMap.setFile(location); - return fileMap; + + final String defaultValue = map.get(Maps.DEFAULT_MAP_KEY); // TODO: Catmandu uses 'default' + record.transformFields(params, k -> map.getOrDefault(k, defaultValue)); } }; @@ -246,6 +264,9 @@ private Map fileMap(final String location, final String separato private static final String EMPTY = ""; private static final String DOT_APPEND = "." + Value.APPEND_FIELD; - abstract void apply(Record record, List params, Map options); + private static final String FILEMAP_SEPARATOR_OPTION = "sep_char"; + private static final String FILEMAP_DEFAULT_SEPARATOR = ","; + + abstract void apply(Metafix metafix, Record record, List params, Map options); } diff --git a/metafix/src/main/java/org/metafacture/metafix/Metafix.java b/metafix/src/main/java/org/metafacture/metafix/Metafix.java index 315ce701..09ac913a 100644 --- a/metafix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/metafix/src/main/java/org/metafacture/metafix/Metafix.java @@ -18,6 +18,7 @@ package org.metafacture.metafix; +import org.metafacture.framework.MetafactureException; import org.metafacture.framework.StandardEventNames; import org.metafacture.framework.StreamPipe; import org.metafacture.framework.StreamReceiver; @@ -25,17 +26,22 @@ import org.metafacture.mangling.StreamFlattener; import org.metafacture.metafix.fix.Expression; import org.metafacture.metafix.fix.Fix; +import org.metafacture.metamorph.api.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Closeable; import java.io.FileNotFoundException; import java.io.FileReader; +import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Deque; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -48,22 +54,26 @@ * @author Christoph Böhme (Metamorph) * @author Fabian Steeg (Metafix) */ -public class Metafix implements StreamPipe { +public class Metafix implements StreamPipe, Maps { // checkstyle-disable-line ClassDataAbstractionCoupling public static final String VAR_START = "$["; public static final String VAR_END = "]"; public static final Map NO_VARS = Collections.emptyMap(); - private static final String ENTITIES_NOT_BALANCED = "Entity starts and ends are not balanced"; private static final Logger LOG = LoggerFactory.getLogger(Metafix.class); + private static final String ENTITIES_NOT_BALANCED = "Entity starts and ends are not balanced"; + + private final Deque entityCountStack = new LinkedList<>(); + private final List resources = new ArrayList<>(); + private final List expressions = new ArrayList<>(); + private final Map> maps = new HashMap<>(); + private final StreamFlattener flattener = new StreamFlattener(); + // TODO: Use SimpleRegexTrie / WildcardTrie for wildcard, alternation and character class support private Record currentRecord = new Record(); private Fix fix; - private final List expressions = new ArrayList<>(); private Map vars = NO_VARS; - private final StreamFlattener flattener = new StreamFlattener(); - private final Deque entityCountStack = new LinkedList<>(); private int entityCount; private StreamReceiver outputStreamReceiver; private String recordIdentifier; @@ -105,8 +115,7 @@ public void literal(final String name, final String value) { } private void buildPipeline(final Reader fixDef, final Map theVars) { - final Fix f = FixStandaloneSetup.parseFix(fixDef); - this.fix = f; + this.fix = FixStandaloneSetup.parseFix(fixDef); this.vars = theVars; } @@ -131,8 +140,7 @@ public void endRecord() { } flattener.endRecord(); LOG.debug("End record, walking fix: {}", currentRecord); - final RecordTransformer transformer = new RecordTransformer(currentRecord, vars, fix); - currentRecord = transformer.transform(); + currentRecord = new RecordTransformer(this, fix).transform(); if (!currentRecord.getReject()) { outputStreamReceiver.startRecord(recordIdentifier); LOG.debug("Sending results to {}", outputStreamReceiver); @@ -207,6 +215,15 @@ public void resetStream() { @Override public void closeStream() { + for (final Closeable closeable : resources) { + try { + closeable.close(); + } + catch (final IOException e) { + throw new MetafactureException("Error while executing the Metafix transformation pipeline: " + e.getMessage(), e); + } + } + outputStreamReceiver.closeStream(); } @@ -234,4 +251,34 @@ public Record getCurrentRecord() { return currentRecord; } + @Override + public Collection getMapNames() { + return Collections.unmodifiableSet(maps.keySet()); + } + + @Override + public Map getMap(final String mapName) { + return maps.getOrDefault(mapName, Collections.emptyMap()); + } + + @Override + public String getValue(final String mapName, final String key) { + final Map map = getMap(mapName); + return map.containsKey(key) ? map.get(key) : map.get(Maps.DEFAULT_MAP_KEY); + } + + @Override + public Map putMap(final String mapName, final Map map) { + if (map instanceof Closeable) { + resources.add((Closeable) map); + } + + return maps.put(mapName, map); + } + + @Override + public String putValue(final String mapName, final String key, final String value) { + return maps.computeIfAbsent(mapName, k -> new HashMap<>()).put(key, value); + } + } diff --git a/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java b/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java index f49162ed..7bfb9c70 100644 --- a/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java +++ b/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -39,7 +39,7 @@ import java.util.stream.Collectors; /** - * Transform a record using a Fix + * Transform a record using a {@link Fix}. * * @author Fabian Steeg * @@ -48,14 +48,16 @@ class RecordTransformer { private static final Logger LOG = LoggerFactory.getLogger(RecordTransformer.class); - private Fix fix; + private final Metafix metafix; + private final Fix fix; + private Record record; - private Map vars; - RecordTransformer(final Record record, final Map vars, final Fix fix) { - this.record = record.shallowClone(); - this.vars = vars; + RecordTransformer(final Metafix metafix, final Fix fix) { + this.metafix = metafix; this.fix = fix; + + record = metafix.getCurrentRecord().shallowClone(); } Record transform() { @@ -68,7 +70,7 @@ Record getRecord() { } private String resolveVars(final String string) { - return string == null ? null : StringUtil.format(string, Metafix.VAR_START, Metafix.VAR_END, false, vars); + return string == null ? null : StringUtil.format(string, Metafix.VAR_START, Metafix.VAR_END, false, metafix.getVars()); } private void processSubexpressions(final List expressions) { @@ -181,7 +183,7 @@ private void processFunction(final Expression expression, final List par final FixMethod method = FixMethod.valueOf(expression.getName()); final List resolvedParams = params.stream().map(this::resolveVars).collect(Collectors.toList()); final Map options = options(((MethodCall) expression).getOptions()); - method.apply(record, resolvedParams, options); + method.apply(metafix, record, resolvedParams, options); } catch (final IllegalArgumentException e) { throw new MetafactureException(e); diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java index 132ce77a..795e1807 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java @@ -17,7 +17,9 @@ package org.metafacture.metafix; import org.metafacture.framework.StreamReceiver; +import org.metafacture.metamorph.api.MorphExecutionException; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -37,6 +39,9 @@ @ExtendWith(MockitoExtension.class) public class MetafixLookupTest { + private static final String CSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.csv"; + private static final String TSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.tsv"; + @RegisterExtension private MockitoRule mockitoRule = MockitoJUnit.rule(); @@ -48,62 +53,26 @@ public MetafixLookupTest() { @Test public void inline() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)"), - i -> { - i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); - i.literal("title", "Aloha"); - i.literal("title", "Moin"); - i.literal("title", "Hey"); - i.endRecord(); - - i.startRecord("3"); - i.endRecord(); - }, o -> { - o.get().startRecord("1"); - o.get().endRecord(); - - o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("1", "Alohaeha"); - o.get().literal("2", "Moin zäme"); - o.get().literal("3", "Tach"); - o.get().endEntity(); - o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + assertMap( + "lookup('title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" + ); } @Test public void inlineMultilineIndent() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + assertMap( "lookup('title',", " Aloha: Alohaeha,", - " Moin: 'Moin zäme')"), - i -> { - i.startRecord("1"); - i.literal("title", "Aloha"); - i.literal("title", "Moin"); - i.endRecord(); - }, o -> { - o.get().startRecord("1"); - o.get().startEntity("title"); - o.get().literal("1", "Alohaeha"); - o.get().literal("2", "Moin zäme"); - o.get().endEntity(); - o.get().endRecord(); - }); + " Moin: 'Moin zäme',", + " __default: Tach)" + ); } @Test public void inlineDotNotationNested() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('data.title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)"), + "lookup('data.title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" + ), i -> { i.startRecord("1"); i.startEntity("data"); @@ -112,7 +81,8 @@ public void inlineDotNotationNested() { i.literal("title", "Hey"); i.endEntity(); i.endRecord(); - }, (o, f) -> { + }, + (o, f) -> { o.get().startRecord("1"); o.get().startEntity("data"); o.get().startEntity("title"); @@ -121,74 +91,181 @@ public void inlineDotNotationNested() { o.get().literal("3", "Tach"); f.apply(2).endEntity(); o.get().endRecord(); - }); + } + ); } @Test public void csv() { + assertMap( + "lookup('title', '" + CSV_MAP + "')" + ); + } + + @Test + public void tsv() { + assertMap( + "lookup('title', '" + TSV_MAP + "', sep_char:'\t')" + ); + } + + @Test + public void shouldLookupInSeparateInternalMap() { + assertMap( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", + "lookup('title', 'testMap')" + ); + } + + @Test + public void shouldLookupInSeparateExternalFileMap() { + assertMap( + "put_filemap('" + CSV_MAP + "')", + "lookup('title', '" + CSV_MAP + "')" + ); + } + + @Test + public void shouldLookupInSeparateExternalFileMapWithName() { + assertMap( + "put_filemap('" + CSV_MAP + "', 'testMap')", + "lookup('title', 'testMap')" + ); + } + + @Test + public void shouldLookupInSeparateExternalFileMapWithOptions() { + assertMap( + "put_filemap('" + TSV_MAP + "', sep_char: '\t')", + "lookup('title', '" + TSV_MAP + "')" + ); + } + + @Test + public void shouldLookupInSeparateExternalFileMapWithNameAndOptions() { + assertMap( + "put_filemap('" + TSV_MAP + "', 'testMap', sep_char: '\t')", + "lookup('title', 'testMap')" + ); + } + + @Test + public void shouldDefineMultipleSeparateMaps() { + assertMap( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", + "put_map('testMap2', __default: Hi)", + "lookup('title', 'testMap')" + ); + } + + @Test + public void shouldOverwriteExistingSeparateMap() { + assertMap( + "put_map('testMap', __default: Hi)", + "put_filemap('" + CSV_MAP + "', 'testMap')", + "lookup('title', 'testMap')" + ); + } + + @Test + public void shouldIgnoreOptionsOnLookupInSeparateInternalMap() { + assertMap( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", + "lookup('title', 'testMap', __default: Hi)" + ); + } + + @Test + public void shouldIgnoreOptionsOnLookupInSeparateExternalFileMap() { + assertMap( + "put_filemap('" + CSV_MAP + "')", + "lookup('title', '" + CSV_MAP + "', sep_char: '\t')" + ); + } + + @Test + public void shouldNotLookupInExternalFileMapWithWrongOptions() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', 'src/test/resources/org/metafacture/metafix/maps/test.csv')"), + "lookup('title', '" + CSV_MAP + "', sep_char: '\t')" + ), i -> { i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); - i.literal("title", "Aloha"); i.literal("title", "Moin"); i.literal("title", "Hey"); i.endRecord(); - - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); o.get().endRecord(); + } + ); + } - o.get().startRecord("2"); + @Test + public void shouldIgnoreOptionsOnSubsequentLookupInExternalFileMap() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title', '" + CSV_MAP + "')", + "lookup('title', '" + CSV_MAP + "', sep_char: '\t')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); o.get().startEntity("title"); - o.get().literal("1", "Alohaeha"); - o.get().literal("2", "Moin zäme"); + o.get().literal("1", "Tach"); + o.get().literal("2", "Tach"); o.get().literal("3", "Tach"); o.get().endEntity(); o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test - public void tsv() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', 'src/test/resources/org/metafacture/metafix/maps/test.tsv', sep_char:'\t')"), + public void shouldFailLookupInUnknownNamedMap() { + Assertions.assertThrows(MorphExecutionException.class, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title', 'testMap')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + }, + o -> { + } + ), + "File not found: testMap" + ); + } + + private void assertMap(final String... fixDef) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(fixDef), i -> { i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); i.literal("title", "Aloha"); i.literal("title", "Moin"); i.literal("title", "Hey"); i.endRecord(); - - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); - o.get().endRecord(); - - o.get().startRecord("2"); o.get().startEntity("title"); o.get().literal("1", "Alohaeha"); o.get().literal("2", "Moin zäme"); o.get().literal("3", "Tach"); o.get().endEntity(); o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } } diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java new file mode 100644 index 00000000..7f13da99 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java @@ -0,0 +1,111 @@ +/* + * Copyright 2021 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.helpers.DefaultStreamReceiver; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.FileNotFoundException; +import java.util.function.Consumer; + +/** + * Tests Metafix script level methods. + */ +public class MetafixScriptTest { + + private static final String MAP_NAME = "testMap"; + + private static final String CSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.csv"; + private static final String TSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.tsv"; + + public MetafixScriptTest() { + } + + @Test + public void shouldPutEmptyInternalMap() { + assertMap("put_map('" + MAP_NAME + "')", MAP_NAME); + } + + @Test + public void shouldPutMultipleInternalMaps() { + assertFix("put_map('" + MAP_NAME + "')\nput_map('" + MAP_NAME + "2')", f -> { + assertMap(f, MAP_NAME); + assertMap(f, MAP_NAME + "2"); + }); + } + + @Test + public void shouldPutInternalMapWithOptions() { + assertMap("put_map('" + MAP_NAME + "', k1: 'v1', k2: 'v2')", MAP_NAME); + } + + @Test + public void shouldPutExternalFileMap() { + assertMap("put_filemap('" + CSV_MAP + "')", CSV_MAP); + } + + @Test + public void shouldPutMultipleExternalFileMaps() { + assertFix("put_filemap('" + CSV_MAP + "')\nput_filemap('" + TSV_MAP + "')", f -> { + assertMap(f, CSV_MAP); + assertMap(f, TSV_MAP); + }); + } + + @Test + public void shouldPutExternalFileMapWithName() { + assertMap("put_filemap('" + CSV_MAP + "', '" + MAP_NAME + "')", MAP_NAME); + } + + @Test + public void shouldPutExternalFileMapWithOptions() { + assertMap("put_filemap('" + TSV_MAP + "', sep_char: '\t')", TSV_MAP); + } + + @Test + public void shouldPutExternalFileMapWithNameAndOptions() { + assertMap("put_filemap('" + TSV_MAP + "', '" + MAP_NAME + "', sep_char: '\t')", MAP_NAME); + } + + private void assertMap(final String fixDef, final String mapName) { + assertFix(fixDef, f -> assertMap(f, mapName)); + } + + private void assertMap(final Metafix metafix, final String mapName) { + Assertions.assertTrue(metafix.getMapNames().contains(mapName)); + Assertions.assertNotNull(metafix.getMap(mapName)); + } + + private void assertFix(final String fixDef, final Consumer consumer) { + try { + final Metafix metafix = new Metafix(fixDef); + + // Prepare and trigger script execution + metafix.setReceiver(new DefaultStreamReceiver()); + metafix.startRecord(""); + metafix.endRecord(); + + consumer.accept(metafix); + } + catch (final FileNotFoundException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixTest.java new file mode 100644 index 00000000..64a2224a --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixTest.java @@ -0,0 +1,127 @@ +/* + * Copyright 2021 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.metamorph.api.Maps; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * Tests Metafix API methods. + */ +public class MetafixTest { + + private static final String MAP_NAME = "testMap"; + private static final String KEY = "outName"; + private static final String VALUE = "testValue"; + + public MetafixTest() { + } + + @Test + public void shouldGetMapNames() { + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, new HashMap<>()); + metafix.putMap(MAP_NAME + "2", new HashMap<>()); + + final Collection actualNames = metafix.getMapNames(); + final Collection expectedNames = Arrays.asList(MAP_NAME, MAP_NAME + "2"); + + Assertions.assertTrue(actualNames.containsAll(expectedNames), "missing names"); + Assertions.assertTrue(expectedNames.containsAll(actualNames), "unexpected names"); + } + + @Test + public void shouldGetMap() { + final Map map = new HashMap<>(); + + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, map); + + Assertions.assertSame(map, metafix.getMap(MAP_NAME)); + } + + @Test + public void shouldGetEmptyUnknownMap() { + final Metafix metafix = new Metafix(); + Assertions.assertEquals(new HashMap<>(), metafix.getMap(MAP_NAME)); + } + + @Test + public void shouldPutValueIntoNewMap() { + final Metafix metafix = new Metafix(); + metafix.putValue(MAP_NAME, KEY, VALUE); + + Assertions.assertNotNull(metafix.getMap(MAP_NAME)); + Assertions.assertEquals(VALUE, metafix.getValue(MAP_NAME, KEY)); + } + + @Test + public void shouldPutValueIntoExistingMap() { + final Map map = new HashMap<>(); + + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, map); + metafix.putValue(MAP_NAME, KEY, VALUE); + + Assertions.assertEquals(VALUE, map.get(KEY)); + Assertions.assertEquals(VALUE, metafix.getValue(MAP_NAME, KEY)); + } + + @Test + public void shouldGetValueFromMap() { + final Map map = new HashMap<>(); + map.put(KEY, VALUE); + + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, map); + + Assertions.assertEquals(VALUE, metafix.getValue(MAP_NAME, KEY)); + } + + @Test + public void shouldNotGetValueFromUnknownMap() { + final Metafix metafix = new Metafix(); + Assertions.assertNull(metafix.getValue(MAP_NAME, KEY)); + } + + @Test + public void shouldNotGetValueForUnknownKey() { + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, new HashMap<>()); + + Assertions.assertNull(metafix.getValue(MAP_NAME, KEY)); + } + + @Test + public void shouldGetDefaultValueForUnknownKey() { + final Map map = new HashMap<>(); + map.put(Maps.DEFAULT_MAP_KEY, VALUE); + + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, map); + + Assertions.assertEquals(VALUE, metafix.getValue(MAP_NAME, KEY)); + } + +}