diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/YAML.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/YAML.java index 92e12e561f..fa8130cc3f 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/YAML.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/YAML.java @@ -1,6 +1,7 @@ package com.onthegomap.planetiler.util; import com.fasterxml.jackson.databind.ObjectMapper; +import com.onthegomap.planetiler.reader.FileFormatException; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -8,6 +9,12 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.snakeyaml.engine.v2.api.Load; import org.snakeyaml.engine.v2.api.LoadSettings; @@ -33,12 +40,64 @@ public static T load(Path file, Class clazz) { public static T load(InputStream stream, Class clazz) { try (stream) { Object parsed = snakeYaml.loadFromInputStream(stream); + handleMergeOperator(parsed); return convertValue(parsed, clazz); } catch (IOException e) { throw new UncheckedIOException(e); } } + private static void handleMergeOperator(Object parsed) { + handleMergeOperator(parsed, Collections.newSetFromMap(new IdentityHashMap<>())); + } + + /** + * SnakeYaml doesn't handle the merge operator so manually post-process + * the parsed yaml object to merge referenced objects into the parent one. + */ + private static void handleMergeOperator(Object parsed, Set parentNodes) { + if (!parentNodes.add(parsed)) { + throw new FileFormatException("Illegal recursive reference in yaml file"); + } + if (parsed instanceof Map map) { + Object toMerge = map.remove("<<"); + if (toMerge != null) { + var orig = new LinkedHashMap<>(map); + // to preserve the map key order we insert the merged operator objects first, then the original ones + map.clear(); + mergeInto(map, toMerge, false, parentNodes); + mergeInto(map, orig, true, parentNodes); + } + for (var value : map.values()) { + handleMergeOperator(value, parentNodes); + } + } else if (parsed instanceof List list) { + for (var item : list) { + handleMergeOperator(item, parentNodes); + } + } + parentNodes.remove(parsed); + } + + @SuppressWarnings("rawtypes") + private static void mergeInto(Map dest, Object source, boolean replace, Set parentNodes) { + if (!parentNodes.add(source)) { + throw new FileFormatException("Illegal recursive reference in yaml file"); + } + if (source instanceof Map map) { + if (replace) { + dest.putAll(map); + } else { + map.forEach(dest::putIfAbsent); + } + } else if (source instanceof List nesteds) { + for (var nested : nesteds) { + mergeInto(dest, nested, replace, parentNodes); + } + } + parentNodes.remove(source); + } + public static T load(String config, Class clazz) { try (var stream = new ByteArrayInputStream(config.getBytes(StandardCharsets.UTF_8))) { return load(stream, clazz); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/YamlTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/YamlTest.java index 59c5201bfb..f2c19d414d 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/YamlTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/YamlTest.java @@ -1,7 +1,9 @@ package com.onthegomap.planetiler.util; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import com.onthegomap.planetiler.reader.FileFormatException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -45,4 +47,258 @@ void testLoadLargeYamlMap() { } assertEquals(expected, YAML.load(builder.toString(), Map.class)); } + + @Test + void testMergeOperator() { + assertSameYaml(""" + source: &label + a: 1 + dest: + <<: *label + b: 2 + """, """ + source: + a: 1 + dest: + a: 1 + b: 2 + """); + } + + @Test + void testMergeOperatorNested() { + assertSameYaml(""" + source: &label + a: 1 + dest: + l1: + l2: + l3: + <<: *label + b: 2 + """, """ + source: + a: 1 + dest: + l1: + l2: + l3: + a: 1 + b: 2 + """); + } + + @Test + void testMergeOperatorOverride() { + assertSameYaml(""" + source: &label + a: 1 + dest: + <<: *label + a: 2 + """, """ + source: + a: 1 + dest: + a: 2 + """); + } + + @Test + void testMergeOperatorMultiple() { + assertSameYaml(""" + source: &label1 + a: 1 + z: 1 + source2: &label2 + a: 2 + b: 3 + dest: + <<: [*label1, *label2] + b: 4 + c: 5 + """, """ + source: + a: 1 + z: 1 + source2: + a: 2 + b: 3 + dest: + a: 1 # from label1 since it came first + b: 4 + c: 5 + z: 1 + """); + } + + @Test + void testMergeNotAnchor() { + assertSameYaml(""" + <<: + a: 1 + b: 2 + b: 3 + c: 4 + """, """ + a: 1 + b: 3 + c: 4 + """); + } + + @Test + void testMergeOperatorSecond() { + assertSameYaml(""" + source: &label + a: 1 + dest: + c: 3 + <<: *label + b: 2 + """, """ + source: + a: 1 + dest: + a: 1 + b: 2 + c: 3 + """); + } + + @Test + void testMergeOperatorFromDraft1() { + assertSameYaml(""" + - { x: 1, y: 2 } + - { x: 0, y: 2 } + - { r: 10 } + - { r: 1 } + - # Explicit keys + x: 1 + y: 2 + r: 10 + label: center/big + """, """ + - &CENTER { x: 1, y: 2 } + - &LEFT { x: 0, y: 2 } + - &BIG { r: 10 } + - &SMALL { r: 1 } + - # Merge one map + << : *CENTER + r: 10 + label: center/big + """); + } + + @Test + void testMergeOperatorFromDraft2() { + assertSameYaml(""" + - { x: 1, y: 2 } + - { x: 0, y: 2 } + - { r: 10 } + - { r: 1 } + - # Explicit keys + x: 1 + y: 2 + r: 10 + label: center/big + """, """ + - &CENTER { x: 1, y: 2 } + - &LEFT { x: 0, y: 2 } + - &BIG { r: 10 } + - &SMALL { r: 1 } + - # Merge multiple maps + << : [ *CENTER, *BIG ] + label: center/big + """); + } + + @Test + void testMergeOperatorFromDraft3() { + assertSameYaml(""" + - { x: 1, y: 2 } + - { x: 0, y: 2 } + - { r: 10 } + - { r: 1 } + - # Explicit keys + x: 1 + y: 2 + r: 10 + label: center/big + """, """ + - &CENTER { x: 1, y: 2 } + - &LEFT { x: 0, y: 2 } + - &BIG { r: 10 } + - &SMALL { r: 1 } + - # Override + << : [ *BIG, *LEFT, *SMALL ] + x: 1 + label: center/big + """); + } + + @Test + void testAnchorAndAliasMap() { + assertSameYaml(""" + source: &label + a: 1 + dest: *label + """, """ + source: + a: 1 + dest: + a: 1 + """); + } + + @Test + void testAnchorAndAliasList() { + assertSameYaml(""" + source: &label + - 1 + dest: *label + """, """ + source: [1] + dest: [1] + """); + } + + @Test + void testAllowRefInMergeDoc() { + assertSameYaml(""" + source: &label + a: &label1 + c: 1 + b: *label1 + d: + <<: *label1 + dest: *label + """, """ + source: {a: {c: 1}, b: {c: 1}, d: {c: 1}} + dest: {a: {c: 1}, b: {c: 1}, d: {c: 1}} + """); + } + + @Test + void testFailsOnRecursiveRefs() { + assertThrows(FileFormatException.class, () -> YAML.load(""" + source: &label + - *label + """, Object.class)); + assertThrows(FileFormatException.class, () -> YAML.load(""" + source: &label + <<: *label + """, Object.class)); + assertThrows(FileFormatException.class, () -> YAML.load(""" + source: &label + a: *label + """, Object.class)); + } + + private static void assertSameYaml(String a, String b) { + assertEquals( + YAML.load(b, Object.class), + YAML.load(a, Object.class) + ); + } }