From 50d12f4300cba833692eb5512a0d1721aaee5e92 Mon Sep 17 00:00:00 2001 From: "brian.mulier" Date: Mon, 1 Jul 2024 13:40:01 +0200 Subject: [PATCH] feat(ui): introduce KV Store --- .../runners/pebble/TypedObjectWriter.java | 38 +++++- .../io/kestra/core/storages/kv/KVEntry.java | 6 +- .../io/kestra/core/storages/kv/KVStore.java | 24 +++- .../core/storages/kv/KVStoreValueWrapper.java | 6 - .../icons/io.kestra.plugin.core.kv.Get.svg | 4 + .../io.kestra.plugin.core.kv.GetKeys.svg | 3 + .../icons/io.kestra.plugin.core.kv.Set.svg | 4 + .../icons/io.kestra.plugin.core.kv.svg | 3 + .../runners/pebble/TypedObjectWriterTest.java | 113 ++++++++++++++++ .../io/kestra/plugin/core/kv/SetTest.java | 4 +- .../executions/date-select/DateFilter.vue | 6 +- .../executions/date-select/DateSelect.vue | 16 ++- .../date-select/RelativeDateSelect.vue | 74 ---------- .../executions/date-select/TimeSelect.vue | 128 ++++++++++++++++++ ui/src/models/permission.js | 2 +- ui/src/translations/en.json | 26 +++- ui/src/translations/fr.json | 30 +++- .../controllers/api/KVController.java | 48 +++++-- .../controllers/api/KVControllerTest.java | 95 +++++++------ 19 files changed, 484 insertions(+), 146 deletions(-) create mode 100644 core/src/main/resources/icons/io.kestra.plugin.core.kv.Get.svg create mode 100644 core/src/main/resources/icons/io.kestra.plugin.core.kv.GetKeys.svg create mode 100644 core/src/main/resources/icons/io.kestra.plugin.core.kv.Set.svg create mode 100644 core/src/main/resources/icons/io.kestra.plugin.core.kv.svg create mode 100644 core/src/test/java/io/kestra/core/runners/pebble/TypedObjectWriterTest.java delete mode 100644 ui/src/components/executions/date-select/RelativeDateSelect.vue create mode 100644 ui/src/components/executions/date-select/TimeSelect.vue diff --git a/core/src/main/java/io/kestra/core/runners/pebble/TypedObjectWriter.java b/core/src/main/java/io/kestra/core/runners/pebble/TypedObjectWriter.java index 5ce5cc57bb..17307c39a0 100644 --- a/core/src/main/java/io/kestra/core/runners/pebble/TypedObjectWriter.java +++ b/core/src/main/java/io/kestra/core/runners/pebble/TypedObjectWriter.java @@ -58,7 +58,17 @@ public void writeSpecialized(short s) { if (current == null) { current = s; } else { - Short currentS = this.ofSameTypeOrThrow(current, Short.class); + Short currentS = null; + try { + currentS = this.ofSameTypeOrThrow(current, Short.class); + } catch (Exception e) { + try { + current = this.ofSameTypeOrThrow(current, Integer.class) + s; + return; + } catch (Exception ex) { + throw e; + } + } current = currentS + s; } } @@ -68,7 +78,17 @@ public void writeSpecialized(byte b) { if (current == null) { current = b; } else { - Byte currentB = this.ofSameTypeOrThrow(current, Byte.class); + Byte currentB = null; + try { + currentB = this.ofSameTypeOrThrow(current, Byte.class); + } catch (Exception e) { + try { + current = this.ofSameTypeOrThrow(current, Integer.class) + b; + return; + } catch (Exception ex) { + throw e; + } + } current = currentB + b; } } @@ -78,8 +98,18 @@ public void writeSpecialized(char c) { if (current == null) { current = c; } else { - Character currentC = this.ofSameTypeOrThrow(current, Character.class); - current = currentC + c; + Character currentC; + try { + currentC = this.ofSameTypeOrThrow(current, Character.class); + } catch (Exception e) { + try { + current = this.ofSameTypeOrThrow(current, String.class) + c; + return; + } catch (Exception ex) { + throw e; + } + } + current = "" + currentC + c; } } diff --git a/core/src/main/java/io/kestra/core/storages/kv/KVEntry.java b/core/src/main/java/io/kestra/core/storages/kv/KVEntry.java index d236a0ac10..5da4696a84 100644 --- a/core/src/main/java/io/kestra/core/storages/kv/KVEntry.java +++ b/core/src/main/java/io/kestra/core/storages/kv/KVEntry.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Optional; public record KVEntry(String key, Instant creationDate, Instant updateDate, Instant expirationDate) { public static KVEntry from(FileAttributes fileAttributes) throws IOException { @@ -11,7 +13,9 @@ public static KVEntry from(FileAttributes fileAttributes) throws IOException { fileAttributes.getFileName().replace(".ion", ""), Instant.ofEpochMilli(fileAttributes.getCreationTime()), Instant.ofEpochMilli(fileAttributes.getLastModifiedTime()), - new KVMetadata(fileAttributes.getMetadata()).getExpirationDate() + Optional.ofNullable(new KVMetadata(fileAttributes.getMetadata()).getExpirationDate()) + .map(expirationDate -> expirationDate.truncatedTo(ChronoUnit.MILLIS)) + .orElse(null) ); } } diff --git a/core/src/main/java/io/kestra/core/storages/kv/KVStore.java b/core/src/main/java/io/kestra/core/storages/kv/KVStore.java index a4c05ba01f..d5d4a7574f 100644 --- a/core/src/main/java/io/kestra/core/storages/kv/KVStore.java +++ b/core/src/main/java/io/kestra/core/storages/kv/KVStore.java @@ -3,12 +3,13 @@ import io.kestra.core.exceptions.ResourceExpiredException; import io.kestra.core.serializers.JacksonMapper; import io.kestra.core.storages.StorageContext; -import io.kestra.core.utils.Rethrow; import java.io.IOException; import java.net.URI; +import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.regex.Pattern; import static io.kestra.core.utils.Rethrow.throwFunction; @@ -16,6 +17,8 @@ * Service interface for accessing the files attached to a namespace Key-Value store. */ public interface KVStore { + Pattern durationPattern = Pattern.compile("^P(?=.)(?:\\d*D)?(?:T(?=.)(?:\\d*H)?(?:\\d*M)?(?:\\d*S)?)?$"); + default void validateKey(String key) { if (key == null || key.isEmpty()) { throw new IllegalArgumentException("Key cannot be null or empty"); @@ -38,13 +41,28 @@ default URI storageUri(String key, String namespace) { } default void put(String key, KVStoreValueWrapper kvStoreValueWrapper) throws IOException { - this.putRaw(key, KVStoreValueWrapper.ionStringify(kvStoreValueWrapper)); + Object value = kvStoreValueWrapper.value(); + String ionValue; + if (value instanceof Duration duration) { + ionValue = duration.toString(); + } else { + ionValue = JacksonMapper.ofIon().writeValueAsString(value); + } + + this.putRaw(key, new KVStoreValueWrapper<>(kvStoreValueWrapper.kvMetadata(), ionValue)); } void putRaw(String key, KVStoreValueWrapper kvStoreValueWrapper) throws IOException; default Optional get(String key) throws IOException, ResourceExpiredException { - return this.getRaw(key).map(throwFunction(raw -> JacksonMapper.ofIon().readValue(raw, Object.class))); + return this.getRaw(key).map(throwFunction(raw -> { + Object value = JacksonMapper.ofIon().readValue(raw, Object.class); + if (value instanceof String valueStr && durationPattern.matcher(valueStr).matches()) { + return Duration.parse(valueStr); + } + + return value; + })); } Optional getRaw(String key) throws IOException, ResourceExpiredException; diff --git a/core/src/main/java/io/kestra/core/storages/kv/KVStoreValueWrapper.java b/core/src/main/java/io/kestra/core/storages/kv/KVStoreValueWrapper.java index dfd02e84d7..3581d4abec 100644 --- a/core/src/main/java/io/kestra/core/storages/kv/KVStoreValueWrapper.java +++ b/core/src/main/java/io/kestra/core/storages/kv/KVStoreValueWrapper.java @@ -1,7 +1,5 @@ package io.kestra.core.storages.kv; -import com.fasterxml.jackson.core.JsonProcessingException; -import io.kestra.core.serializers.JacksonMapper; import io.kestra.core.storages.StorageObject; import java.io.IOException; @@ -36,8 +34,4 @@ static KVStoreValueWrapper from(StorageObject storageObject) throws IOEx return new KVStoreValueWrapper<>(new KVMetadata(storageObject.metadata()), ionString); } } - - static KVStoreValueWrapper ionStringify(KVStoreValueWrapper kvStoreValueWrapper) throws JsonProcessingException { - return new KVStoreValueWrapper<>(kvStoreValueWrapper.kvMetadata(), JacksonMapper.ofIon().writeValueAsString(kvStoreValueWrapper.value())); - } } diff --git a/core/src/main/resources/icons/io.kestra.plugin.core.kv.Get.svg b/core/src/main/resources/icons/io.kestra.plugin.core.kv.Get.svg new file mode 100644 index 0000000000..caf9d76310 --- /dev/null +++ b/core/src/main/resources/icons/io.kestra.plugin.core.kv.Get.svg @@ -0,0 +1,4 @@ + + + + diff --git a/core/src/main/resources/icons/io.kestra.plugin.core.kv.GetKeys.svg b/core/src/main/resources/icons/io.kestra.plugin.core.kv.GetKeys.svg new file mode 100644 index 0000000000..a534ad4775 --- /dev/null +++ b/core/src/main/resources/icons/io.kestra.plugin.core.kv.GetKeys.svg @@ -0,0 +1,3 @@ + + + diff --git a/core/src/main/resources/icons/io.kestra.plugin.core.kv.Set.svg b/core/src/main/resources/icons/io.kestra.plugin.core.kv.Set.svg new file mode 100644 index 0000000000..d3eb5b4220 --- /dev/null +++ b/core/src/main/resources/icons/io.kestra.plugin.core.kv.Set.svg @@ -0,0 +1,4 @@ + + + + diff --git a/core/src/main/resources/icons/io.kestra.plugin.core.kv.svg b/core/src/main/resources/icons/io.kestra.plugin.core.kv.svg new file mode 100644 index 0000000000..e92707b778 --- /dev/null +++ b/core/src/main/resources/icons/io.kestra.plugin.core.kv.svg @@ -0,0 +1,3 @@ + + + diff --git a/core/src/test/java/io/kestra/core/runners/pebble/TypedObjectWriterTest.java b/core/src/test/java/io/kestra/core/runners/pebble/TypedObjectWriterTest.java new file mode 100644 index 0000000000..79418a554f --- /dev/null +++ b/core/src/test/java/io/kestra/core/runners/pebble/TypedObjectWriterTest.java @@ -0,0 +1,113 @@ +package io.kestra.core.runners.pebble; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +public class TypedObjectWriterTest { + @Test + void invalidAddition() throws IOException { + try (TypedObjectWriter writer = new TypedObjectWriter()){ + writer.writeSpecialized(1); + IllegalArgumentException illegalArgumentException = Assertions.assertThrows(IllegalArgumentException.class, () -> writer.writeSpecialized('a')); + assertThat(illegalArgumentException.getMessage(), is("Tried to add java.lang.Character to java.lang.Integer")); + } + } + + @Test + void writeInts() throws IOException { + try (TypedObjectWriter writer = new TypedObjectWriter()){ + writer.writeSpecialized(1); + writer.writeSpecialized(2); + writer.writeSpecialized(3); + assertThat(writer.output(), is(6)); + } + } + + @Test + void writeLongs() throws IOException { + try (TypedObjectWriter writer = new TypedObjectWriter()){ + writer.writeSpecialized(1L); + writer.writeSpecialized(2L); + writer.writeSpecialized(3L); + assertThat(writer.output(), is(6L)); + } + } + + @Test + void writeDoubles() throws IOException { + try (TypedObjectWriter writer = new TypedObjectWriter()){ + writer.writeSpecialized(1.0); + writer.writeSpecialized(2.0); + writer.writeSpecialized(3.0); + assertThat(writer.output(), is(6.0)); + } + } + + @Test + void writeFloats() throws IOException { + try (TypedObjectWriter writer = new TypedObjectWriter()){ + writer.writeSpecialized(1.0f); + writer.writeSpecialized(2.0f); + writer.writeSpecialized(3.0f); + assertThat(writer.output(), is(6.0f)); + } + } + + @Test + void writeShorts() throws IOException { + try (TypedObjectWriter writer = new TypedObjectWriter()){ + writer.writeSpecialized((short) 1); + writer.writeSpecialized((short) 2); + writer.writeSpecialized((short) 3); + assertThat(writer.output(), is(6)); + } + } + + @Test + void writeBytes() throws IOException { + try (TypedObjectWriter writer = new TypedObjectWriter()){ + byte aByte = "a".getBytes()[0]; + writer.writeSpecialized(aByte); + byte bByte = "b".getBytes()[0]; + writer.writeSpecialized(bByte); + byte cByte = "c".getBytes()[0]; + writer.writeSpecialized(cByte); + assertThat(writer.output(), is((aByte + bByte) + cByte)); + } + } + + @Test + void writeChars() throws IOException { + try (TypedObjectWriter writer = new TypedObjectWriter()){ + writer.writeSpecialized('a'); + writer.writeSpecialized('b'); + writer.writeSpecialized('c'); + assertThat(writer.output(), is("abc")); + } + } + + @Test + void writeStrings() throws IOException { + try (TypedObjectWriter writer = new TypedObjectWriter()){ + writer.writeSpecialized("a"); + writer.writeSpecialized("b"); + writer.writeSpecialized("c"); + assertThat(writer.output(), is("abc")); + } + } + + @Test + void writeObjects() throws IOException { + try (TypedObjectWriter writer = new TypedObjectWriter()){ + writer.write(Map.of("a", "b")); + IllegalArgumentException illegalArgumentException = Assertions.assertThrows(IllegalArgumentException.class, () -> writer.write(Map.of("c", "d"))); + assertThat(illegalArgumentException.getMessage(), is("Tried to add java.util.ImmutableCollections$Map1 to java.util.ImmutableCollections$Map1")); + } + } +} diff --git a/core/src/test/java/io/kestra/plugin/core/kv/SetTest.java b/core/src/test/java/io/kestra/plugin/core/kv/SetTest.java index 498fad9063..980214d8a9 100644 --- a/core/src/test/java/io/kestra/plugin/core/kv/SetTest.java +++ b/core/src/test/java/io/kestra/plugin/core/kv/SetTest.java @@ -52,7 +52,7 @@ void defaultCase() throws Exception { set.run(runContext); final KVStore kv = runContext.storage().namespaceKv(namespaceId); - assertThat(kv.get(key), is(value)); + assertThat(kv.get(key).get(), is(value)); assertThat(kv.list().get(0).expirationDate(), nullValue()); } @@ -76,7 +76,7 @@ void ttl() throws Exception { set.run(runContext); final KVStore kv = runContext.storage().namespaceKv(namespaceId); - assertThat(kv.get(key), is(value)); + assertThat(kv.get(key).get(), is(value)); Instant expirationDate = kv.list().get(0).expirationDate(); assertThat(expirationDate.isAfter(Instant.now().plus(Duration.ofMinutes(4))) && expirationDate.isBefore(Instant.now().plus(Duration.ofMinutes(6))), is(true)); } diff --git a/ui/src/components/executions/date-select/DateFilter.vue b/ui/src/components/executions/date-select/DateFilter.vue index 663a940c77..5810f7bb39 100644 --- a/ui/src/components/executions/date-select/DateFilter.vue +++ b/ui/src/components/executions/date-select/DateFilter.vue @@ -18,7 +18,7 @@ @update:model-value="onAbsFilterChange" class="w-auto" /> - import DateRange from "../../layout/DateRange.vue"; - import RelativeDateSelect from "./RelativeDateSelect.vue"; + import TimeSelect from "./TimeSelect.vue"; export default { components: { DateRange, - RelativeDateSelect + TimeSelect }, emits: [ "update:isRelative", diff --git a/ui/src/components/executions/date-select/DateSelect.vue b/ui/src/components/executions/date-select/DateSelect.vue index 8303930f83..87e0747737 100644 --- a/ui/src/components/executions/date-select/DateSelect.vue +++ b/ui/src/components/executions/date-select/DateSelect.vue @@ -1,8 +1,10 @@