feat(core): add a setting model and repository

kestra-io · Aug 10, 2022 · 73168c4 · 73168c4
1 parent 3514b9e
commit 73168c4
Show file tree

Hide file tree

Showing 36 changed files with 520 additions and 18 deletions.
diff --git a/cli/src/main/resources/application.yml b/cli/src/main/resources/application.yml
@@ -246,6 +246,9 @@ kestra:
       executordelayed:
         table: "executordelayed"
         cls: io.kestra.core.runners.ExecutionDelay
+      settings:
+        table: "settings"
+        cls: io.kestra.core.models.Setting
 
     queues:
       min-poll-interval: 100ms
@@ -285,6 +288,10 @@ kestra:
         index: "${kestra.elasticsearch.defaults.indice-prefix}logs"
         cls: io.kestra.core.models.executions.LogEntry
         mapping-file: log
+      settings:
+        index: "${kestra.elasticsearch.defaults.indice-prefix}settings"
+        cls: io.kestra.core.models.Setting
+        mapping-file: setting
 
   indexer:
     models:

diff --git a/core/src/main/java/io/kestra/core/models/Setting.java b/core/src/main/java/io/kestra/core/models/Setting.java
@@ -0,0 +1,23 @@
+package io.kestra.core.models;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import lombok.experimental.SuperBuilder;
+
+import javax.validation.constraints.NotNull;
+
+@SuperBuilder
+@ToString
+@EqualsAndHashCode
+@Getter
+@NoArgsConstructor
+public class Setting {
+    public static final String INSTANCE_UUID = "instance.uuid";
+    @NotNull
+    private String key;
+
+    @NotNull
+    private Object value;
+}
diff --git a/core/src/main/java/io/kestra/core/models/collectors/ConfigurationUsage.java b/core/src/main/java/io/kestra/core/models/collectors/ConfigurationUsage.java
@@ -21,7 +21,7 @@ public static ConfigurationUsage of(ApplicationContext applicationContext) {
         return ConfigurationUsage.builder()
             .repositoryType(applicationContext.getProperty("kestra.repository.type", String.class).orElse(null))
             .queueType(applicationContext.getProperty("kestra.queue.type", String.class).orElse(null))
-            .repositoryType(applicationContext.getProperty("kestra.storage.type", String.class).orElse(null))
+            .storageType(applicationContext.getProperty("kestra.storage.type", String.class).orElse(null))
             .secretType(applicationContext.getProperty("kestra.secret.type", String.class).orElse(null))
             .javaSecurityEnabled(applicationContext.getProperty("kestra.ee.java-security.enabled", Boolean.class).orElse(null))
             .build();

diff --git a/core/src/main/java/io/kestra/core/models/collectors/Usage.java b/core/src/main/java/io/kestra/core/models/collectors/Usage.java
@@ -25,6 +25,9 @@ public class Usage {
     @NotNull
     private final String startUuid;
 
+    @NotNull
+    private final String instanceUuid;
+
     @NotNull
     private final ServerType serverType;
 

diff --git a/core/src/main/java/io/kestra/core/queues/QueueService.java b/core/src/main/java/io/kestra/core/queues/QueueService.java
@@ -1,5 +1,6 @@
 package io.kestra.core.queues;
 
+import io.kestra.core.models.Setting;
 import io.kestra.core.models.executions.Execution;
 import io.kestra.core.models.executions.ExecutionKilled;
 import io.kestra.core.models.executions.LogEntry;
@@ -42,6 +43,8 @@ public String key(Object object) {
             return ((ExecutionDelay) object).getExecutionId();
         } else if (object.getClass() == ExecutorState.class) {
             return ((ExecutorState) object).getExecutionId();
+        } else if (object.getClass() == Setting.class) {
+            return ((Setting) object).getKey();
         } else {
             throw new IllegalArgumentException("Unknown type '" + object.getClass().getName() + "'");
         }

diff --git a/core/src/main/java/io/kestra/core/repositories/SettingRepositoryInterface.java b/core/src/main/java/io/kestra/core/repositories/SettingRepositoryInterface.java
@@ -0,0 +1,17 @@
+package io.kestra.core.repositories;
+
+import io.kestra.core.models.Setting;
+
+import java.util.List;
+import java.util.Optional;
+import javax.validation.ConstraintViolationException;
+
+public interface SettingRepositoryInterface {
+    Optional<Setting> findByKey(String key);
+
+    List<Setting> findAll();
+
+    Setting save(Setting setting) throws ConstraintViolationException;
+
+    Setting delete(Setting setting);
+}
diff --git a/core/src/main/java/io/kestra/core/services/CollectorService.java b/core/src/main/java/io/kestra/core/services/CollectorService.java
@@ -1,9 +1,11 @@
 package io.kestra.core.services;
 
 import io.kestra.core.models.ServerType;
+import io.kestra.core.models.Setting;
 import io.kestra.core.models.collectors.*;
 import io.kestra.core.repositories.ExecutionRepositoryInterface;
 import io.kestra.core.repositories.FlowRepositoryInterface;
+import io.kestra.core.repositories.SettingRepositoryInterface;
 import io.kestra.core.serializers.JacksonMapper;
 import io.kestra.core.utils.IdUtils;
 import io.kestra.core.utils.VersionProvider;
@@ -44,6 +46,9 @@ public class CollectorService {
     @Inject
     private ExecutionRepositoryInterface executionRepository;
 
+    @Inject
+    private SettingRepositoryInterface settingRepository;
+
     @Inject
     private VersionProvider versionProvider;
 
@@ -64,8 +69,17 @@ protected synchronized Usage defaultUsage() {
         boolean first = defaultUsage == null;
 
         if (first) {
+            Setting instanceIdSetting = settingRepository
+                .findByKey(Setting.INSTANCE_UUID)
+                .orElseGet(() -> settingRepository.save(Setting.builder()
+                    .key(Setting.INSTANCE_UUID)
+                    .value(IdUtils.create())
+                    .build()
+                ));
+
             defaultUsage = Usage.builder()
                 .startUuid(UUID)
+                .instanceUuid(instanceIdSetting.getValue().toString())
                 .serverType(serverType)
                 .version(versionProvider.getVersion())
                 .zoneId(ZoneId.systemDefault())

diff --git a/core/src/test/java/io/kestra/core/repositories/AbstracSettingRepositoryTest.java b/core/src/test/java/io/kestra/core/repositories/AbstracSettingRepositoryTest.java
@@ -0,0 +1,50 @@
+package io.kestra.core.repositories;
+
+import io.kestra.core.models.Setting;
+import io.kestra.core.utils.IdUtils;
+import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Optional;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+@MicronautTest(transactional = false)
+public abstract class AbstracSettingRepositoryTest {
+    @Inject
+    protected SettingRepositoryInterface settingRepository;
+
+    @Test
+    void all() {
+        Setting setting = Setting.builder()
+            .key(Setting.INSTANCE_UUID)
+            .value(IdUtils.create())
+            .build();
+
+        Optional<Setting> find = settingRepository.findByKey(setting.getKey());
+        assertThat(find.isPresent(), is(false));
+
+        Setting save = settingRepository.save(setting);
+
+        find = settingRepository.findByKey(save.getKey());
+
+        assertThat(find.isPresent(), is(true));
+        assertThat(find.get().getValue(), is(save.getValue()));
+
+        List<Setting> all = settingRepository.findAll();
+        assertThat(all.size(), is(1));
+        assertThat(all.get(0).getValue(), is(setting.getValue()));
+
+        Setting delete = settingRepository.delete(setting);
+        assertThat(delete.getValue(), is(setting.getValue()));
+
+        all = settingRepository.findAll();
+        assertThat(all.size(), is(0));
+
+        find = settingRepository.findByKey(setting.getKey());
+        assertThat(find.isPresent(), is(false));
+    }
+}
diff --git a/core/src/test/java/io/kestra/core/services/CollectorServiceTest.java b/core/src/test/java/io/kestra/core/services/CollectorServiceTest.java
@@ -3,16 +3,29 @@
 import com.google.common.collect.ImmutableMap;
 import io.kestra.core.Helpers;
 import io.kestra.core.models.ServerType;
+import io.kestra.core.models.Setting;
 import io.kestra.core.models.collectors.Usage;
+import io.kestra.core.repositories.SettingRepositoryInterface;
 import io.kestra.core.utils.IdUtils;
 import io.micronaut.context.ApplicationContext;
+import io.micronaut.context.annotation.Property;
+import io.micronaut.context.annotation.Requires;
+import io.micronaut.core.util.StringUtils;
+import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
+import jakarta.inject.Singleton;
 import org.junit.jupiter.api.Test;
 
 import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import javax.validation.ConstraintViolationException;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
 
+@MicronautTest
 class CollectorServiceTest {
     @Test
     public void metrics() throws URISyntaxException {
@@ -33,8 +46,39 @@ public void metrics() throws URISyntaxException {
             assertThat(metrics.getHost().getHardware().getLogicalProcessorCount(), notNullValue());
             assertThat(metrics.getHost().getJvm().getName(), notNullValue());
             assertThat(metrics.getHost().getOs().getFamily(), notNullValue());
-            assertThat(metrics.getConfigurations().getRepositoryType(), is("local"));
+            assertThat(metrics.getConfigurations().getRepositoryType(), is("memory"));
             assertThat(metrics.getConfigurations().getQueueType(), is("memory"));
+            assertThat(metrics.getInstanceUuid(), is(TestSettingRepository.instanceUuid));
+        }
+    }
+
+    @Singleton
+    @Requires(property = "kestra.unittest")
+    public static class TestSettingRepository implements SettingRepositoryInterface {
+        public static Object instanceUuid = null;
+
+        @Override
+        public Optional<Setting> findByKey(String key) {
+            return Optional.empty();
+        }
+
+        @Override
+        public List<Setting> findAll() {
+            return new ArrayList<>();
+        }
+
+        @Override
+        public Setting save(Setting setting) throws ConstraintViolationException {
+            if (setting.getKey().equals(Setting.INSTANCE_UUID)) {
+                TestSettingRepository.instanceUuid = setting.getValue();
+            }
+
+            return setting;
+        }
+
+        @Override
+        public Setting delete(Setting setting) {
+            return setting;
         }
     }
 }
diff --git a/core/src/test/java/io/kestra/core/tasks/storages/SizeTest.java b/core/src/test/java/io/kestra/core/tasks/storages/SizeTest.java
@@ -38,6 +38,6 @@ void run() throws Exception {
             .build();
 
         Size.Output run = bash.run(runContext);
-        assertThat(run.getSize(), is(757L));
+        assertThat(run.getSize(), is(775L));
     }
 }
diff --git a/core/src/test/resources/application.yml b/core/src/test/resources/application.yml
@@ -38,4 +38,6 @@ kestra:
 
   anonymous-usage-report:
     enabled: false
-    uri: http://localhost:8080/
+    uri: http://localhost:8080/
+
+  unittest: true
diff --git a/jdbc-h2/src/main/java/io/kestra/repository/h2/H2SettingRepository.java b/jdbc-h2/src/main/java/io/kestra/repository/h2/H2SettingRepository.java
@@ -0,0 +1,18 @@
+package io.kestra.repository.h2;
+
+import io.kestra.core.models.Setting;
+import io.kestra.core.models.triggers.Trigger;
+import io.kestra.jdbc.repository.AbstractJdbcSettingRepository;
+import io.kestra.jdbc.repository.AbstractJdbcTriggerRepository;
+import io.micronaut.context.ApplicationContext;
+import jakarta.inject.Inject;
+import jakarta.inject.Singleton;
+
+@Singleton
+@H2RepositoryEnabled
+public class H2SettingRepository extends AbstractJdbcSettingRepository {
+    @Inject
+    public H2SettingRepository(ApplicationContext applicationContext) {
+        super(new H2Repository<>(Setting.class, applicationContext), applicationContext);
+    }
+}
diff --git a/jdbc-h2/src/main/resources/migrations/h2/V2__setting.sql b/jdbc-h2/src/main/resources/migrations/h2/V2__setting.sql
@@ -0,0 +1,5 @@
+CREATE TABLE settings (
+    "key" VARCHAR(250) NOT NULL PRIMARY KEY,
+    "value" TEXT NOT NULL
+);
+
diff --git a/jdbc-h2/src/test/java/io/kestra/repository/h2/H2SettingRepositoryTest.java b/jdbc-h2/src/test/java/io/kestra/repository/h2/H2SettingRepositoryTest.java
@@ -0,0 +1,7 @@
+package io.kestra.repository.h2;
+
+import io.kestra.jdbc.repository.AbstractJdbcSettingRepositoryTest;
+
+public class H2SettingRepositoryTest extends AbstractJdbcSettingRepositoryTest {
+
+}
diff --git a/jdbc-h2/src/test/resources/application.yml b/jdbc-h2/src/test/resources/application.yml
@@ -52,4 +52,7 @@ kestra:
         cls: io.kestra.core.runners.ExecutorState
       executordelayed:
         table: "executordelayed"
-        cls: io.kestra.core.runners.ExecutionDelay
+        cls: io.kestra.core.runners.ExecutionDelay
+      settings:
+        table: "settings"
+        cls: io.kestra.core.models.Setting
diff --git a/jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlExecutionRepository.java b/jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlExecutionRepository.java
@@ -1,7 +1,6 @@
 package io.kestra.repository.mysql;
 
 import io.kestra.core.models.executions.Execution;
-import io.kestra.core.repositories.ExecutionRepositoryInterface;
 import io.kestra.jdbc.repository.AbstractJdbcExecutionRepository;
 import io.kestra.jdbc.runner.AbstractJdbcExecutorStateStorage;
 import io.micronaut.context.ApplicationContext;
@@ -13,7 +12,7 @@
 
 @Singleton
 @MysqlRepositoryEnabled
-public class MysqlExecutionRepository extends AbstractJdbcExecutionRepository implements ExecutionRepositoryInterface {
+public class MysqlExecutionRepository extends AbstractJdbcExecutionRepository {
     @Inject
     public MysqlExecutionRepository(ApplicationContext applicationContext, AbstractJdbcExecutorStateStorage executorStateStorage) {
         super(new MysqlRepository<>(Execution.class, applicationContext), applicationContext, executorStateStorage);

diff --git a/jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlSettingRepository.java b/jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlSettingRepository.java
@@ -0,0 +1,17 @@
+package io.kestra.repository.mysql;
+
+import io.kestra.core.models.Setting;
+import io.kestra.core.models.triggers.Trigger;
+import io.kestra.jdbc.repository.AbstractJdbcSettingRepository;
+import io.kestra.jdbc.repository.AbstractJdbcTriggerRepository;
+import io.micronaut.context.ApplicationContext;
+import jakarta.inject.Inject;
+import jakarta.inject.Singleton;
+
+@Singleton
+@MysqlRepositoryEnabled
+public class MysqlSettingRepository extends AbstractJdbcSettingRepository {
+    @Inject
+    public MysqlSettingRepository(ApplicationContext applicationContext) {
+        super(new MysqlRepository<>(Setting.class, applicationContext), applicationContext);
+    }}
diff --git a/jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlTemplateRepository.java b/jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlTemplateRepository.java
@@ -12,7 +12,7 @@
 
 @Singleton
 @MysqlRepositoryEnabled
-public class MysqlTemplateRepository extends AbstractJdbcTemplateRepository implements TemplateRepositoryInterface {
+public class MysqlTemplateRepository extends AbstractJdbcTemplateRepository {
     @Inject
     public MysqlTemplateRepository(ApplicationContext applicationContext) {
         super(new MysqlRepository<>(Template.class, applicationContext), applicationContext);

diff --git a/jdbc-mysql/src/main/resources/migrations/mysql/V2__setting.sql b/jdbc-mysql/src/main/resources/migrations/mysql/V2__setting.sql
@@ -0,0 +1,4 @@
+CREATE TABLE settings (
+    `key` VARCHAR(250) NOT NULL PRIMARY KEY,
+    `value` JSON NOT NULL
+) ENGINE INNODB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
diff --git a/jdbc-mysql/src/test/java/io/kestra/repository/mysql/MysqlSettingRepositoryTest.java b/jdbc-mysql/src/test/java/io/kestra/repository/mysql/MysqlSettingRepositoryTest.java
@@ -0,0 +1,7 @@
+package io.kestra.repository.mysql;
+
+import io.kestra.jdbc.repository.AbstractJdbcSettingRepositoryTest;
+
+public class MysqlSettingRepositoryTest extends AbstractJdbcSettingRepositoryTest {
+
+}
diff --git a/jdbc-mysql/src/test/resources/application.yml b/jdbc-mysql/src/test/resources/application.yml
@@ -53,4 +53,7 @@ kestra:
         cls: io.kestra.core.runners.ExecutorState
       executordelayed:
         table: "executordelayed"
-        cls: io.kestra.core.runners.ExecutionDelay
+        cls: io.kestra.core.runners.ExecutionDelay
+      settings:
+        table: "settings"
+        cls: io.kestra.core.models.Setting
diff --git a/jdbc-postgres/src/main/java/io/kestra/repository/postgres/PostgresExecutionRepository.java b/jdbc-postgres/src/main/java/io/kestra/repository/postgres/PostgresExecutionRepository.java
@@ -18,7 +18,7 @@
 
 @Singleton
 @PostgresRepositoryEnabled
-public class PostgresExecutionRepository extends AbstractJdbcExecutionRepository implements ExecutionRepositoryInterface {
+public class PostgresExecutionRepository extends AbstractJdbcExecutionRepository {
     @Inject
     public PostgresExecutionRepository(ApplicationContext applicationContext, AbstractJdbcExecutorStateStorage executorStateStorage) {
         super(new PostgresRepository<>(Execution.class, applicationContext), applicationContext, executorStateStorage);