From 4924bdd6a693fcd23979b26eabaf14b6691d4a9f Mon Sep 17 00:00:00 2001 From: Benjamin Decreusefond Date: Fri, 8 Nov 2024 09:56:40 +0100 Subject: [PATCH 1/6] add possibility to mount config map --- .../ephemeral/EphemeralExecutorService.java | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java b/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java index 614650326..2ac2ff41e 100644 --- a/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java +++ b/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java @@ -1,9 +1,7 @@ package org.terrakube.api.plugin.scheduler.job.tcl.executor.ephemeral; import com.fasterxml.jackson.databind.ObjectMapper; -import io.fabric8.kubernetes.api.model.EnvFromSource; -import io.fabric8.kubernetes.api.model.EnvVar; -import io.fabric8.kubernetes.api.model.SecretEnvSource; +import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder; import io.fabric8.kubernetes.client.KubernetesClient; import lombok.AllArgsConstructor; @@ -30,6 +28,7 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) final String jobName = "job-" + job.getId(); deleteEphemeralJob(job); log.info("Ephemeral Executor Image {}, Job: {}, Namespace: {}, NodeSelector: {}", ephemeralConfiguration.getImage(), jobName, ephemeralConfiguration.getNamespace(), ephemeralConfiguration.getNodeSelector()); + SecretEnvSource secretEnvSource = new SecretEnvSource(); secretEnvSource.setName(ephemeralConfiguration.getSecret()); EnvFromSource envFromSource = new EnvFromSource(); @@ -49,7 +48,6 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) } catch (Exception e) { log.error(e.getMessage()); } - final List executorEnvVarFlags = Arrays.asList(executorFlagBatch, executorFlagBatchJsonContent); Optional nodeSelector = Optional.ofNullable(executorContext.getEnvironmentVariables().containsKey(NODE_SELECTOR) ? executorContext.getEnvironmentVariables().get(NODE_SELECTOR) : null); @@ -80,6 +78,31 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) executorContext.getEnvironmentVariables().containsKey(SERVICE_ACCOUNT) ? executorContext.getEnvironmentVariables().get(SERVICE_ACCOUNT) : null); String serviceAccount = serviceAccountInfo.isPresent() ? serviceAccountInfo.get() : null; + // Volume and VolumeMount for ConfigMap if specified + List volumes = new ArrayList<>(); + List volumeMounts = new ArrayList<>(); + + Optional configMapNameOpt = Optional.ofNullable(executorContext.getEnvironmentVariables().get("CONFIG_MAP_NAME")); + Optional configMapMountPathOpt = Optional.ofNullable(executorContext.getEnvironmentVariables().get("CONFIG_MAP_MOUNT_PATH")); + if (configMapNameOpt.isPresent()) { + String configMapName = configMapNameOpt.get(); + String mountPath = configMapMountPathOpt.orElse("/etc/config"); // Default mount path if not specified + + // Define ConfigMap volume + Volume configMapVolume = new Volume(); + configMapVolume.setName("config-volume"); + ConfigMapVolumeSource configMapVolumeSource = new ConfigMapVolumeSource(); + configMapVolumeSource.setName(configMapName); + configMapVolume.setConfigMap(configMapVolumeSource); + volumes.add(configMapVolume); + + // Define VolumeMount for the container + VolumeMount configMapMount = new VolumeMount(); + configMapMount.setName("config-volume"); + configMapMount.setMountPath(mountPath); + volumeMounts.add(configMapMount); + } + io.fabric8.kubernetes.api.model.batch.v1.Job k8sJob = new JobBuilder() .withApiVersion("batch/v1") .withNewMetadata() @@ -95,11 +118,13 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) .withNewSpec() .withNodeSelector(nodeSelectorInfo) .withServiceAccountName(serviceAccount) + .withVolumes(volumes) .addNewContainer() .withName(jobName) .withEnvFrom(executorEnvVarFromSecret) .withImage(ephemeralConfiguration.getImage()) .withEnv(executorEnvVarFlags) + .withVolumeMounts(volumeMounts) .endContainer() .withRestartPolicy("Never") .endSpec() From 9dff23f52f519a06e38b1a831eae5854bd88b49a Mon Sep 17 00:00:00 2001 From: Benjamin Decreusefond Date: Fri, 8 Nov 2024 10:01:11 +0100 Subject: [PATCH 2/6] change default path --- .../job/tcl/executor/ephemeral/EphemeralExecutorService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java b/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java index 2ac2ff41e..b135171a5 100644 --- a/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java +++ b/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java @@ -86,7 +86,7 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) Optional configMapMountPathOpt = Optional.ofNullable(executorContext.getEnvironmentVariables().get("CONFIG_MAP_MOUNT_PATH")); if (configMapNameOpt.isPresent()) { String configMapName = configMapNameOpt.get(); - String mountPath = configMapMountPathOpt.orElse("/etc/config"); // Default mount path if not specified + String mountPath = configMapMountPathOpt.orElse("/home/"); // Default mount path if not specified // Define ConfigMap volume Volume configMapVolume = new Volume(); From b224a57d385de473b65a321c1608a47da45598c3 Mon Sep 17 00:00:00 2001 From: Benjamin Decreusefond Date: Fri, 8 Nov 2024 10:42:19 +0100 Subject: [PATCH 3/6] move default path to tmp --- .../job/tcl/executor/ephemeral/EphemeralExecutorService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java b/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java index b135171a5..cca4c58eb 100644 --- a/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java +++ b/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java @@ -86,7 +86,7 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) Optional configMapMountPathOpt = Optional.ofNullable(executorContext.getEnvironmentVariables().get("CONFIG_MAP_MOUNT_PATH")); if (configMapNameOpt.isPresent()) { String configMapName = configMapNameOpt.get(); - String mountPath = configMapMountPathOpt.orElse("/home/"); // Default mount path if not specified + String mountPath = configMapMountPathOpt.orElse("/tmp"); // Default mount path if not specified // Define ConfigMap volume Volume configMapVolume = new Volume(); From 11fb6b76ab63a2c731fa9503dfd06fc10ffa4f9c Mon Sep 17 00:00:00 2001 From: Benjamin Decreusefond Date: Fri, 8 Nov 2024 10:59:58 +0100 Subject: [PATCH 4/6] update message for webhook branch --- ui/src/domain/Workspaces/Create.jsx | 2 +- ui/src/domain/Workspaces/Details.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/domain/Workspaces/Create.jsx b/ui/src/domain/Workspaces/Create.jsx index ab03183d3..7e6a9db18 100644 --- a/ui/src/domain/Workspaces/Create.jsx +++ b/ui/src/domain/Workspaces/Create.jsx @@ -799,7 +799,7 @@ export const CreateWorkspace = () => { name="webhookBranch" label="Webhook Branch" tooltip="A list of branch prefixes that will trigger a run." - extra="A list of brach prefixes besides the default VCS branch that will trigger a run, for example 'feat,fix'. Values are separated by comma." + extra="A list of branch regex besides the default VCS branch that will trigger a run, for example 'feat,fix'. Values are separated by comma." rules={[{ required: false }]} > diff --git a/ui/src/domain/Workspaces/Details.jsx b/ui/src/domain/Workspaces/Details.jsx index d8cdd3f70..cee7f0ecd 100644 --- a/ui/src/domain/Workspaces/Details.jsx +++ b/ui/src/domain/Workspaces/Details.jsx @@ -1478,7 +1478,7 @@ export const WorkspaceDetails = ({ setOrganizationName, selectedTab }) => { name="pushWebhookBranch" label="Webhook Branch" tooltip="A list of branch prefixes that will trigger a run." - extra="A list of brach prefixes besides the default VCS branch that will trigger a run, for example 'feat,fix'. Values are separated by comma." + extra="A list of branch regex besides the default VCS branch that will trigger a run, for example 'feat,fix'. Values are separated by comma." rules={[{ required: false }]} > From 22fd2b0971b7725d3eadacc253255ea198747b48 Mon Sep 17 00:00:00 2001 From: Benjamin Decreusefond Date: Fri, 8 Nov 2024 11:48:26 +0100 Subject: [PATCH 5/6] refacto if statement --- .../tcl/executor/ephemeral/EphemeralExecutorService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java b/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java index cca4c58eb..6c9a0446f 100644 --- a/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java +++ b/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java @@ -50,7 +50,7 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) } final List executorEnvVarFlags = Arrays.asList(executorFlagBatch, executorFlagBatchJsonContent); - Optional nodeSelector = Optional.ofNullable(executorContext.getEnvironmentVariables().containsKey(NODE_SELECTOR) ? executorContext.getEnvironmentVariables().get(NODE_SELECTOR) : null); + Optional nodeSelector = Optional.ofNullable(executorContext.getEnvironmentVariables().getOrDefault(NODE_SELECTOR, null)); Map nodeSelectorInfo = new HashMap(); log.info("Custom Node selector: {} {}", nodeSelector.isPresent(), nodeSelector.isEmpty()); if(nodeSelector.isPresent()) { @@ -64,7 +64,7 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) nodeSelectorInfo = ephemeralConfiguration.getNodeSelector(); } - Optional annotationsInfo = Optional.ofNullable(executorContext.getEnvironmentVariables().containsKey(ANNOTATIONS) ? executorContext.getEnvironmentVariables().get(ANNOTATIONS) : null); + Optional annotationsInfo = Optional.ofNullable(executorContext.getEnvironmentVariables().getOrDefault(ANNOTATIONS, null)); Map annotations = new HashMap(); log.info("Custom Annotations: {}", annotationsInfo.isPresent()); if(annotationsInfo.isPresent()) { @@ -75,8 +75,8 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) } Optional serviceAccountInfo = Optional.ofNullable( - executorContext.getEnvironmentVariables().containsKey(SERVICE_ACCOUNT) ? executorContext.getEnvironmentVariables().get(SERVICE_ACCOUNT) : null); - String serviceAccount = serviceAccountInfo.isPresent() ? serviceAccountInfo.get() : null; + executorContext.getEnvironmentVariables().getOrDefault(SERVICE_ACCOUNT, null)); + String serviceAccount = serviceAccountInfo.orElse(null); // Volume and VolumeMount for ConfigMap if specified List volumes = new ArrayList<>(); From 089cc70328068978409214a1fedf85328f59fbdd Mon Sep 17 00:00:00 2001 From: Benjamin Decreusefond Date: Fri, 8 Nov 2024 11:52:21 +0100 Subject: [PATCH 6/6] use variable instead --- .../tcl/executor/ephemeral/EphemeralExecutorService.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java b/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java index 6c9a0446f..291f21790 100644 --- a/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java +++ b/api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java @@ -20,6 +20,9 @@ public class EphemeralExecutorService { private static final String NODE_SELECTOR = "EPHEMERAL_CONFIG_NODE_SELECTOR_TAGS"; private static final String SERVICE_ACCOUNT = "EPHEMERAL_CONFIG_SERVICE_ACCOUNT"; private static final String ANNOTATIONS = "EPHEMERAL_CONFIG_ANNOTATIONS"; + private static final String CONFIG_MAP_NAME = "EPHEMERAL_CONFIG_MAP_NAME"; + private static final String CONFIG_MAP_PATH = "EPHEMERAL_CONFIG_MAP_MOUNT_PATH"; + KubernetesClient kubernetesClient; EphemeralConfiguration ephemeralConfiguration; @@ -28,7 +31,6 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) final String jobName = "job-" + job.getId(); deleteEphemeralJob(job); log.info("Ephemeral Executor Image {}, Job: {}, Namespace: {}, NodeSelector: {}", ephemeralConfiguration.getImage(), jobName, ephemeralConfiguration.getNamespace(), ephemeralConfiguration.getNodeSelector()); - SecretEnvSource secretEnvSource = new SecretEnvSource(); secretEnvSource.setName(ephemeralConfiguration.getSecret()); EnvFromSource envFromSource = new EnvFromSource(); @@ -48,6 +50,7 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) } catch (Exception e) { log.error(e.getMessage()); } + final List executorEnvVarFlags = Arrays.asList(executorFlagBatch, executorFlagBatchJsonContent); Optional nodeSelector = Optional.ofNullable(executorContext.getEnvironmentVariables().getOrDefault(NODE_SELECTOR, null)); @@ -82,8 +85,8 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) List volumes = new ArrayList<>(); List volumeMounts = new ArrayList<>(); - Optional configMapNameOpt = Optional.ofNullable(executorContext.getEnvironmentVariables().get("CONFIG_MAP_NAME")); - Optional configMapMountPathOpt = Optional.ofNullable(executorContext.getEnvironmentVariables().get("CONFIG_MAP_MOUNT_PATH")); + Optional configMapNameOpt = Optional.ofNullable(executorContext.getEnvironmentVariables().get(CONFIG_MAP_NAME)); + Optional configMapMountPathOpt = Optional.ofNullable(executorContext.getEnvironmentVariables().get(CONFIG_MAP_PATH)); if (configMapNameOpt.isPresent()) { String configMapName = configMapNameOpt.get(); String mountPath = configMapMountPathOpt.orElse("/tmp"); // Default mount path if not specified