feat: Adding support to handle variable collection

api/pom.xml

@@ -28,7 +28,7 @@
         <mysql.version>9.1.0</mysql.version>
         <mockserver-spring-test-listener.version>5.15.0</mockserver-spring-test-listener.version>
         <snakeyaml.version>2.3</snakeyaml.version>
-        <quartz.version>2.5.0-rc1</quartz.version>
+        <quartz.version>2.5.0-rc2</quartz.version>
         <aws-sdk.version>2.29.6</aws-sdk.version>
         <gcp-libraries-bom.version>26.50.0</gcp-libraries-bom.version>
         <jjwt.version>0.11.5</jjwt.version>

api/src/main/java/org/terrakube/api/plugin/proxy/ProxyService.java

@@ -108,7 +108,7 @@ public void fetchWorkspaceVars(UUID workspaceId) {
         List<Globalvar> globalVariables = globalVarRepository.findByOrganization(organization);
         globalVariables.forEach(globalvar -> VARS.put(globalvar.getKey(), globalvar.getValue()));
 
-        List<Variable> variables = variableRepository.findByWorkspace(workspace);
+        List<Variable> variables = variableRepository.findByWorkspace(workspace).orElse(new ArrayList<>());
         variables.forEach(variable -> VARS.put(variable.getKey(), variable.getValue()));
     }
 

api/src/main/java/org/terrakube/api/plugin/scheduler/ScheduleJob.java

@@ -167,7 +167,7 @@ private void executePendingJob(Job job, JobExecutionContext jobExecutionContext)
                 case terraformApply:
                 case terraformDestroy:
                 case customScripts:
-                    if (executorService.execute(job, stepId, flow.get()))
+                    if (executorService.execute(job, stepId, flow.get()) != null)
                         log.info("Executing Job {} Step Id {}", job.getId(), stepId);
                     else {
                         log.error("Error when sending context to executor marking job {} as failed, step count {}", job.getId(), job.getStep().size());
@@ -304,7 +304,7 @@ private void executeApprovedJobs(Job job) {
             String stepId = tclService.getCurrentStepId(job);
             job.setApprovalTeam("");
             jobRepository.save(job);
-            if (executorService.execute(job, stepId, flow.get()))
+            if (executorService.execute(job, stepId, flow.get()) != null)
                 log.info("Executing Job {} Step Id {}", job.getId(), stepId);
         }
     }

api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ExecutorService.java

@@ -3,11 +3,8 @@
 import java.net.URISyntaxException;
 import java.security.NoSuchAlgorithmException;
 import java.security.spec.InvalidKeySpecException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
+import java.util.*;
+import java.util.stream.Collectors;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -20,10 +17,10 @@
 import org.terrakube.api.plugin.scheduler.job.tcl.model.Flow;
 import org.terrakube.api.plugin.token.dynamic.DynamicCredentialsService;
 import org.terrakube.api.plugin.vcs.TokenService;
-import org.terrakube.api.repository.GlobalVarRepository;
-import org.terrakube.api.repository.JobRepository;
-import org.terrakube.api.repository.SshRepository;
-import org.terrakube.api.repository.VcsRepository;
+import org.terrakube.api.repository.*;
+import org.terrakube.api.rs.collection.Collection;
+import org.terrakube.api.rs.collection.Reference;
+import org.terrakube.api.rs.collection.item.Item;
 import org.terrakube.api.rs.globalvar.Globalvar;
 import org.terrakube.api.rs.job.Job;
 import org.terrakube.api.rs.job.JobStatus;
@@ -63,11 +60,16 @@ public class ExecutorService {
 
     @Autowired
     EphemeralExecutorService ephemeralExecutorService;
+
     @Autowired
     TokenService tokenService;
+    @Autowired
+    private VariableRepository variableRepository;
+    @Autowired
+    private ReferenceRepository referenceRepository;
 
     @Transactional
-    public boolean execute(Job job, String stepId, Flow flow) {
+    public ExecutorContext execute(Job job, String stepId, Flow flow) {
         log.info("Pending Job: {} WorkspaceId: {}", job.getId(), job.getWorkspace().getId());
 
         ExecutorContext executorContext = new ExecutorContext();
@@ -112,19 +114,18 @@ public boolean execute(Job job, String stepId, Flow flow) {
         environmentVariables.put("TF_IN_AUTOMATION", "1");
         environmentVariables.put("workspaceName", job.getWorkspace().getName());
         environmentVariables.put("organizationName", job.getOrganization().getName());
-        List<Variable> variableList = job.getWorkspace().getVariable();
-        if (variableList != null)
-            for (Variable variable : variableList) {
-                if (variable.getCategory().equals(Category.TERRAFORM)) {
-                    log.info("Adding terraform");
-                    terraformVariables.put(variable.getKey(), variable.getValue());
-                } else {
-                    log.info("Adding environment variable");
-                    environmentVariables.put(variable.getKey(), variable.getValue());
-                }
-                log.info("Variable Key: {} Value {}", variable.getKey(),
-                        variable.isSensitive() ? "sensitive" : variable.getValue());
+        List<Variable> variableList = variableRepository.findByWorkspace(job.getWorkspace()).orElse(new ArrayList<>());
+        for (Variable variable : variableList) {
+            if (variable.getCategory().equals(Category.TERRAFORM)) {
+                log.info("Adding terraform");
+                terraformVariables.put(variable.getKey(), variable.getValue());
+            } else {
+                log.info("Adding environment variable");
+                environmentVariables.put(variable.getKey(), variable.getValue());
             }
+            log.info("Variable Key: {} Value {}", variable.getKey(),
+                    variable.isSensitive() ? "sensitive" : variable.getValue());
+        }
 
         environmentVariables = loadOtherEnvironmentVariables(job, flow, environmentVariables);
         terraformVariables = loadOtherTerraformVariables(job, flow, terraformVariables);
@@ -181,7 +182,7 @@ private boolean iacType(Job job) {
                 : true;
     }
 
-    private boolean sendToExecutor(Job job, ExecutorContext executorContext) {
+    private ExecutorContext sendToExecutor(Job job, ExecutorContext executorContext) {
         RestTemplate restTemplate = new RestTemplate();
         boolean executed = false;
         try {
@@ -205,7 +206,7 @@ private boolean sendToExecutor(Job job, ExecutorContext executorContext) {
             executed = false;
         }
 
-        return executed;
+        return executed ? executorContext : null;
     }
 
     private HashMap<String, String> loadOtherEnvironmentVariables(Job job, Flow flow,
@@ -302,14 +303,39 @@ private HashMap<String, String> loadInputData(Job job, Category categoryVar, Has
     }
 
     private HashMap<String, String> loadDefault(Job job, Category category, HashMap<String, String> workspaceData) {
-        if (job.getOrganization().getGlobalvar() != null)
-            for (Globalvar globalvar : job.getOrganization().getGlobalvar()) {
-                if (globalvar.getCategory().equals(category)) {
-                    workspaceData.putIfAbsent(globalvar.getKey(), globalvar.getValue());
-                    log.info("Adding {} Variable Key: {} Value {}", category, globalvar.getKey(),
-                            globalvar.isSensitive() ? "sensitive" : globalvar.getValue());
+        for (Globalvar globalvar : globalVarRepository.findByOrganization(job.getOrganization())) {
+            if (globalvar.getCategory().equals(category)) {
+                workspaceData.putIfAbsent(globalvar.getKey(), globalvar.getValue());
+                log.info("Adding {} Global Variable Key: {} Value {}", category, globalvar.getKey(),
+                        globalvar.isSensitive() ? "sensitive" : globalvar.getValue());
+            }
+        }
+
+        List<Reference> referenceList = referenceRepository.findByWorkspace(job.getWorkspace()).orElse(new ArrayList<>());
+
+        List<Collection> collectionList = new ArrayList();
+        for (Reference reference : referenceList) {
+            collectionList.add(reference.getCollection());
+        }
+
+        List<Collection> sortedList = collectionList.stream()
+                .sorted(Comparator.comparing(Collection::getPriority).reversed())
+                .toList();
+
+        sortedList.stream().forEach(collection -> {
+            log.info("Adding data from collection {} using priority {}", collection.getName(), collection.getPriority());
+
+            List<Item> itemList = new ArrayList();
+            itemList = collection.getItem();
+            for (Item item : itemList) {
+                if (item.getCategory().equals(category)) {
+                    workspaceData.putIfAbsent(item.getKey(), item.getValue());
                 }
             }
+
+        });
+
+
         return workspaceData;
     }
 

api/src/main/java/org/terrakube/api/plugin/scheduler/job/tcl/executor/ephemeral/EphemeralExecutorService.java

@@ -26,7 +26,7 @@ public class EphemeralExecutorService {
     KubernetesClient kubernetesClient;
     EphemeralConfiguration ephemeralConfiguration;
 
-    public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext) {
+    public ExecutorContext sendToEphemeralExecutor(Job job, ExecutorContext executorContext) {
         final String jobName = "job-" + job.getId();
         deleteEphemeralJob(job);
         log.info("Ephemeral Executor Image {}, Job: {}, Namespace: {}, NodeSelector: {}", ephemeralConfiguration.getImage(), jobName, ephemeralConfiguration.getNamespace(), ephemeralConfiguration.getNodeSelector());
@@ -108,8 +108,13 @@ public boolean sendToEphemeralExecutor(Job job, ExecutorContext executorContext)
                 .build();
 
         log.info("Running ephemeral job");
-        kubernetesClient.batch().v1().jobs().inNamespace(ephemeralConfiguration.getNamespace()).createOrReplace(k8sJob);
-        return true;
+        try {
+            kubernetesClient.batch().v1().jobs().inNamespace(ephemeralConfiguration.getNamespace()).createOrReplace(k8sJob);
+            return executorContext;
+        } catch (Exception ex) {
+            log.error(ex.getMessage());
+        }
+        return null;
     }
 
     public void deleteEphemeralJob(Job job){

api/src/main/java/org/terrakube/api/repository/AgentRepository.java

@@ -0,0 +1,9 @@
+package org.terrakube.api.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.terrakube.api.rs.agent.Agent;
+
+import java.util.UUID;
+
+public interface AgentRepository extends JpaRepository<Agent, UUID> {
+}

api/src/main/java/org/terrakube/api/repository/CollectionRepository.java

@@ -0,0 +1,9 @@
+package org.terrakube.api.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.terrakube.api.rs.collection.Collection;
+
+import java.util.UUID;
+
+public interface CollectionRepository extends JpaRepository<Collection, UUID> {
+}

api/src/main/java/org/terrakube/api/repository/ReferenceRepository.java

@@ -0,0 +1,14 @@
+package org.terrakube.api.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.terrakube.api.rs.collection.Reference;
+import org.terrakube.api.rs.workspace.Workspace;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+public interface ReferenceRepository extends JpaRepository<Reference, UUID> {
+
+    Optional<List<Reference>> findByWorkspace(Workspace workspace);
+}

api/src/main/java/org/terrakube/api/repository/VariableRepository.java

@@ -5,10 +5,11 @@
 import org.terrakube.api.rs.workspace.Workspace;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.UUID;
 
 public interface VariableRepository extends JpaRepository<Variable, UUID> {
 
-    List<Variable> findByWorkspace(Workspace organization);
+    Optional<List<Variable>> findByWorkspace(Workspace organization);
 
 }

api/src/main/java/org/terrakube/api/rs/Organization.java

@@ -7,6 +7,7 @@
 import org.hibernate.annotations.JdbcTypeCode;
 import org.hibernate.annotations.SQLRestriction;
 import org.terrakube.api.rs.agent.Agent;
+import org.terrakube.api.rs.collection.Collection;
 import org.terrakube.api.rs.globalvar.Globalvar;
 import org.terrakube.api.rs.hooks.organization.OrganizationManageHook;
 import org.terrakube.api.rs.job.Job;
@@ -64,6 +65,10 @@ public class Organization {
     @Column(name = "description")
     private String description;
 
+    @UpdatePermission(expression = "user belongs organization")
+    @OneToMany(mappedBy = "organization")
+    private List<Collection> collection;
+
     @UpdatePermission(expression = "user belongs organization")
     @OneToMany(mappedBy = "organization")
     private List<Workspace> workspace;

api/src/main/java/org/terrakube/api/rs/checks/collection/TeamManageCollection.java

@@ -0,0 +1,46 @@
+package org.terrakube.api.rs.checks.collection;
+
+import com.yahoo.elide.annotation.SecurityCheck;
+import com.yahoo.elide.core.security.ChangeSpec;
+import com.yahoo.elide.core.security.RequestScope;
+import com.yahoo.elide.core.security.checks.OperationCheck;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.terrakube.api.plugin.security.groups.GroupService;
+import org.terrakube.api.plugin.security.user.AuthenticatedUser;
+import org.terrakube.api.rs.checks.module.TeamManageModule;
+import org.terrakube.api.rs.collection.Collection;
+import org.terrakube.api.rs.team.Team;
+
+import java.util.List;
+import java.util.Optional;
+
+@Slf4j
+@SecurityCheck(TeamManageCollection.RULE)
+public class TeamManageCollection extends OperationCheck<Collection> {
+    public static final String RULE = "team manage collection";
+
+    @Autowired
+    AuthenticatedUser authenticatedUser;
+
+    @Autowired
+    GroupService groupService;
+
+    @Override
+    public boolean ok(Collection collection, RequestScope requestScope, Optional<ChangeSpec> optional) {
+        log.debug("team manage collection {}", collection.getId());
+        boolean isServiceAccount = authenticatedUser.isServiceAccount(requestScope.getUser());
+        List<Team> teamList = collection.getOrganization().getTeam();
+        for (Team team : teamList) {
+            if (isServiceAccount){
+                if (groupService.isServiceMember(requestScope.getUser(), team.getName()) && team.isManageCollection() ){
+                    return true;
+                }
+            } else {
+                if (groupService.isMember(requestScope.getUser(), team.getName()) && team.isManageCollection())
+                    return true;
+            }
+        }
+        return false;
+    }
+}

api/src/main/java/org/terrakube/api/rs/checks/collection/TeamViewCollection.java

@@ -0,0 +1,34 @@
+package org.terrakube.api.rs.checks.collection;
+
+import com.yahoo.elide.annotation.SecurityCheck;
+import com.yahoo.elide.core.security.ChangeSpec;
+import com.yahoo.elide.core.security.RequestScope;
+import com.yahoo.elide.core.security.checks.OperationCheck;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.terrakube.api.plugin.security.user.AuthenticatedUser;
+import org.terrakube.api.rs.checks.membership.MembershipService;
+import org.terrakube.api.rs.collection.Collection;
+import org.terrakube.api.rs.team.Team;
+
+import java.util.List;
+import java.util.Optional;
+
+@Slf4j
+@SecurityCheck(TeamViewCollection.RULE)
+public class TeamViewCollection extends OperationCheck<Collection> {
+    public static final String RULE = "team view collection";
+
+    @Autowired
+    AuthenticatedUser authenticatedUser;
+
+    @Autowired
+    MembershipService membershipService;
+
+    @Override
+    public boolean ok(Collection collection, RequestScope requestScope, Optional<ChangeSpec> optional) {
+        log.debug("team view template {}", collection.getId());
+        List<Team> teamList = collection.getOrganization().getTeam();
+        return authenticatedUser.isSuperUser(requestScope.getUser()) ? true : membershipService.checkMembership(requestScope.getUser(), teamList);
+    }
+}

api/src/main/java/org/terrakube/api/rs/checks/collection/UserReadCollection.java

@@ -0,0 +1,30 @@
+package org.terrakube.api.rs.checks.collection;
+
+import com.yahoo.elide.annotation.SecurityCheck;
+import com.yahoo.elide.core.security.ChangeSpec;
+import com.yahoo.elide.core.security.RequestScope;
+import com.yahoo.elide.core.security.checks.OperationCheck;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.terrakube.api.plugin.security.user.AuthenticatedUser;
+import org.terrakube.api.rs.collection.item.Item;
+
+import java.util.Optional;
+
+@Slf4j
+@SecurityCheck(UserReadCollection.RULE)
+public class UserReadCollection extends OperationCheck<Item> {
+
+    public static final String RULE = "user read collection";
+
+    @Autowired
+    AuthenticatedUser authenticatedUser;
+
+    @Override
+    public boolean ok(Item item, RequestScope requestScope, Optional<ChangeSpec> optional) {
+        log.debug("user view collection {}", item.getId());
+        return !item.isSensitive();
+    }
+
+
+}