Fixes #39 changed WorkflowToolData to allow optional datasets and upd…

…ated executors to handle this case
ncsa · Nov 14, 2024 · 2fa4b0b · 2fa4b0b
1 parent acd1592
commit 2fa4b0b
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+### Changed
+- WorkflowToolData to allow optional datasets [#39](https://github.com/ncsa/datawolf/issues/39)
+
 
 ## [4.7.0] - 2024-10-29
 

diff --git a/datawolf-core/src/main/java/edu/illinois/ncsa/datawolf/Engine.java b/datawolf-core/src/main/java/edu/illinois/ncsa/datawolf/Engine.java
@@ -45,6 +45,7 @@
 import javax.inject.Named;
 import javax.inject.Singleton;
 
+import edu.illinois.ncsa.datawolf.domain.WorkflowToolData;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -566,17 +567,22 @@ public void run() {
                                 if (local >= (LocalExecutor.getWorkers() + getExtraLocalExecutor())) {
                                     canrun = 1;
                                 } else {
-                                    // Transaction transaction = null;
+                                    // Map of step inputs and whether they are required
+                                    Map<String, Boolean> requiredInputs = new HashMap<>();
                                     try {
                                         unitOfWork.begin();
-                                        // transaction =
-// SpringData.getTransaction();
-                                        // transaction.start();
 
                                         execution = executionDao.findOne(exec.getExecutionId());
                                         execution.getDatasets().values();
                                         step = workflowStepDao.findOne(exec.getStepId());
-                                        step.getInputs().values();
+
+                                        // Populate map of step inputs and whether they are required
+                                        for(String inputKey : step.getInputs().values()) {
+                                            WorkflowToolData inputData = step.getInput(inputKey);
+                                            requiredInputs.put(inputKey, inputData.isAllowNull());
+                                        }
+
+//                                        step.getInputs().values();
                                     } catch (Exception e) {
                                         logger.error("Error getting job information.", e);
                                     } finally {
@@ -594,7 +600,8 @@ public void run() {
                                     if (step != null) {
                                         for (String id : step.getInputs().values()) {
                                             if (execution != null) {
-                                                if (!execution.hasDataset(id)) {
+                                                boolean allowNull = requiredInputs.get(id);
+                                                if (!execution.hasDataset(id) && !allowNull) {
                                                     canrun = 1;
                                                 } else if (execution.getDataset(id) == null) {
                                                     canrun = 2;

diff --git a/datawolf-domain/src/main/java/edu/illinois/ncsa/datawolf/domain/WorkflowToolData.java b/datawolf-domain/src/main/java/edu/illinois/ncsa/datawolf/domain/WorkflowToolData.java
@@ -51,6 +51,9 @@ public class WorkflowToolData extends AbstractBean {
     /** Mime type of the workflow tool data */
     private String            mimeType         = "";                          //$NON-NLS-1$
 
+    /** Can the dataset be null */
+    private boolean           allowNull        = false;
+
     /**
      * Create a new instance of the workflow tool.
      */
@@ -68,7 +71,7 @@ public String getDataId() {
     /**
      * Sets the id of the workflow tool data
      * 
-     * @param id
+     * @param dataid
      *            sets the id of the workflow tool data.
      * 
      */
@@ -136,6 +139,25 @@ public void setMimeType(String mimeType) {
         this.mimeType = mimeType;
     }
 
+    /**
+     * Return true if the dataset is allowed to be empty.
+     *
+     * @return the true if the parameter can be empty.
+     */
+    public boolean isAllowNull() {
+        return allowNull;
+    }
+
+    /**
+     * Sets whether the workflow dataset can be empty.
+     *
+     * @param allowNull
+     *            the allowNull to set
+     */
+    public void setAllowNull(boolean allowNull) {
+        this.allowNull = allowNull;
+    }
+
     @Override
     public String toString() {
         return title + " [" + mimeType + "]";

diff --git a/...ne/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java b/...ne/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java
@@ -159,6 +159,11 @@ public void execute(File cwd) throws AbortException, FailedException {
 
                     if (option.getInputOutput() != InputOutput.OUTPUT) {
                         String key = step.getInputs().get(option.getOptionId());
+                        if (execution.getDataset(key).isEmpty()) {
+                            // No dataset has been set, must be an optional dataset so skip it
+                            break;
+                        }
+
                         Dataset ds = datasetDao.findOne(execution.getDataset(key));// option.getOptionId()));
                         if (ds == null) {
                             throw (new AbortException("Dataset is missing."));

diff --git a/datawolf-executor-hpc/src/main/java/edu/illinois/ncsa/datawolf/executor/hpc/HPCExecutor.java b/datawolf-executor-hpc/src/main/java/edu/illinois/ncsa/datawolf/executor/hpc/HPCExecutor.java
@@ -205,6 +205,10 @@ public State submitRemoteJob(File cwd) throws AbortException, FailedException {
                             String optionId = option.getOptionId();
                             Map<String, String> inputs = step.getInputs();
                             String key = inputs.get(optionId);
+                            if (execution.getDataset(key).isEmpty()) {
+                                // No dataset has been set, must be an optional dataset so skip it
+                                break;
+                            }
                             Dataset ds = datasetDao.findOne(execution.getDataset(key));
                             if (ds == null) {
                                 throw (new AbortException("Dataset is missing."));

diff --git a/...lf-executor-java/src/main/java/edu/illinois/ncsa/datawolf/executor/java/JavaExecutor.java b/...lf-executor-java/src/main/java/edu/illinois/ncsa/datawolf/executor/java/JavaExecutor.java
@@ -149,6 +149,10 @@ public void execute(File cwd) throws AbortException, FailedException {
                 // set inputs
                 if (tool.getInputs() != null) {
                     for (Entry<String, String> entry : step.getInputs().entrySet()) {
+                        if (execution.getDataset(entry.getValue()).isEmpty()) {
+                            // No dataset has been set, must be an optional dataset so skip it
+                            break;
+                        }
                         Dataset ds = datasetDao.findOne(execution.getDataset(entry.getValue()));
                         if (ds == null) {
                             throw (new AbortException("Dataset is missing."));

diff --git a/...etes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java b/...etes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java
@@ -139,6 +139,11 @@ public State submitRemoteJob(File cwd) throws AbortException, FailedException {
 
                     if (option.getInputOutput() != InputOutput.OUTPUT) {
                         String key = step.getInputs().get(option.getOptionId());
+                        if (execution.getDataset(key).isEmpty()) {
+                            // No dataset has been set, must be an optional dataset so skip it
+                            break;
+                        }
+
                         Dataset ds = datasetDao.findOne(execution.getDataset(key));// option.getOptionId()));
                         if (ds == null) {
                             throw (new AbortException("Dataset is missing."));

diff --git a/datawolf-jpa/src/main/resources/META-INF/orm.xml b/datawolf-jpa/src/main/resources/META-INF/orm.xml
@@ -220,6 +220,7 @@
 			<basic name="title" />
 			<basic name="description" />
 			<basic name="mimeType" />
+			<basic name="allowNull" />
 		</attributes>
 	</entity>
 	<entity class="WorkflowToolParameter" name="WorkflowToolParameter">
@@ -236,4 +237,5 @@
 		</attributes>
 	</entity>
 	<!-- how to handle enum types? -->
-</entity-mappings>
+</entity-mappings>
+