BXC-4689 - Source File Processing Command (#111)

* Add command for processing source files, and configuration for boxctron jobs

* Separate ssh host for transfer and running scripts

* Change script path to script directory path so it can be used for multiple jobs, and have job specify the specific filename

* Change user and key to being options exclusively rather than coming from config. Setup a default email address

* Throw error if command fails based on exit status. Add logging and cleanup

* Add umask for directory creation

* Add environment to config for job. Extend timeout for starting job and handling of responses. Add some logging to help track what's happening

* Update test ssh server to respond to sbatch commands, to get IT test working

src/main/java/edu/unc/lib/boxc/migration/cdm/CLIMain.java

@@ -43,7 +43,8 @@
         PermissionsCommand.class,
         ExportObjectsCommand.class,
         ListProjectsCommand.class,
-        ArchiveProjectsCommand.class
+        ArchiveProjectsCommand.class,
+        ProcessSourceFilesCommand.class
     })
 public class CLIMain implements Callable<Integer> {
     @Option(names = { "-w", "--work-dir" },

src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java

@@ -0,0 +1,119 @@
+package edu.unc.lib.boxc.migration.cdm;
+
+import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException;
+import edu.unc.lib.boxc.migration.cdm.jobs.VelocicroptorRemoteJob;
+import edu.unc.lib.boxc.migration.cdm.model.BxcEnvironment;
+import edu.unc.lib.boxc.migration.cdm.model.MigrationProject;
+import edu.unc.lib.boxc.migration.cdm.options.ProcessSourceFilesOptions;
+import edu.unc.lib.boxc.migration.cdm.services.CdmIndexService;
+import edu.unc.lib.boxc.migration.cdm.services.MigrationProjectFactory;
+import edu.unc.lib.boxc.migration.cdm.services.SourceFileService;
+import edu.unc.lib.boxc.migration.cdm.services.SourceFilesToRemoteService;
+import edu.unc.lib.boxc.migration.cdm.util.SshClientService;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import picocli.CommandLine;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+
+import static edu.unc.lib.boxc.migration.cdm.util.CLIConstants.outputLogger;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * @author bbpennel
+ */
+@CommandLine.Command(name = "process_source_files",
+        description = {
+                "Perform a processing job on the source files mapped in this project."})
+public class ProcessSourceFilesCommand implements Callable<Integer> {
+    private static final Logger log = getLogger(ProcessSourceFilesCommand.class);
+    private static final String DEFAULT_EMAIL_DOMAIN = "@ad.unc.edu";
+
+    @CommandLine.ParentCommand
+    private CLIMain parentCommand;
+    private VelocicroptorRemoteJob velocicroptorRemoteJob;
+
+    private MigrationProject project;
+    private BxcEnvironment boxcEnv;
+
+    @CommandLine.Mixin
+    private ProcessSourceFilesOptions options;
+
+    @Override
+    public Integer call() throws Exception {
+        long start = System.nanoTime();
+        try {
+            validateActionName(options.getActionName());
+            loadProjectEnvironment();
+            setDefaultOptions();
+            initialize();
+            velocicroptorRemoteJob.run(options);
+            outputLogger.info("Completed {} job to process source files for {} in {}s",
+                    options.getActionName(), project.getProjectName(), (System.nanoTime() - start) / 1e9);
+            return 0;
+        } catch (MigrationException | IllegalArgumentException e) {
+            outputLogger.info("Source file processing command failed: {}", e.getMessage());
+            log.warn("Source file processing command failed", e);
+            return 1;
+        } catch (Exception e) {
+            log.error("Source file processing command failed", e);
+            outputLogger.info("Source file processing command failed: {}", e.getMessage(), e);
+            return 1;
+        }
+    }
+
+    private void validateActionName(String actionName) {
+        if (!actionName.equals("velocicroptor")) {
+            throw new IllegalArgumentException("Invalid action name provided: " + actionName);
+        }
+    }
+
+    private void setDefaultOptions() {
+        if (options.getEmailAddress() == null) {
+            options.setEmailAddress(options.getUsername() + DEFAULT_EMAIL_DOMAIN);
+        }
+    }
+
+    private void loadProjectEnvironment() throws IOException {
+        Path currentPath = parentCommand.getWorkingDirectory();
+        project = MigrationProjectFactory.loadMigrationProject(currentPath);
+        var config = parentCommand.getChompbConfig();
+        boxcEnv = config.getBxcEnvironments().get(project.getProjectProperties().getBxcEnvironmentId());
+    }
+
+    private void initialize() throws IOException {
+        // Separate service for executing scripts on the remote server
+        var sshClientScriptService = new SshClientService();
+        sshClientScriptService.setSshHost(boxcEnv.getBoxctronScriptHost());
+        sshClientScriptService.setSshPort(boxcEnv.getBoxctronPort());
+        sshClientScriptService.setSshKeyPath(options.getSshKeyPath());
+        sshClientScriptService.setSshUsername(options.getUsername());
+        sshClientScriptService.initialize();
+        // Separate service for transferring files to the remote server
+        var sshClientTransferService = new SshClientService();
+        sshClientTransferService.setSshHost(boxcEnv.getBoxctronTransferHost());
+        sshClientTransferService.setSshPort(boxcEnv.getBoxctronPort());
+        sshClientTransferService.setSshKeyPath(options.getSshKeyPath());
+        sshClientTransferService.setSshUsername(options.getUsername());
+        sshClientTransferService.initialize();
+        var cdmIndexService = new CdmIndexService();
+        cdmIndexService.setProject(project);
+        var sourceFileService = new SourceFileService();
+        sourceFileService.setProject(project);
+        sourceFileService.setIndexService(cdmIndexService);
+        var sourceFilesToRemoteService = new SourceFilesToRemoteService();
+        sourceFilesToRemoteService.setSourceFileService(sourceFileService);
+        sourceFilesToRemoteService.setSshClientService(sshClientTransferService);
+        velocicroptorRemoteJob = new VelocicroptorRemoteJob();
+        velocicroptorRemoteJob.setProject(project);
+        velocicroptorRemoteJob.setSshClientService(sshClientScriptService);
+        velocicroptorRemoteJob.setOutputServer(boxcEnv.getBoxctronOutputServer());
+        velocicroptorRemoteJob.setOutputPath(boxcEnv.getBoxctronOutputBasePath());
+        velocicroptorRemoteJob.setRemoteProjectsPath(boxcEnv.getBoxctronRemoteProjectsPath());
+        velocicroptorRemoteJob.setAdminEmail(boxcEnv.getBoxctronAdminEmail());
+        velocicroptorRemoteJob.setRemoteJobScriptsPath(boxcEnv.getBoxctronRemoteJobScriptsPath());
+        velocicroptorRemoteJob.setSourceFilesToRemoteService(sourceFilesToRemoteService);
+    }
+}

src/main/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJob.java

@@ -6,6 +6,7 @@
 import edu.unc.lib.boxc.migration.cdm.options.ProcessSourceFilesOptions;
 import edu.unc.lib.boxc.migration.cdm.services.SourceFilesToRemoteService;
 import edu.unc.lib.boxc.migration.cdm.util.SshClientService;
+import org.slf4j.Logger;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -16,21 +17,25 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.slf4j.LoggerFactory.getLogger;
+
 /**
  * Job which prepares and executes a remote velocicroptor job to crop color bars from images
  * @author bbpennel
  */
 public class VelocicroptorRemoteJob {
+    private static final Logger log = getLogger(VelocicroptorRemoteJob.class);
     protected static final String RESULTS_REL_PATH = "processing/results/velocicroptor";
     private static final String JOB_ID_PATTERN_FORMAT = "ddMMyyyyHHmmssSSS";
     private static final DateTimeFormatter JOB_ID_FORMATTER = DateTimeFormatter.ofPattern(JOB_ID_PATTERN_FORMAT)
             .withZone(ZoneId.systemDefault());
+    private static final String JOB_FILENAME = "velocicroptor_job.sh";
 
     private SshClientService sshClientService;
     private MigrationProject project;
     private SourceFilesToRemoteService sourceFilesToRemoteService;
     private Path remoteProjectsPath;
-    private Path remoteJobScriptPath;
+    private Path remoteJobScriptsPath;
     private String adminEmail;
     private String outputServer;
     private Path outputPath;
@@ -61,7 +66,11 @@ public String run(ProcessSourceFilesOptions options) {
             String configJson = mapper.writeValueAsString(config);
 
             // Trigger remote job, passing config as argument
-            sshClientService.executeRemoteCommand("sbatch " + remoteJobScriptPath.toString() + " '" + configJson + "'");
+            var scriptPath = remoteJobScriptsPath.resolve(JOB_FILENAME).toAbsolutePath();
+            var sbatchCommand = "sbatch " + scriptPath + " '" + configJson + "'";
+            log.info("Executing remote job with command: {}", sbatchCommand);
+            var response = sshClientService.executeRemoteCommand("sbatch " + scriptPath + " '" + configJson + "'");
+            log.info("Job submitted with response: {}", response);
         } catch (IOException e) {
             throw new MigrationException(e);
         }
@@ -73,6 +82,7 @@ private Map<String, String> createJobConfig(ProcessSourceFilesOptions options, I
         config.put("job_id", jobId);
         config.put("job_name", options.getActionName());
         config.put("chompb_proj_name", project.getProjectName());
+        config.put("environment", project.getProjectProperties().getBxcEnvironmentId());
         config.put("admin_address", adminEmail);
         // User that initiated the job
         config.put("username", options.getUsername());
@@ -100,8 +110,8 @@ public void setRemoteProjectsPath(Path remoteProjectsPath) {
         this.remoteProjectsPath = remoteProjectsPath;
     }
 
-    public void setRemoteJobScriptPath(Path remoteJobScriptPath) {
-        this.remoteJobScriptPath = remoteJobScriptPath;
+    public void setRemoteJobScriptsPath(Path remoteJobScriptsPath) {
+        this.remoteJobScriptsPath = remoteJobScriptsPath;
     }
 
     public void setAdminEmail(String adminEmail) {

src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java

@@ -1,5 +1,7 @@
 package edu.unc.lib.boxc.migration.cdm.model;
 
+import java.nio.file.Path;
+
 /**
  * Configuration information for a Box-c environment
  *
@@ -9,6 +11,15 @@ public class BxcEnvironment {
     private String httpBaseUrl;
     private String solrServerUrl;
 
+    private String boxctronScriptHost;
+    private String boxctronTransferHost;
+    private int boxctronPort;
+    private Path boxctronRemoteProjectsPath;
+    private String boxctronAdminEmail;
+    private String boxctronOutputServer;
+    private Path boxctronOutputBasePath;
+    private Path boxctronRemoteJobScriptsPath;
+
     public String getHttpBaseUrl() {
         return httpBaseUrl;
     }
@@ -24,4 +35,68 @@ public String getSolrServerUrl() {
     public void setSolrServerUrl(String solrServerUrl) {
         this.solrServerUrl = solrServerUrl;
     }
+
+    public String getBoxctronScriptHost() {
+        return boxctronScriptHost;
+    }
+
+    public void setBoxctronScriptHost(String boxctronScriptHost) {
+        this.boxctronScriptHost = boxctronScriptHost;
+    }
+
+    public String getBoxctronTransferHost() {
+        return boxctronTransferHost;
+    }
+
+    public void setBoxctronTransferHost(String boxctronTransferHost) {
+        this.boxctronTransferHost = boxctronTransferHost;
+    }
+
+    public int getBoxctronPort() {
+        return boxctronPort;
+    }
+
+    public void setBoxctronPort(int boxctronPort) {
+        this.boxctronPort = boxctronPort;
+    }
+
+    public Path getBoxctronRemoteProjectsPath() {
+        return boxctronRemoteProjectsPath;
+    }
+
+    public void setBoxctronRemoteProjectsPath(Path boxctronRemoteProjectsPath) {
+        this.boxctronRemoteProjectsPath = boxctronRemoteProjectsPath;
+    }
+
+    public String getBoxctronAdminEmail() {
+        return boxctronAdminEmail;
+    }
+
+    public void setBoxctronAdminEmail(String boxctronAdminEmail) {
+        this.boxctronAdminEmail = boxctronAdminEmail;
+    }
+
+    public String getBoxctronOutputServer() {
+        return boxctronOutputServer;
+    }
+
+    public void setBoxctronOutputServer(String boxctronOutputServer) {
+        this.boxctronOutputServer = boxctronOutputServer;
+    }
+
+    public Path getBoxctronOutputBasePath() {
+        return boxctronOutputBasePath;
+    }
+
+    public void setBoxctronOutputBasePath(Path boxctronOutputBasePath) {
+        this.boxctronOutputBasePath = boxctronOutputBasePath;
+    }
+
+    public Path getBoxctronRemoteJobScriptsPath() {
+        return boxctronRemoteJobScriptsPath;
+    }
+
+    public void setBoxctronRemoteJobScriptsPath(Path boxctronRemoteJobScriptsPath) {
+        this.boxctronRemoteJobScriptsPath = boxctronRemoteJobScriptsPath;
+    }
 }

src/main/java/edu/unc/lib/boxc/migration/cdm/options/ProcessSourceFilesOptions.java

@@ -2,6 +2,8 @@
 
 import picocli.CommandLine;
 
+import java.nio.file.Path;
+
 /**
  * Options for job to process source files
  * @author bbpennel
@@ -16,6 +18,11 @@ public class ProcessSourceFilesOptions {
             defaultValue = "${sys:user.name}")
     private String username;
 
+    @CommandLine.Option(names = {"-k", "--ssh-key"},
+            description = "Path to the ssh key to use for the remote server.",
+            required = true)
+    private Path sshKeyPath;
+
     @CommandLine.Option(names = {"-e", "--email"},
             description = "Email of the user that started this job")
     private String emailAddress;
@@ -36,6 +43,14 @@ public void setUsername(String username) {
         this.username = username;
     }
 
+    public Path getSshKeyPath() {
+        return sshKeyPath;
+    }
+
+    public void setSshKeyPath(Path sshKeyPath) {
+        this.sshKeyPath = sshKeyPath;
+    }
+
     public String getEmailAddress() {
         return emailAddress;
     }