BXC-4635 init project from filesystem (#98)

* init project from filesystem

* fix test names, remove unused imports

* update projectSource option, private methods initCdmProject and initFilesProject, createCdmMigrationProject and createFilesMigrationProject, update tests

* fix tests

* modify createFilesMigrationProject, add constants for project source cdm and files, fix/add test, add error message

* remove constants, modify test

src/main/java/edu/unc/lib/boxc/migration/cdm/InitializeProjectCommand.java

@@ -53,6 +53,12 @@ public class InitializeProjectCommand implements Callable<Integer> {
                 "If not specified, then the project will be initialized in the current directory.",
                 "If the project name is different from the CDM collection ID, then use -c to specify the ID." })
     private String projectName;
+    @Option(names = {"-s", "--source"},
+            description = {"Specify the source of migration data when initializing a new project. Accepted values: " +
+                    "cdm (CDM collection), files (filesystem, doesn't need to be linked with CDM collection)." +
+                    "Defaults to cdm."},
+            defaultValue = "cdm")
+    private String projectSource;
 
     private CdmFieldService fieldService;
     private CloseableHttpClient httpClient;
@@ -87,11 +93,27 @@ public Integer call() throws Exception {
             log.error("Unable to read application configuration", e);
             return 1;
         }
-        var cdmEnvConfig = config.getCdmEnvironments().get(cdmEnvId);
 
         Path currentPath = parentCommand.getWorkingDirectory();
         String projDisplayName = projectName == null ? currentPath.getFileName().toString() : projectName;
+        Integer integer = -1;
+
+        if (projectSource.equalsIgnoreCase(MigrationProject.PROJECT_SOURCE_CDM)) {
+            integer = initCdmProject(config, currentPath, projDisplayName, start);
+        } else if (projectSource.equalsIgnoreCase(MigrationProject.PROJECT_SOURCE_FILES)) {
+            integer = initFilesProject(currentPath, projDisplayName, start);
+        } else {
+            log.error("Invalid project source: {}", projectSource);
+            outputLogger.info("Invalid project source: {}", projectSource);
+        }
+
+        return integer;
+    }
+
+    private Integer initCdmProject(ChompbConfig config, Path currentPath, String projDisplayName, long start) throws Exception {
+        var cdmEnvConfig = config.getCdmEnvironments().get(cdmEnvId);
         String collId = cdmCollectionId == null ? projDisplayName : cdmCollectionId;
+        String username = System.getProperty("user.name");
 
         // Retrieve field information from CDM
         CdmFieldInfo fieldInfo;
@@ -105,12 +127,10 @@ public Integer call() throws Exception {
             return 1;
         }
 
-        String username = System.getProperty("user.name");
-
         // Instantiate the project
-        MigrationProject project = null;
+        MigrationProject project;
         try {
-            project = MigrationProjectFactory.createMigrationProject(
+            project = MigrationProjectFactory.createCdmMigrationProject(
                     currentPath, projectName, cdmCollectionId, username, cdmEnvId, bxcEnvId);
 
             // Persist field info to the project
@@ -134,4 +154,18 @@ public Integer call() throws Exception {
         outputLogger.info("Initialized project {} in {}s", projDisplayName, (System.nanoTime() - start) / 1e9);
         return 0;
     }
+
+    private Integer initFilesProject(Path currentPath, String projDisplayName, long start) throws Exception {
+        String username = System.getProperty("user.name");
+
+        try {
+            MigrationProjectFactory.createFilesMigrationProject(currentPath, projectName, username,  bxcEnvId);
+        } catch (InvalidProjectStateException e) {
+            outputLogger.info("Failed to initialize project {}: {}", projDisplayName, e.getMessage());
+            return 1;
+        }
+
+        outputLogger.info("Initialized project {} in {}s", projDisplayName, (System.nanoTime() - start) / 1e9);
+        return 0;
+    }
 }

src/main/java/edu/unc/lib/boxc/migration/cdm/model/MigrationProject.java

@@ -25,6 +25,8 @@ public class MigrationProject {
     public static final String REDIRECT_MAPPING_FILENAME = "redirect_mappings.csv";
     public static final String POST_MIGR_REPORT_FILENAME = "post_migration_report.csv";
     public static final String PERMISSIONS_FILENAME = "patron_permissions.csv";
+    public static final String PROJECT_SOURCE_CDM = "cdm";
+    public static final String PROJECT_SOURCE_FILES = "files";
 
     private Path projectPath;
     private MigrationProjectProperties properties;

src/main/java/edu/unc/lib/boxc/migration/cdm/model/MigrationProjectProperties.java

@@ -28,6 +28,7 @@ public class MigrationProjectProperties {
     private String collectionNumber;
     private String cdmEnvironmentId;
     private String bxcEnvironmentId;
+    private String projectSource;
 
     public MigrationProjectProperties() {
         sipsSubmitted = new HashSet<>();
@@ -230,4 +231,12 @@ public String getBxcEnvironmentId() {
     public void setBxcEnvironmentId(String bxcEnvironmentId) {
         this.bxcEnvironmentId = bxcEnvironmentId;
     }
+
+    public String getProjectSource() {
+        return projectSource;
+    }
+
+    public void setProjectSource(String projectSource) {
+        this.projectSource = projectSource;
+    }
 }

src/main/java/edu/unc/lib/boxc/migration/cdm/services/MigrationProjectFactory.java

@@ -21,10 +21,17 @@ public class MigrationProjectFactory {
     private MigrationProjectFactory() {
     }
 
-    public static MigrationProject createMigrationProject(Path path, String name,
-                                                          String collectionId, String user, String cdmEnvId)
+    public static MigrationProject createCdmMigrationProject(Path path, String name, String collectionId,
+                                                             String user, String cdmEnvId, String bxcEnvId)
+            throws IOException {
+        return createMigrationProject(path, name, collectionId, user, cdmEnvId, bxcEnvId,
+                MigrationProject.PROJECT_SOURCE_CDM);
+    }
+
+    public static MigrationProject createFilesMigrationProject(Path path, String name, String user, String bxcEnvId)
             throws IOException {
-        return createMigrationProject(path, name, collectionId, user, cdmEnvId, null);
+        return createMigrationProject(path, name, null, user, null, bxcEnvId,
+                MigrationProject.PROJECT_SOURCE_FILES);
     }
 
     /**
@@ -40,7 +47,7 @@ public static MigrationProject createMigrationProject(Path path, String name,
      */
     public static MigrationProject createMigrationProject(Path path, String name,
                                                           String collectionId, String user,
-                                                          String cdmEnvId, String bxcEnvId)
+                                                          String cdmEnvId, String bxcEnvId, String projectSource)
             throws IOException {
         Assert.notNull(path, "Project path not set");
         Assert.notNull(user, "Username not set");
@@ -73,9 +80,14 @@ public static MigrationProject createMigrationProject(Path path, String name,
         properties.setCreator(user);
         properties.setCreatedDate(Instant.now());
         properties.setName(projectName);
-        properties.setCdmCollectionId(collectionId == null ? projectName : collectionId);
+        if (projectSource.equalsIgnoreCase(MigrationProject.PROJECT_SOURCE_CDM)) {
+            properties.setCdmCollectionId(collectionId == null ? projectName : collectionId);
+        } else {
+            properties.setCdmCollectionId(null);
+        }
         properties.setCdmEnvironmentId(cdmEnvId);
         properties.setBxcEnvironmentId(bxcEnvId);
+        properties.setProjectSource(projectSource);
         project.setProjectProperties(properties);
         ProjectPropertiesSerialization.write(propertiesPath, properties);
 

src/test/java/edu/unc/lib/boxc/migration/cdm/AbstractCommandIT.java

@@ -52,7 +52,8 @@ protected void initTestHelper() throws IOException {
 
     protected void initProject() throws IOException {
         project = MigrationProjectFactory.createMigrationProject(baseDir, PROJECT_ID, defaultCollectionId, USERNAME,
-                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
+                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID,
+                MigrationProject.PROJECT_SOURCE_CDM);
     }
 
     protected void initProjectAndHelper() throws IOException {

src/test/java/edu/unc/lib/boxc/migration/cdm/CdmExportCommandIT.java

@@ -6,6 +6,7 @@
 import edu.unc.lib.boxc.migration.cdm.services.CdmFieldService;
 import edu.unc.lib.boxc.migration.cdm.services.CdmFileRetrievalService;
 import edu.unc.lib.boxc.migration.cdm.services.MigrationProjectFactory;
+import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.TestSshServer;
 import org.apache.commons.io.FileUtils;
@@ -182,8 +183,9 @@ private Path createProject() throws Exception {
     }
 
     private Path createProject(String collId) throws Exception {
-        MigrationProject project = MigrationProjectFactory.createMigrationProject(
-                baseDir, collId, null, USERNAME, CdmEnvironmentHelper.DEFAULT_ENV_ID);
+        MigrationProject project = MigrationProjectFactory.createCdmMigrationProject(
+                baseDir, collId, null, USERNAME,
+                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
         CdmFieldInfo fieldInfo = new CdmFieldInfo();
         CdmFieldEntry fieldEntry = new CdmFieldEntry();
         fieldEntry.setNickName("title");

src/test/java/edu/unc/lib/boxc/migration/cdm/InitializeProjectCommandIT.java

@@ -19,8 +19,6 @@
 import java.util.List;
 import java.util.Optional;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import edu.unc.lib.boxc.migration.cdm.services.ChompbConfigService;
 import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import org.apache.commons.io.IOUtils;
@@ -105,7 +103,7 @@ public void initValidProjectUsingCurrentDirTest() throws Exception {
     }
 
     @Test
-    public void initCdmCollectioNotFoundTest() throws Exception {
+    public void initCdmCollectionNotFoundTest() throws Exception {
         String[] initArgs = new String[] {
                 "-w", baseDir.toString(),
                 "--env-config", chompbConfigPath,
@@ -186,6 +184,29 @@ public void initNoEnvMappingPathTest() throws Exception {
         assertProjectDirectoryNotCreate();
     }
 
+    @Test
+    public void initNewProjectFromFilesystemTest() throws Exception {
+        String[] initArgs = new String[] {
+                "-w", baseDir.toString(),
+                "--env-config", chompbConfigPath,
+                "init",
+                "-p", "test_file_project",
+                "-s", "files" };
+        executeExpectSuccess(initArgs);
+
+        MigrationProject project = MigrationProjectFactory.loadMigrationProject(baseDir.resolve("test_file_project"));
+        MigrationProjectProperties properties = project.getProjectProperties();
+        assertEquals(USERNAME, properties.getCreator());
+        assertEquals("test_file_project", properties.getName(), "Project name did not match expected value");
+        assertNull(properties.getCdmCollectionId());
+        assertNotNull(properties.getCreatedDate(), "Created date not set");
+        assertNull(properties.getHookId());
+        assertNull(properties.getCollectionNumber());
+        assertNull(properties.getCdmEnvironmentId());
+        assertEquals(BxcEnvironmentHelper.DEFAULT_ENV_ID, properties.getBxcEnvironmentId());
+        assertTrue(Files.exists(project.getDescriptionsPath()), "Description folder not created");
+    }
+
     private void assertProjectDirectoryNotCreate() throws IOException {
         try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(baseDir)) {
             for (Path path : dirStream) {

src/test/java/edu/unc/lib/boxc/migration/cdm/services/AggregateFileMappingServiceTest.java

@@ -6,6 +6,7 @@
 import edu.unc.lib.boxc.migration.cdm.options.AggregateFileMappingOptions;
 import edu.unc.lib.boxc.migration.cdm.options.GroupMappingOptions;
 import edu.unc.lib.boxc.migration.cdm.options.GroupMappingSyncOptions;
+import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.SipServiceHelper;
 import org.junit.jupiter.api.BeforeEach;
@@ -38,8 +39,9 @@ public class AggregateFileMappingServiceTest {
 
     @BeforeEach
     public void init() throws Exception {
-        project = MigrationProjectFactory.createMigrationProject(
-                tmpFolder, PROJECT_NAME, null, "user", CdmEnvironmentHelper.DEFAULT_ENV_ID);
+        project = MigrationProjectFactory.createCdmMigrationProject(
+                tmpFolder, PROJECT_NAME, null, "user",
+                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
         Files.createDirectories(project.getExportPath());
 
         basePath = tmpFolder.resolve("testFolder");

src/test/java/edu/unc/lib/boxc/migration/cdm/services/ArchivalDestinationsServiceTest.java

@@ -70,8 +70,8 @@ public class ArchivalDestinationsServiceTest {
     public void setup() throws Exception {
         closeable = openMocks(this);
         project = MigrationProjectFactory.createMigrationProject(
-                tmpFolder, PROJECT_NAME, null, "user",
-                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
+                tmpFolder, PROJECT_NAME, null, "user", CdmEnvironmentHelper.DEFAULT_ENV_ID,
+                BxcEnvironmentHelper.DEFAULT_ENV_ID, MigrationProject.PROJECT_SOURCE_CDM);
         testHelper = new SipServiceHelper(project, tmpFolder);
         service = new ArchivalDestinationsService();
         service.setProject(project);

src/test/java/edu/unc/lib/boxc/migration/cdm/services/CdmExportServiceTest.java

@@ -6,6 +6,7 @@
 import edu.unc.lib.boxc.migration.cdm.model.MigrationProject;
 import edu.unc.lib.boxc.migration.cdm.options.CdmExportOptions;
 import edu.unc.lib.boxc.migration.cdm.services.export.ExportStateService;
+import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
@@ -53,8 +54,9 @@ public class CdmExportServiceTest {
     @BeforeEach
     public void setup() throws Exception {
         closeable = openMocks(this);
-        project = MigrationProjectFactory.createMigrationProject(
-                tmpFolder, PROJECT_NAME, null, "user", CdmEnvironmentHelper.DEFAULT_ENV_ID);
+        project = MigrationProjectFactory.createCdmMigrationProject(
+                tmpFolder, PROJECT_NAME, null, "user",
+                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
         fieldService = new CdmFieldService();
         exportStateService = new ExportStateService();
         exportStateService.setProject(project);

src/test/java/edu/unc/lib/boxc/migration/cdm/services/CdmFieldServiceTest.java

@@ -15,6 +15,7 @@
 import java.util.List;
 import java.util.Optional;
 
+import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import org.apache.commons.io.FileUtils;
 import org.apache.http.HttpEntity;
@@ -56,8 +57,9 @@ public class CdmFieldServiceTest {
     @BeforeEach
     public void setup() throws Exception {
         closeable = openMocks(this);
-        project = MigrationProjectFactory.createMigrationProject(
-                tmpFolder, PROJECT_NAME, null, "user", CdmEnvironmentHelper.DEFAULT_ENV_ID);
+        project = MigrationProjectFactory.createCdmMigrationProject(
+                tmpFolder, PROJECT_NAME, null, "user",
+                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
         service = new CdmFieldService();
         service.setHttpClient(httpClient);
         service.setCdmBaseUri(CDM_BASE_URL);

src/test/java/edu/unc/lib/boxc/migration/cdm/services/CdmIndexServiceTest.java

@@ -5,11 +5,10 @@
 import edu.unc.lib.boxc.migration.cdm.model.CdmFieldInfo;
 import edu.unc.lib.boxc.migration.cdm.model.MigrationProject;
 import edu.unc.lib.boxc.migration.cdm.model.MigrationProjectProperties;
+import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.util.ProjectPropertiesSerialization;
 import org.apache.commons.io.FileUtils;
-import org.jdom2.output.Format;
-import org.jdom2.output.XMLOutputter;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
@@ -48,8 +47,9 @@ public class CdmIndexServiceTest {
 
     @BeforeEach
     public void setup() throws Exception {
-        project = MigrationProjectFactory.createMigrationProject(
-                tmpFolder, PROJECT_NAME, null, "user", CdmEnvironmentHelper.DEFAULT_ENV_ID);
+        project = MigrationProjectFactory.createCdmMigrationProject(
+                tmpFolder, PROJECT_NAME, null, "user",
+                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
         Files.createDirectories(project.getExportPath());
 
         fieldService = new CdmFieldService();

src/test/java/edu/unc/lib/boxc/migration/cdm/services/DescriptionsServiceTest.java

@@ -17,6 +17,7 @@
 import java.util.Set;
 import java.util.stream.Stream;
 
+import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import org.apache.commons.io.FileUtils;
 import org.jdom2.Document;
@@ -50,7 +51,8 @@ public class DescriptionsServiceTest {
     @BeforeEach
     public void setup() throws Exception {
         basePath = tmpFolder;
-        project = MigrationProjectFactory.createMigrationProject(basePath, PROJECT_NAME, null, "user", CdmEnvironmentHelper.DEFAULT_ENV_ID);
+        project = MigrationProjectFactory.createCdmMigrationProject(basePath, PROJECT_NAME, null,
+                "user", CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
         Files.createDirectories(project.getDescriptionsPath());
         service = new DescriptionsService();
         service.setProject(project);

src/test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java

@@ -6,6 +6,7 @@
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.fail;
 
+import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -35,8 +36,9 @@ public class FieldAssessmentTemplateServiceTest {
 
     @BeforeEach
     public void setup() throws Exception {
-        project = MigrationProjectFactory.createMigrationProject(
-                tmpFolder, PROJECT_NAME, null, "user", CdmEnvironmentHelper.DEFAULT_ENV_ID);
+        project = MigrationProjectFactory.createCdmMigrationProject(
+                tmpFolder, PROJECT_NAME, null, "user",
+                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
         fieldService = new CdmFieldService();
 
         service = new FieldAssessmentTemplateService();

src/test/java/edu/unc/lib/boxc/migration/cdm/services/FieldUrlAssessmentServiceTest.java

@@ -19,6 +19,7 @@
 import java.util.stream.Collectors;
 
 import edu.unc.lib.boxc.migration.cdm.model.CdmFieldInfo;
+import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.SipServiceHelper;
 import org.apache.commons.csv.CSVFormat;
@@ -52,8 +53,9 @@ public class FieldUrlAssessmentServiceTest {
 
     @BeforeEach
     public void setup(WireMockRuntimeInfo wmRuntimeInfo) throws Exception {
-        project = MigrationProjectFactory.createMigrationProject(
-                tmpFolder, PROJECT_NAME, null, "user", CdmEnvironmentHelper.DEFAULT_ENV_ID);
+        project = MigrationProjectFactory.createCdmMigrationProject(
+                tmpFolder, PROJECT_NAME, null, "user",
+                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
         Files.createDirectories(project.getExportPath());
 
         var testHelper = new SipServiceHelper(project, tmpFolder);