BXC-4632 - Permit SIP generation when streaming url but no source

pom.xml

@@ -23,6 +23,7 @@
         <csv.version>1.9.0</csv.version>
         <junit.jupiter.version>5.9.2</junit.jupiter.version>
         <solr.solrj.version>8.10.1</solr.solrj.version>
+        <embedded-redis.version>0.7.3</embedded-redis.version>
     </properties>
 
     <build>
@@ -224,6 +225,7 @@
                     <artifactId>slf4j-simple</artifactId>
                 </exclusion>
             </exclusions>
+            <version>${embedded-redis.version}</version>
         </dependency>
         <dependency>
             <groupId>org.awaitility</groupId>

src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGenerator.java

@@ -50,6 +50,8 @@ public class WorkGenerator {
     // use local streamingUrl property for now because Cdr.streamingUrl only exists in a feature branch
     public static final Property streamingUrl = createProperty(
             "http://cdr.unc.edu/definitions/model#streamingUrl");
+    public static final Property streamingType = createProperty(
+            "http://cdr.unc.edu/definitions/model#streamingType");
     protected PIDMinter pidMinter;
     protected RedirectMappingService redirectMappingService;
     protected SourceFilesInfo sourceFilesInfo;
@@ -147,12 +149,14 @@ protected Path getDescriptionPath(String cdmId, boolean allowMissing) {
     protected SourceFilesInfo.SourceFileMapping getSourceFileMapping(String cdmId) {
         SourceFilesInfo.SourceFileMapping sourceMapping = sourceFilesInfo.getMappingByCdmId(cdmId);
         if (sourceMapping == null || sourceMapping.getSourcePaths() == null) {
-            String message = "Cannot transform object " + cdmId + ", no source file has been mapped";
-            if (options.isForce()) {
-                outputLogger.info(message);
-                throw new SipService.SkipObjectException();
-            } else {
-                throw new InvalidProjectStateException(message);
+            if (!streamingMetadataService.verifyRecordHasStreamingMetadata(cdmId)) {
+                String message = "Cannot transform object " + cdmId + ", no source file has been mapped";
+                if (options.isForce()) {
+                    outputLogger.info(message);
+                    throw new SipService.SkipObjectException();
+                } else {
+                    throw new InvalidProjectStateException(message);
+                }
             }
         }
         return sourceMapping;
@@ -165,9 +169,11 @@ protected Resource makeFileResource(PID fileObjPid, Path sourcePath) {
         workBag.add(fileObjResc);
 
         // Link source file
-        Resource origResc = DepositModelHelpers.addDatastream(fileObjResc, ORIGINAL_FILE);
-        origResc.addLiteral(CdrDeposit.stagingLocation, sourcePath.toUri().toString());
-        origResc.addLiteral(CdrDeposit.label, sourcePath.getFileName().toString());
+        if (sourcePath != null) {
+            Resource origResc = DepositModelHelpers.addDatastream(fileObjResc, ORIGINAL_FILE);
+            origResc.addLiteral(CdrDeposit.stagingLocation, sourcePath.toUri().toString());
+            origResc.addLiteral(CdrDeposit.label, sourcePath.getFileName().toString());
+        }
         return fileObjResc;
     }
 
@@ -247,6 +253,7 @@ protected void addStreamingMetadata(String cdmId, Resource resource) {
             String streamingUrlValue = "https://durastream.lib.unc.edu/player?spaceId=" + duracloudSpace
                     + "&filename=" + streamingFile;
             resource.addProperty(streamingUrl, streamingUrlValue);
+            resource.addProperty(streamingType, "video");
         }
     }
 }

src/test/java/edu/unc/lib/boxc/migration/cdm/CompleteMigrationIT.java

@@ -40,6 +40,7 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.get;
 import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.streamingUrl;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -334,6 +335,92 @@ public void migrateGroupedCollectionTest() throws Exception {
         assertDepositStatusSet(sip);
     }
 
+    @Test
+    public void fileWithStreamingOnlyNoSourceFileTest() throws Exception {
+        mockFieldInfoUrl("cdm_fields_resp.json", COLLECTION_ID);
+
+        String[] argsInit = new String[] {
+                "-w", baseDir.toString(),
+                "--env-config", chompbConfigPath,
+                "init",
+                "-p", COLLECTION_ID,
+                "-e", "test"};
+        executeExpectSuccess(argsInit);
+
+        Path projPath = baseDir.resolve(COLLECTION_ID);
+        MigrationProject project = new MigrationProject(projPath);
+
+        String[] argsExport = new String[] {
+                "-w", projPath.toString(),
+                "--env-config", chompbConfigPath,
+                "export",
+                "-p", TestSshServer.PASSWORD };
+        executeExpectSuccess(argsExport);
+
+        String[] argsIndex = new String[] {
+                "-w", projPath.toString(),
+                "index"};
+        executeExpectSuccess(argsIndex);
+
+        String[] argsDest = new String[] {
+                "-w",  projPath.toString(),
+                "destinations", "generate",
+                "-dd", DEST_UUID};
+        executeExpectSuccess(argsDest);
+
+        testHelper = new SipServiceHelper(project, filesBasePath);
+        Path sourcePath1 = testHelper.addSourceFile("276_182_E.tif");
+        Path sourcePath2 = testHelper.addSourceFile("276_183_E.tif");
+        // No source file item 27, so that it will only have streaming
+
+        String[] argsSource = new String[] {
+                "-w", projPath.toString(),
+                "source_files", "generate",
+                "-b", testHelper.getSourceFilesBasePath().toString(),
+                "-n", "file"};
+        executeExpectSuccess(argsSource);
+
+        Files.copy(Paths.get("src/test/resources/mods_collections/gilmer_mods1.xml"),
+                project.getDescriptionsPath().resolve("gilmer_mods1.xml"));
+        String[] argsDesc = new String[] {
+                "-w", projPath.toString(),
+                "descriptions", "expand" };
+        executeExpectSuccess(argsDesc);
+
+        String[] args = new String[] {
+                "-w", projPath.toString(),
+                "sips", "generate" };
+        executeExpectSuccess(args);
+
+        MigrationSip sip = testHelper.extractSipFromOutput(output);
+
+        DepositDirectoryManager dirManager = testHelper.createDepositDirectoryManager(sip);
+        Model model = testHelper.getSipModel(sip);
+
+        Bag depBag = model.getBag(sip.getDepositPid().getRepositoryPath());
+        List<RDFNode> depBagChildren = depBag.iterator().toList();
+        assertEquals(3, depBagChildren.size());
+
+        Resource workResc1 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-23");
+        testHelper.assertObjectPopulatedInSip(workResc1, dirManager, model, sourcePath1, null, "25");
+        Resource workResc2 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-24");
+        testHelper.assertObjectPopulatedInSip(workResc2, dirManager, model, sourcePath2, null, "26");
+        // Work 3 has no source file, but does have a streaming url
+        Resource workResc3 = testHelper.getResourceByCreateTime(depBagChildren, "2005-12-08");
+        testHelper.assertObjectPopulatedInSip(workResc3, dirManager, model, null, null, "27");
+        Resource fileResc3 = testHelper.getFirstSipFileInWork(workResc3, dirManager, model);
+        assertTrue(fileResc3.hasProperty(streamingUrl));
+
+        String[] argsSubmit = new String[] {
+                "-w", projPath.toString(),
+                "submit",
+                "-g", GROUPS };
+        executeExpectSuccess(argsSubmit);
+
+        initDepositStatusFactory();
+        assertDepositStatusSet(sip);
+    }
+
     private String findFileIdByStagingLocation(Bag workBag, Path stagingLoc) {
         Resource fileResc = testHelper.findChildByStagingLocation(workBag, stagingLoc);
         return PIDs.get(fileResc.getURI()).getId();

src/test/java/edu/unc/lib/boxc/migration/cdm/SipsCommandIT.java

@@ -19,6 +19,7 @@
 import java.nio.file.Path;
 import java.util.List;
 
+import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.streamingUrl;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -222,6 +223,55 @@ public void generateWithChildDescriptionsTest() throws Exception {
         assertChildFileModsPopulated(dirManager, workResc3, "27/original_file");
     }
 
+    @Test
+    public void generateStreamingFileOnlyTest() throws Exception {
+        testHelper.indexExportData("mini_gilmer");
+        testHelper.generateDefaultDestinationsMapping(DEST_UUID, null);
+        testHelper.populateDescriptions("gilmer_mods1.xml");
+        List<Path> stagingLocs = testHelper.populateSourceFiles("276_182_E.tif", "276_183_E.tif", "276_203_E.tif");
+
+        String[] args = new String[] {
+                "-w", project.getProjectPath().toString(),
+                "sips", "generate" };
+        executeExpectSuccess(args);
+
+        MigrationSip sip = extractSipFromOutput();
+
+        DepositDirectoryManager dirManager = testHelper.createDepositDirectoryManager(sip);
+        Model model = testHelper.getSipModel(sip);
+
+        Bag depBag = model.getBag(sip.getDepositPid().getRepositoryPath());
+        List<RDFNode> depBagChildren = depBag.iterator().toList();
+        assertEquals(3, depBagChildren.size());
+
+        Resource workResc1 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-23");
+        testHelper.assertObjectPopulatedInSip(workResc1, dirManager, model, stagingLocs.get(0), null, "25");
+        Bag workResc1Bag = model.getBag(workResc1);
+        List<RDFNode> workResc1Children = workResc1Bag.iterator().toList();
+        assertEquals(1, workResc1Children.size());
+        Resource workResc1FileObj = workResc1Children.get(0).asResource();
+        assertFalse(workResc1FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
+                "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));
+
+        Resource workResc2 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-24");
+        testHelper.assertObjectPopulatedInSip(workResc2, dirManager, model, stagingLocs.get(1), null, "26");
+        Bag workResc2Bag = model.getBag(workResc2);
+        List<RDFNode> workResc2Children = workResc2Bag.iterator().toList();
+        assertEquals(1, workResc2Children.size());
+        Resource workResc2FileObj = workResc2Children.get(0).asResource();
+        assertFalse(workResc2FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
+                "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));
+
+        Resource workResc3 = testHelper.getResourceByCreateTime(depBagChildren, "2005-12-08");
+        testHelper.assertObjectPopulatedInSip(workResc3, dirManager, model, stagingLocs.get(2), null, "27");
+        Bag workResc3Bag = model.getBag(workResc3);
+        List<RDFNode> workResc3Children = workResc3Bag.iterator().toList();
+        assertEquals(1, workResc3Children.size());
+        Resource workResc3FileObj = workResc3Children.get(0).asResource();
+        assertTrue(workResc3FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
+                "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));
+    }
+
     private void assertChildFileModsPopulated(DepositDirectoryManager dirManager, Resource workResc,
                                               String expectedCdmId) throws Exception {
         var workChildPid = retrieveOnlyWorkChildPid(workResc);

src/test/java/edu/unc/lib/boxc/migration/cdm/test/SipServiceHelper.java

@@ -156,19 +156,25 @@ public DepositDirectoryManager createDepositDirectoryManager(MigrationSip sip) {
         return new DepositDirectoryManager(sip.getDepositPid(), project.getSipsPath(), true);
     }
 
-    public void assertObjectPopulatedInSip(Resource objResc, DepositDirectoryManager dirManager, Model depModel,
-            Path stagingPath, Path accessPath, String cdmId) throws Exception {
+    public Resource getFirstSipFileInWork(Resource objResc, DepositDirectoryManager dirManager, Model depModel) {
         assertTrue(objResc.hasProperty(RDF.type, Cdr.Work));
         Bag workBag = depModel.getBag(objResc);
         List<RDFNode> workChildren = workBag.iterator().toList();
         assertEquals(1, workChildren.size());
-        Resource fileObjResc = workChildren.get(0).asResource();
+        return workChildren.get(0).asResource();
+    }
+
+    public void assertObjectPopulatedInSip(Resource objResc, DepositDirectoryManager dirManager, Model depModel,
+            Path stagingPath, Path accessPath, String cdmId) throws Exception {
+        Resource fileObjResc = getFirstSipFileInWork(objResc, dirManager, depModel);
         assertTrue(fileObjResc.hasProperty(RDF.type, Cdr.FileObject));
 
         // Check for source file
-        Resource origResc = fileObjResc.getProperty(CdrDeposit.hasDatastreamOriginal).getResource();
-        assertTrue(origResc.hasLiteral(CdrDeposit.stagingLocation, stagingPath.toUri().toString()));
-        assertTrue(origResc.hasLiteral(CdrDeposit.label, stagingPath.getFileName().toString()));
+        if (stagingPath != null) {
+            Resource origResc = fileObjResc.getProperty(CdrDeposit.hasDatastreamOriginal).getResource();
+            assertTrue(origResc.hasLiteral(CdrDeposit.stagingLocation, stagingPath.toUri().toString()));
+            assertTrue(origResc.hasLiteral(CdrDeposit.label, stagingPath.getFileName().toString()));
+        }
 
         if (accessPath == null) {
             // Verify no access copy