BXC-4624 add streamingType property (#97)

* add streamingType property

* lowercase property values

* more lowercase

* case insensitive check of file extension

* add original file extension to getStreamingMetadata, add video streaming metadata to test resources, modify tests

* fix tests

* all caps names for constants

src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java

@@ -85,7 +85,10 @@ public String[] getStreamingMetadata(String cdmId) {
             duracloudSpace = DURACLOUD_CLOSED;
         }
 
-        return new String[] {streamingFile, duracloudSpace, STREAMING_HOST};
+        // original file extension (needed for streamingType)
+        String streamingFileExtension = streamingValues[0];
+
+        return new String[] {streamingFile, duracloudSpace, STREAMING_HOST, streamingFileExtension};
     }
 
     private String[] getStreamingFieldValues(String cdmId) {

src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGenerator.java

@@ -18,6 +18,8 @@
 import edu.unc.lib.boxc.model.api.ids.PIDMinter;
 import edu.unc.lib.boxc.model.api.rdf.Cdr;
 import edu.unc.lib.boxc.model.api.rdf.CdrDeposit;
+import edu.unc.lib.boxc.search.api.SearchFieldKey;
+import org.apache.commons.io.FilenameUtils;
 import org.apache.jena.rdf.model.Bag;
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.Property;
@@ -48,9 +50,9 @@
 public class WorkGenerator {
     private static final Logger log = getLogger(WorkGenerator.class);
     // use local streamingUrl property for now because Cdr.streamingUrl only exists in a feature branch
-    public static final Property streamingUrl = createProperty(
+    public static final Property STREAMING_URL = createProperty(
             "http://cdr.unc.edu/definitions/model#streamingUrl");
-    public static final Property streamingType = createProperty(
+    public static final Property STREAMING_TYPE = createProperty(
             "http://cdr.unc.edu/definitions/model#streamingType");
     protected PIDMinter pidMinter;
     protected RedirectMappingService redirectMappingService;
@@ -250,10 +252,16 @@ protected void addStreamingMetadata(String cdmId, Resource resource) {
             String[] streamingMetadata = streamingMetadataService.getStreamingMetadata(cdmId);
             String duracloudSpace = streamingMetadata[1];
             String streamingFile = streamingMetadata[0];
+            String streamingFileOriginalExtension = streamingMetadata[3];
             String streamingUrlValue = "https://durastream.lib.unc.edu/player?spaceId=" + duracloudSpace
                     + "&filename=" + streamingFile;
-            resource.addProperty(streamingUrl, streamingUrlValue);
-            resource.addProperty(streamingType, "video");
+            resource.addProperty(STREAMING_URL, streamingUrlValue);
+            // set streamingType to sound if mp3 and video if mp4 or anything else (for now)
+            if (FilenameUtils.getExtension(streamingFileOriginalExtension).equalsIgnoreCase("mp3")) {
+                resource.addProperty(STREAMING_TYPE, "sound");
+            } else {
+                resource.addProperty(STREAMING_TYPE, "video");
+            }
         }
     }
 }

src/test/java/edu/unc/lib/boxc/migration/cdm/CompleteMigrationIT.java

@@ -7,7 +7,6 @@
 import edu.unc.lib.boxc.migration.cdm.model.MigrationProject;
 import edu.unc.lib.boxc.migration.cdm.model.MigrationSip;
 import edu.unc.lib.boxc.migration.cdm.services.CdmFieldService;
-import edu.unc.lib.boxc.migration.cdm.services.ChompbConfigService;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.SipServiceHelper;
 import edu.unc.lib.boxc.migration.cdm.test.TestSshServer;
@@ -40,7 +39,8 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.get;
 import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
-import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.streamingUrl;
+import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_TYPE;
+import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_URL;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -403,13 +403,18 @@ public void fileWithStreamingOnlyNoSourceFileTest() throws Exception {
 
         Resource workResc1 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-23");
         testHelper.assertObjectPopulatedInSip(workResc1, dirManager, model, sourcePath1, null, "25");
+        // Work 2 has a source file and a streaming url
         Resource workResc2 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-24");
         testHelper.assertObjectPopulatedInSip(workResc2, dirManager, model, sourcePath2, null, "26");
+        Resource fileResc2 = testHelper.getFirstSipFileInWork(workResc2, dirManager, model);
+        assertTrue(fileResc2.hasProperty(STREAMING_URL));
+        assertTrue(fileResc2.hasProperty(STREAMING_TYPE));
         // Work 3 has no source file, but does have a streaming url
         Resource workResc3 = testHelper.getResourceByCreateTime(depBagChildren, "2005-12-08");
         testHelper.assertObjectPopulatedInSip(workResc3, dirManager, model, null, null, "27");
         Resource fileResc3 = testHelper.getFirstSipFileInWork(workResc3, dirManager, model);
-        assertTrue(fileResc3.hasProperty(streamingUrl));
+        assertTrue(fileResc3.hasProperty(STREAMING_URL));
+        assertTrue(fileResc3.hasProperty(STREAMING_TYPE));
 
         String[] argsSubmit = new String[] {
                 "-w", projPath.toString(),

src/test/java/edu/unc/lib/boxc/migration/cdm/SipsCommandIT.java

@@ -19,7 +19,8 @@
 import java.nio.file.Path;
 import java.util.List;
 
-import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.streamingUrl;
+import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_TYPE;
+import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_URL;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -48,8 +49,7 @@ public void generateSourceFileNotMappedTest() throws Exception {
         testHelper.indexExportData("mini_gilmer");
         testHelper.generateDefaultDestinationsMapping(DEST_UUID, null);
         testHelper.populateDescriptions("gilmer_mods1.xml");
-        List<Path> stagingLocs = testHelper.populateSourceFiles("276_182_E.tif", "276_203_E.tif");
-        Files.delete(stagingLocs.get(1));
+        List<Path> stagingLocs = testHelper.populateSourceFiles("276_183_E.tif", "276_203_E.tif");
 
         String[] args = new String[] {
                 "-w", project.getProjectPath().toString(),
@@ -65,7 +65,7 @@ public void generateSourceFileNotMappedTest() throws Exception {
                 "--force" };
         executeExpectSuccess(argsF);
 
-        assertOutputContains("Cannot transform object 26, no source file has been mapped");
+        assertOutputContains("Cannot transform object 25, no source file has been mapped");
 
         MigrationSip sipF = extractSipFromOutput();
 
@@ -76,8 +76,8 @@ public void generateSourceFileNotMappedTest() throws Exception {
         List<RDFNode> depBagChildrenF = depBagF.iterator().toList();
         assertEquals(2, depBagChildrenF.size());
 
-        Resource workResc1F = testHelper.getResourceByCreateTime(depBagChildrenF, "2005-11-23");
-        testHelper.assertObjectPopulatedInSip(workResc1F, dirManagerF, modelF, stagingLocs.get(0), null, "25");
+        Resource workResc2F = testHelper.getResourceByCreateTime(depBagChildrenF, "2005-11-24");
+        testHelper.assertObjectPopulatedInSip(workResc2F, dirManagerF, modelF, stagingLocs.get(0), null, "26");
         Resource workResc3F = testHelper.getResourceByCreateTime(depBagChildrenF, "2005-12-08");
         testHelper.assertObjectPopulatedInSip(workResc3F, dirManagerF, modelF, stagingLocs.get(1), null, "27");
 
@@ -224,7 +224,7 @@ public void generateWithChildDescriptionsTest() throws Exception {
     }
 
     @Test
-    public void generateStreamingFileOnlyTest() throws Exception {
+    public void generateStreamingFilesOnlyTest() throws Exception {
         testHelper.indexExportData("mini_gilmer");
         testHelper.generateDefaultDestinationsMapping(DEST_UUID, null);
         testHelper.populateDescriptions("gilmer_mods1.xml");
@@ -250,26 +250,30 @@ public void generateStreamingFileOnlyTest() throws Exception {
         List<RDFNode> workResc1Children = workResc1Bag.iterator().toList();
         assertEquals(1, workResc1Children.size());
         Resource workResc1FileObj = workResc1Children.get(0).asResource();
-        assertFalse(workResc1FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
+        assertFalse(workResc1FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" +
                 "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));
 
+        // mp4 file extension
         Resource workResc2 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-24");
         testHelper.assertObjectPopulatedInSip(workResc2, dirManager, model, stagingLocs.get(1), null, "26");
         Bag workResc2Bag = model.getBag(workResc2);
         List<RDFNode> workResc2Children = workResc2Bag.iterator().toList();
         assertEquals(1, workResc2Children.size());
         Resource workResc2FileObj = workResc2Children.get(0).asResource();
-        assertFalse(workResc2FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
-                "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));
+        assertTrue(workResc2FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" +
+                "spaceId=open-hls&filename=gilmer_video-playlist.m3u8"));
+        assertTrue(workResc2FileObj.hasProperty(STREAMING_TYPE, "video"));
 
+        // mp3 file extension
         Resource workResc3 = testHelper.getResourceByCreateTime(depBagChildren, "2005-12-08");
         testHelper.assertObjectPopulatedInSip(workResc3, dirManager, model, stagingLocs.get(2), null, "27");
         Bag workResc3Bag = model.getBag(workResc3);
         List<RDFNode> workResc3Children = workResc3Bag.iterator().toList();
         assertEquals(1, workResc3Children.size());
         Resource workResc3FileObj = workResc3Children.get(0).asResource();
-        assertTrue(workResc3FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
+        assertTrue(workResc3FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" +
                 "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));
+        assertTrue(workResc3FileObj.hasProperty(STREAMING_TYPE, "sound"));
     }
 
     private void assertChildFileModsPopulated(DepositDirectoryManager dirManager, Resource workResc,

src/test/java/edu/unc/lib/boxc/migration/cdm/SourceFilesCommandIT.java

@@ -480,7 +480,6 @@ public void validateValidTest() throws Exception {
     @Test
     public void validateInvalidTest() throws Exception {
         indexExportSamples();
-        addSourceFile("276_182_E.tif");
 
         String[] args = new String[] {
                 "-w", project.getProjectPath().toString(),
@@ -495,7 +494,7 @@ public void validateInvalidTest() throws Exception {
 
         assertOutputContains("FAIL: Source file mapping at path " + project.getSourceFilesMappingPath()
                 + " is invalid");
-        assertOutputContains("- No path mapped at line 3");
+        assertOutputContains("- No path mapped at line 2");
         assertEquals(2, output.split("    - ").length, "Must only be two errors: " + output);
     }
 
@@ -567,7 +566,7 @@ public void statusUnmappedVerboseTest() throws Exception {
         assertOutputMatches(".*Objects Mapped: +2 \\(66.7%\\).*");
         assertOutputMatches(".*Unmapped Objects: +1.*");
         assertOutputMatches(".*Unmapped Objects:.*\n + \\* 26.*");
-        assertOutputMatches(".*Mappings Valid: +No.*");
+        assertOutputMatches(".*Mappings Valid: +Yes.*");
         assertOutputMatches(".*Potential Matches: +0.*");
     }
 

src/test/java/edu/unc/lib/boxc/migration/cdm/services/SipServiceTest.java

@@ -60,7 +60,8 @@
 
 import static edu.unc.lib.boxc.auth.api.AccessPrincipalConstants.AUTHENTICATED_PRINC;
 import static edu.unc.lib.boxc.auth.api.AccessPrincipalConstants.PUBLIC_PRINC;
-import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.streamingUrl;
+import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_TYPE;
+import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_URL;
 import static edu.unc.lib.boxc.migration.cdm.test.PostMigrationReportTestHelper.assertContainsRow;
 import static java.nio.file.StandardOpenOption.APPEND;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -328,7 +329,7 @@ public void generateSipsMissingSourceFile() throws Exception {
         testHelper.generateDefaultDestinationsMapping(DEST_UUID, null);
         testHelper.populateDescriptions("gilmer_mods1.xml");
         // Only populating 2 out of 3 source files expected from the export
-        testHelper.populateSourceFiles("276_182_E.tif", "276_203_E.tif");
+        testHelper.populateSourceFiles("276_203_E.tif");
 
         try {
             service.generateSips(makeOptions());
@@ -361,7 +362,7 @@ public void generateSipsMissingSourceFileWithForce() throws Exception {
 
         Bag depBag = model.getBag(sip.getDepositPid().getRepositoryPath());
         List<RDFNode> depBagChildren = depBag.iterator().toList();
-        assertEquals(2, depBagChildren.size());
+        assertEquals(3, depBagChildren.size());
 
         Resource workResc1 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-23");
         testHelper.assertObjectPopulatedInSip(workResc1, dirManager, model, stagingLocs.get(0), null, "25");
@@ -1448,24 +1449,28 @@ public void generateSipsWithStreamingUrl() throws Exception {
         List<RDFNode> workResc1Children = workResc1Bag.iterator().toList();
         assertEquals(1, workResc1Children.size());
         Resource workResc1FileObj = workResc1Children.get(0).asResource();
-        assertFalse(workResc1FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
+        assertFalse(workResc1FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" +
                 "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));
+
         Resource workResc2 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-24");
         testHelper.assertObjectPopulatedInSip(workResc2, dirManager, model, stagingLocs.get(1), null, "26");
         Bag workResc2Bag = model.getBag(workResc2);
         List<RDFNode> workResc2Children = workResc2Bag.iterator().toList();
         assertEquals(1, workResc2Children.size());
         Resource workResc2FileObj = workResc2Children.get(0).asResource();
-        assertFalse(workResc2FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
-                "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));
+        assertTrue(workResc2FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" +
+                "spaceId=open-hls&filename=gilmer_video-playlist.m3u8"));
+        assertTrue(workResc2FileObj.hasProperty(STREAMING_TYPE, "video"));
+
         Resource workResc3 = testHelper.getResourceByCreateTime(depBagChildren, "2005-12-08");
         testHelper.assertObjectPopulatedInSip(workResc3, dirManager, model, stagingLocs.get(2), null, "27");
         Bag workResc3Bag = model.getBag(workResc3);
         List<RDFNode> workResc3Children = workResc3Bag.iterator().toList();
         assertEquals(1, workResc3Children.size());
         Resource workResc3FileObj = workResc3Children.get(0).asResource();
-        assertTrue(workResc3FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
+        assertTrue(workResc3FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" +
                 "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));
+        assertTrue(workResc3FileObj.hasProperty(STREAMING_TYPE, "sound"));
 
         assertPersistedSipInfoMatches(sip);
     }

src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java

@@ -71,6 +71,7 @@ public void getStreamingMetadataSuccess() throws Exception {
         assertEquals("gilmer_recording-playlist.m3u8", result[0]);
         assertEquals("open-hls", result[1]);
         assertEquals("duracloud", result[2]);
+        assertEquals("gilmer_recording.mp3", result[3]);
     }
 
     @Test

src/test/java/edu/unc/lib/boxc/migration/cdm/status/SourceFilesStatusServiceTest.java

@@ -103,7 +103,7 @@ public void blankPathsTest() throws Exception {
         assertOutputMatches(".*Unmapped Objects: +2 \\(66.7%\\).*");
         assertOutputMatches(".*Unmapped Objects:.* + \\* 26.*");
         assertOutputMatches(".*Unmapped Objects:.* + \\* 27.*");
-        assertOutputMatches(".*Mappings Valid: +No.*");
+        assertOutputMatches(".*Mappings Valid: +Yes.*");
         assertOutputMatches(".*Potential Matches: +0.*");
     }
 
@@ -122,7 +122,7 @@ public void potentialMatchesTest() throws Exception {
         assertOutputMatches(".*Objects Mapped: +2 \\(66.7%\\).*");
         assertOutputMatches(".*Unmapped Objects: +1 \\(33.3%\\).*");
         assertOutputMatches(".*Unknown Objects: +0.*");
-        assertOutputMatches(".*Mappings Valid: +No.*");
+        assertOutputMatches(".*Mappings Valid: +Yes.*");
         assertOutputMatches(".*Potential Matches: +1.*");
     }
 

src/test/resources/descriptions/mini_gilmer/index/description/desc.all

@@ -117,6 +117,8 @@
 <fila>TIFF</fila>
 <filb>TIFF</filb>
 <groupa>group1</groupa>
+<stream>gilmer_video.mp4</stream>
+<duracl>sfc20009-open</duracl>
 <full>/shc/gilmer_maps/</full>
 <fullrs>276_183_E.tif</fullrs>
 <find>27.JP2</find>