Skip to content

Commit

Permalink
BXC-4632 - Permit SIP generation when streaming url but no source (#96)
Browse files Browse the repository at this point in the history 
* Add CompleteMigration test to verify that file with streaming but no source file can generate a sip

* try to fix sip streaming metadata

* Fix test file so that the cdm response for fields matches the fields in mini gilmer, by adding streaming fields. Check for sourcePath existing before linking it during file resource generation

* Set embedded redis version directly since it's been removed from boxc

* Default all streaming types to video for now

---------

Co-authored-by: krwong <krwong@email.unc.edu>
  • Loading branch information
@bbpennel @krwong
bbpennel and krwong authored Jul 3, 2024
1 parent e8607fe commit 8440ba2
Show file tree
Hide file tree
Showing 6 changed files with 1,099 additions and 16 deletions.
2 changes: 2 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
<csv.version>1.9.0</csv.version>
<junit.jupiter.version>5.9.2</junit.jupiter.version>
<solr.solrj.version>8.10.1</solr.solrj.version>
<embedded-redis.version>0.7.3</embedded-redis.version>
</properties>

<build>
Expand Down Expand Up @@ -224,6 +225,7 @@
<artifactId>slf4j-simple</artifactId>
</exclusion>
</exclusions>
<version>${embedded-redis.version}</version>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
Expand Down
25 changes: 16 additions & 9 deletions src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ public class WorkGenerator {
// use local streamingUrl property for now because Cdr.streamingUrl only exists in a feature branch
public static final Property streamingUrl = createProperty(
"http://cdr.unc.edu/definitions/model#streamingUrl");
public static final Property streamingType = createProperty(
"http://cdr.unc.edu/definitions/model#streamingType");
protected PIDMinter pidMinter;
protected RedirectMappingService redirectMappingService;
protected SourceFilesInfo sourceFilesInfo;
Expand Down Expand Up @@ -147,12 +149,14 @@ protected Path getDescriptionPath(String cdmId, boolean allowMissing) {
protected SourceFilesInfo.SourceFileMapping getSourceFileMapping(String cdmId) {
SourceFilesInfo.SourceFileMapping sourceMapping = sourceFilesInfo.getMappingByCdmId(cdmId);
if (sourceMapping == null || sourceMapping.getSourcePaths() == null) {
String message = "Cannot transform object " + cdmId + ", no source file has been mapped";
if (options.isForce()) {
outputLogger.info(message);
throw new SipService.SkipObjectException();
} else {
throw new InvalidProjectStateException(message);
if (!streamingMetadataService.verifyRecordHasStreamingMetadata(cdmId)) {
String message = "Cannot transform object " + cdmId + ", no source file has been mapped";
if (options.isForce()) {
outputLogger.info(message);
throw new SipService.SkipObjectException();
} else {
throw new InvalidProjectStateException(message);
}
}
}
return sourceMapping;
Expand All @@ -165,9 +169,11 @@ protected Resource makeFileResource(PID fileObjPid, Path sourcePath) {
workBag.add(fileObjResc);

// Link source file
Resource origResc = DepositModelHelpers.addDatastream(fileObjResc, ORIGINAL_FILE);
origResc.addLiteral(CdrDeposit.stagingLocation, sourcePath.toUri().toString());
origResc.addLiteral(CdrDeposit.label, sourcePath.getFileName().toString());
if (sourcePath != null) {
Resource origResc = DepositModelHelpers.addDatastream(fileObjResc, ORIGINAL_FILE);
origResc.addLiteral(CdrDeposit.stagingLocation, sourcePath.toUri().toString());
origResc.addLiteral(CdrDeposit.label, sourcePath.getFileName().toString());
}
return fileObjResc;
}

Expand Down Expand Up @@ -247,6 +253,7 @@ protected void addStreamingMetadata(String cdmId, Resource resource) {
String streamingUrlValue = "https://durastream.lib.unc.edu/player?spaceId=" + duracloudSpace
+ "&filename=" + streamingFile;
resource.addProperty(streamingUrl, streamingUrlValue);
resource.addProperty(streamingType, "video");
}
}
}
87 changes: 87 additions & 0 deletions src/test/java/edu/unc/lib/boxc/migration/cdm/CompleteMigrationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import static com.github.tomakehurst.wiremock.client.WireMock.get;
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.streamingUrl;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

Expand Down Expand Up @@ -334,6 +335,92 @@ public void migrateGroupedCollectionTest() throws Exception {
assertDepositStatusSet(sip);
}

@Test
public void fileWithStreamingOnlyNoSourceFileTest() throws Exception {
mockFieldInfoUrl("cdm_fields_resp.json", COLLECTION_ID);

String[] argsInit = new String[] {
"-w", baseDir.toString(),
"--env-config", chompbConfigPath,
"init",
"-p", COLLECTION_ID,
"-e", "test"};
executeExpectSuccess(argsInit);

Path projPath = baseDir.resolve(COLLECTION_ID);
MigrationProject project = new MigrationProject(projPath);

String[] argsExport = new String[] {
"-w", projPath.toString(),
"--env-config", chompbConfigPath,
"export",
"-p", TestSshServer.PASSWORD };
executeExpectSuccess(argsExport);

String[] argsIndex = new String[] {
"-w", projPath.toString(),
"index"};
executeExpectSuccess(argsIndex);

String[] argsDest = new String[] {
"-w", projPath.toString(),
"destinations", "generate",
"-dd", DEST_UUID};
executeExpectSuccess(argsDest);

testHelper = new SipServiceHelper(project, filesBasePath);
Path sourcePath1 = testHelper.addSourceFile("276_182_E.tif");
Path sourcePath2 = testHelper.addSourceFile("276_183_E.tif");
// No source file item 27, so that it will only have streaming

String[] argsSource = new String[] {
"-w", projPath.toString(),
"source_files", "generate",
"-b", testHelper.getSourceFilesBasePath().toString(),
"-n", "file"};
executeExpectSuccess(argsSource);

Files.copy(Paths.get("src/test/resources/mods_collections/gilmer_mods1.xml"),
project.getDescriptionsPath().resolve("gilmer_mods1.xml"));
String[] argsDesc = new String[] {
"-w", projPath.toString(),
"descriptions", "expand" };
executeExpectSuccess(argsDesc);

String[] args = new String[] {
"-w", projPath.toString(),
"sips", "generate" };
executeExpectSuccess(args);

MigrationSip sip = testHelper.extractSipFromOutput(output);

DepositDirectoryManager dirManager = testHelper.createDepositDirectoryManager(sip);
Model model = testHelper.getSipModel(sip);

Bag depBag = model.getBag(sip.getDepositPid().getRepositoryPath());
List<RDFNode> depBagChildren = depBag.iterator().toList();
assertEquals(3, depBagChildren.size());

Resource workResc1 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-23");
testHelper.assertObjectPopulatedInSip(workResc1, dirManager, model, sourcePath1, null, "25");
Resource workResc2 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-24");
testHelper.assertObjectPopulatedInSip(workResc2, dirManager, model, sourcePath2, null, "26");
// Work 3 has no source file, but does have a streaming url
Resource workResc3 = testHelper.getResourceByCreateTime(depBagChildren, "2005-12-08");
testHelper.assertObjectPopulatedInSip(workResc3, dirManager, model, null, null, "27");
Resource fileResc3 = testHelper.getFirstSipFileInWork(workResc3, dirManager, model);
assertTrue(fileResc3.hasProperty(streamingUrl));

String[] argsSubmit = new String[] {
"-w", projPath.toString(),
"submit",
"-g", GROUPS };
executeExpectSuccess(argsSubmit);

initDepositStatusFactory();
assertDepositStatusSet(sip);
}

private String findFileIdByStagingLocation(Bag workBag, Path stagingLoc) {
Resource fileResc = testHelper.findChildByStagingLocation(workBag, stagingLoc);
return PIDs.get(fileResc.getURI()).getId();
Expand Down
50 changes: 50 additions & 0 deletions src/test/java/edu/unc/lib/boxc/migration/cdm/SipsCommandIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.nio.file.Path;
import java.util.List;

import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.streamingUrl;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
Expand Down Expand Up @@ -222,6 +223,55 @@ public void generateWithChildDescriptionsTest() throws Exception {
assertChildFileModsPopulated(dirManager, workResc3, "27/original_file");
}

@Test
public void generateStreamingFileOnlyTest() throws Exception {
testHelper.indexExportData("mini_gilmer");
testHelper.generateDefaultDestinationsMapping(DEST_UUID, null);
testHelper.populateDescriptions("gilmer_mods1.xml");
List<Path> stagingLocs = testHelper.populateSourceFiles("276_182_E.tif", "276_183_E.tif", "276_203_E.tif");

String[] args = new String[] {
"-w", project.getProjectPath().toString(),
"sips", "generate" };
executeExpectSuccess(args);

MigrationSip sip = extractSipFromOutput();

DepositDirectoryManager dirManager = testHelper.createDepositDirectoryManager(sip);
Model model = testHelper.getSipModel(sip);

Bag depBag = model.getBag(sip.getDepositPid().getRepositoryPath());
List<RDFNode> depBagChildren = depBag.iterator().toList();
assertEquals(3, depBagChildren.size());

Resource workResc1 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-23");
testHelper.assertObjectPopulatedInSip(workResc1, dirManager, model, stagingLocs.get(0), null, "25");
Bag workResc1Bag = model.getBag(workResc1);
List<RDFNode> workResc1Children = workResc1Bag.iterator().toList();
assertEquals(1, workResc1Children.size());
Resource workResc1FileObj = workResc1Children.get(0).asResource();
assertFalse(workResc1FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
"spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));

Resource workResc2 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-24");
testHelper.assertObjectPopulatedInSip(workResc2, dirManager, model, stagingLocs.get(1), null, "26");
Bag workResc2Bag = model.getBag(workResc2);
List<RDFNode> workResc2Children = workResc2Bag.iterator().toList();
assertEquals(1, workResc2Children.size());
Resource workResc2FileObj = workResc2Children.get(0).asResource();
assertFalse(workResc2FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
"spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));

Resource workResc3 = testHelper.getResourceByCreateTime(depBagChildren, "2005-12-08");
testHelper.assertObjectPopulatedInSip(workResc3, dirManager, model, stagingLocs.get(2), null, "27");
Bag workResc3Bag = model.getBag(workResc3);
List<RDFNode> workResc3Children = workResc3Bag.iterator().toList();
assertEquals(1, workResc3Children.size());
Resource workResc3FileObj = workResc3Children.get(0).asResource();
assertTrue(workResc3FileObj.hasProperty(streamingUrl, "https://durastream.lib.unc.edu/player?" +
"spaceId=open-hls&filename=gilmer_recording-playlist.m3u8"));
}

private void assertChildFileModsPopulated(DepositDirectoryManager dirManager, Resource workResc,
String expectedCdmId) throws Exception {
var workChildPid = retrieveOnlyWorkChildPid(workResc);
Expand Down
18 changes: 12 additions & 6 deletions src/test/java/edu/unc/lib/boxc/migration/cdm/test/SipServiceHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,19 +156,25 @@ public DepositDirectoryManager createDepositDirectoryManager(MigrationSip sip) {
return new DepositDirectoryManager(sip.getDepositPid(), project.getSipsPath(), true);
}

public void assertObjectPopulatedInSip(Resource objResc, DepositDirectoryManager dirManager, Model depModel,
Path stagingPath, Path accessPath, String cdmId) throws Exception {
public Resource getFirstSipFileInWork(Resource objResc, DepositDirectoryManager dirManager, Model depModel) {
assertTrue(objResc.hasProperty(RDF.type, Cdr.Work));
Bag workBag = depModel.getBag(objResc);
List<RDFNode> workChildren = workBag.iterator().toList();
assertEquals(1, workChildren.size());
Resource fileObjResc = workChildren.get(0).asResource();
return workChildren.get(0).asResource();
}

public void assertObjectPopulatedInSip(Resource objResc, DepositDirectoryManager dirManager, Model depModel,
Path stagingPath, Path accessPath, String cdmId) throws Exception {
Resource fileObjResc = getFirstSipFileInWork(objResc, dirManager, depModel);
assertTrue(fileObjResc.hasProperty(RDF.type, Cdr.FileObject));

// Check for source file
Resource origResc = fileObjResc.getProperty(CdrDeposit.hasDatastreamOriginal).getResource();
assertTrue(origResc.hasLiteral(CdrDeposit.stagingLocation, stagingPath.toUri().toString()));
assertTrue(origResc.hasLiteral(CdrDeposit.label, stagingPath.getFileName().toString()));
if (stagingPath != null) {
Resource origResc = fileObjResc.getProperty(CdrDeposit.hasDatastreamOriginal).getResource();
assertTrue(origResc.hasLiteral(CdrDeposit.stagingLocation, stagingPath.toUri().toString()));
assertTrue(origResc.hasLiteral(CdrDeposit.label, stagingPath.getFileName().toString()));
}

if (accessPath == null) {
// Verify no access copy
Expand Down
Loading

0 comments on commit 8440ba2

Please sign in to comment.