Skip to content

Commit

Permalink
MODINV-1012: Implementation for Validation for NatureContentTerms add…
Browse files Browse the repository at this point in the history
…ed for Instance. Tests added. (#715)

* MODINV-1012: Implementation for Validation for NatureContentTerms added for Instance. Tests added.

* MODINV-1012: Documentation added.

* MODINV-1012: Documentation added.

* MODINV-1012: Tests added.
  • Loading branch information
VRohach authored Apr 19, 2024
1 parent 8da0d8d commit 2019a17
Show file tree
Hide file tree
Showing 7 changed files with 282 additions and 3 deletions.
1 change: 1 addition & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
* The sorting for Items on Instance details page is not worked [MODINV-1001](https://folio-org.atlassian.net/browse/MODINV-1001)
* "PMSystem" displayed as source in "quickmarc" view when record was created by "Non-matches" action of job profile [MODSOURCE-608](https://folio-org.atlassian.net/browse/MODSOURCE-608)
* The result table is not displayed in the file details log [MODINV-1003](https://folio-org.atlassian.net/browse/MODINV-1003)
* Invalid values (as it is) created in JSON when value is not matching accepted options provided in Instance field mapping for Nature of Content term [MODINV-1012](https://folio-org.atlassian.net/browse/MODINV-1012)

## 20.2.0 2023-03-20
* Inventory cannot process Holdings with virtual fields ([MODINV-941](https://issues.folio.org/browse/MODINV-941))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.folio.inventory.dataimport.services.OrderHelperService;
import org.folio.inventory.dataimport.util.AdditionalFieldsUtil;
import org.folio.inventory.dataimport.util.ParsedRecordUtil;
import org.folio.inventory.dataimport.util.ValidationUtil;
import org.folio.inventory.domain.instances.Instance;
import org.folio.inventory.domain.instances.InstanceCollection;
import org.folio.inventory.domain.relationship.RecordToEntity;
Expand Down Expand Up @@ -119,15 +120,24 @@ public CompletableFuture<DataImportEventPayload> handle(DataImportEventPayload d
.compose(v -> {
InstanceCollection instanceCollection = storage.getInstanceCollection(context);
JsonObject instanceAsJson = prepareInstance(dataImportEventPayload, instanceId, jobExecutionId);
List<String> errors = EventHandlingUtil.validateJsonByRequiredFields(instanceAsJson, requiredFields);
if (!errors.isEmpty()) {
String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", errors,
List<String> requiredFieldsErrors = EventHandlingUtil.validateJsonByRequiredFields(instanceAsJson, requiredFields);
if (!requiredFieldsErrors.isEmpty()) {
String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", requiredFieldsErrors,
jobExecutionId, recordId, chunkId);
LOGGER.warn(msg);
return Future.failedFuture(msg);
}

Instance mappedInstance = Instance.fromJson(instanceAsJson);

List<String> invalidUUIDsErrors = ValidationUtil.validateUUIDs(mappedInstance);
if (!invalidUUIDsErrors.isEmpty()) {
String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", invalidUUIDsErrors,
jobExecutionId, recordId, chunkId);
LOGGER.warn(msg);
return Future.failedFuture(msg);
}

return addInstance(mappedInstance, instanceCollection)
.compose(createdInstance -> getPrecedingSucceedingTitlesHelper().createPrecedingSucceedingTitles(mappedInstance, context).map(createdInstance))
.compose(createdInstance -> executeFieldsManipulation(createdInstance, targetRecord))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.folio.inventory.dataimport.cache.MappingMetadataCache;
import org.folio.inventory.dataimport.handlers.matching.util.EventHandlingUtil;
import org.folio.inventory.dataimport.util.AdditionalFieldsUtil;
import org.folio.inventory.dataimport.util.ValidationUtil;
import org.folio.inventory.domain.instances.Instance;
import org.folio.inventory.domain.instances.InstanceCollection;
import org.folio.inventory.exceptions.NotFoundException;
Expand Down Expand Up @@ -193,6 +194,14 @@ private void processInstanceUpdate(DataImportEventPayload dataImportEventPayload
String marcBibAsJson = payloadContext.get(EntityType.MARC_BIBLIOGRAPHIC.value());
org.folio.rest.jaxrs.model.Record targetRecord = Json.decodeValue(marcBibAsJson, org.folio.rest.jaxrs.model.Record.class);
Instance mappedInstance = Instance.fromJson(instanceAsJson);
List<String> invalidUUIDsErrors = ValidationUtil.validateUUIDs(mappedInstance);
if (!invalidUUIDsErrors.isEmpty()) {
String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", invalidUUIDsErrors,
jobExecutionId, recordId, chunkId);
LOGGER.warn(msg);
return Future.failedFuture(msg);
}

return updateInstanceAndRetryIfOlExists(mappedInstance, instanceCollection, dataImportEventPayload)
.compose(updatedInstance -> getPrecedingSucceedingTitlesHelper().getExistingPrecedingSucceedingTitles(mappedInstance, context))
.map(precedingSucceedingTitles -> precedingSucceedingTitles.stream()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package org.folio.inventory.dataimport.util;

import org.folio.inventory.domain.instances.Instance;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

/**
* Util for detailed validation different entities.
*/
public class ValidationUtil {

private ValidationUtil() {
}

/**
* Validate fields inside the Instance entity. Validation based on checking if specific fields were mapped as UUIDs.
* If not - then the list with errors will be returned.
* Example: "Value 'invalid not UUID value' is not a UUID for someFieldName field"
* @param instance target Instance for validation
* @return ArrayList with errors when the needed fields are NOT as UUID.
*/
public static List<String> validateUUIDs(Instance instance) {
ArrayList<String> errorMessages = new ArrayList<>();

//TODO: This will be extended for different fields and entities.That's why there are so many methods just for 1 field.
// Branch for it extending validation: MODINV-1012-extended
validateField(errorMessages, instance.getNatureOfContentTermIds(), "natureOfContentTermIds");

return errorMessages;
}

private static void validateField(List<String> errorMessages, List<String> values, String fieldName) {
values.stream()
.filter(value -> !isUUID(value))
.forEach(value -> errorMessages.add(String.format("Value '%s' is not a UUID for %s field", value, fieldName)));
}

private static boolean isUUID(String value) {
try {
UUID.fromString(value);
return true;
} catch (Exception ex) {
return false;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.folio.processing.mapping.defaultmapper.processor.parameters.MappingParameters;
import org.folio.processing.mapping.mapper.reader.Reader;
import org.folio.processing.mapping.mapper.reader.record.marc.MarcBibReaderFactory;
import org.folio.processing.value.ListValue;
import org.folio.processing.value.MissingValue;
import org.folio.processing.value.StringValue;
import org.folio.rest.client.SourceStorageRecordsClient;
Expand Down Expand Up @@ -172,6 +173,44 @@ public class CreateInstanceEventHandlerTest {
.withContentType(MAPPING_PROFILE)
.withContent(JsonObject.mapFrom(mappingProfile).getMap())))));

private JobProfile jobProfileWithNatureOfContentTerm = new JobProfile()
.withId(UUID.randomUUID().toString())
.withName("Create MARC Bibs with NatureOfContentTerm")
.withDataType(JobProfile.DataType.MARC);

private ActionProfile actionProfileWithNatureOfContentTerm = new ActionProfile()
.withId(UUID.randomUUID().toString())
.withName("Create preliminary Item with NatureOfContentTerm")
.withAction(ActionProfile.Action.CREATE)
.withFolioRecord(INSTANCE);

private MappingProfile mappingProfileWithNatureOfContentTerm = new MappingProfile()
.withId(UUID.randomUUID().toString())
.withName("Prelim item from MARC with NatureOfContentTerm")
.withIncomingRecordType(EntityType.MARC_BIBLIOGRAPHIC)
.withExistingRecordType(EntityType.INSTANCE)
.withMappingDetails(new MappingDetail()
.withMappingFields(Lists.newArrayList(
new MappingRule().withPath("instance.instanceTypeId").withValue("\"instanceTypeIdExpression\"").withEnabled("true"),
new MappingRule().withPath("instance.title").withValue("\"titleExpression\"").withEnabled("true"),
new MappingRule().withPath("instance.natureOfContentTermIds[]").withValue("\"not uuid\"").withEnabled("true").withRepeatableFieldAction(MappingRule.RepeatableFieldAction.EXTEND_EXISTING))));

private ProfileSnapshotWrapper profileSnapshotWrapperWithNatureOfContentTerm = new ProfileSnapshotWrapper()
.withId(UUID.randomUUID().toString())
.withProfileId(jobProfileWithNatureOfContentTerm.getId())
.withContentType(JOB_PROFILE)
.withContent(jobProfileWithNatureOfContentTerm)
.withChildSnapshotWrappers(Collections.singletonList(
new ProfileSnapshotWrapper()
.withProfileId(actionProfileWithNatureOfContentTerm.getId())
.withContentType(ACTION_PROFILE)
.withContent(actionProfileWithNatureOfContentTerm)
.withChildSnapshotWrappers(Collections.singletonList(
new ProfileSnapshotWrapper()
.withProfileId(mappingProfileWithNatureOfContentTerm.getId())
.withContentType(MAPPING_PROFILE)
.withContent(JsonObject.mapFrom(mappingProfileWithNatureOfContentTerm).getMap())))));

private CreateInstanceEventHandler createInstanceEventHandler;

@Before
Expand Down Expand Up @@ -586,6 +625,55 @@ public void shouldNotProcessEventIfRequiredFieldIsEmpty() throws InterruptedExce
future.get(5, TimeUnit.MILLISECONDS);
}

@Test(expected = Exception.class)
public void shouldNotProcessEventIfNatureContentFieldIsNotUUID() throws InterruptedException, ExecutionException, TimeoutException {
Reader fakeReader = Mockito.mock(Reader.class);

String instanceTypeId = "fe19bae4-da28-472b-be90-d442e2428ead";
String recordId = "567859ad-505a-400d-a699-0028a1fdbf84";
String instanceId = "4d4545df-b5ba-4031-a031-70b1c1b2fc5d";
String title = "titleValue";
RecordToEntity recordToInstance = RecordToEntity.builder().recordId(recordId).entityId(instanceId)
.build();

when(fakeReader.read(any(MappingRule.class))).thenReturn(StringValue.of(instanceTypeId), StringValue.of(title), ListValue.of(Lists.newArrayList("not uuid")));

when(fakeReaderFactory.createReader()).thenReturn(fakeReader);

when(storage.getInstanceCollection(any())).thenReturn(instanceRecordCollection);

when(instanceIdStorageService.store(any(), any(), any())).thenReturn(Future.succeededFuture(recordToInstance));

MappingManager.registerReaderFactory(fakeReaderFactory);
MappingManager.registerWriterFactory(new InstanceWriterFactory());

HashMap<String, String> context = new HashMap<>();
Record record = new Record().withParsedRecord(new ParsedRecord().withContent(PARSED_CONTENT));
record.setId(recordId);

context.put(MARC_BIBLIOGRAPHIC.value(), Json.encode(record));

Buffer buffer = BufferImpl.buffer("{\"parsedRecord\":{" +
"\"id\":\"990fad8b-64ec-4de4-978c-9f8bbed4c6d3\"," +
"\"content\":\"{\\\"leader\\\":\\\"00574nam 22001211a 4500\\\",\\\"fields\\\":[{\\\"035\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"(in001)ybp7406411\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"245\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"titleValue\\\"}],\\\"ind1\\\":\\\"1\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"336\\\":{\\\"subfields\\\":[{\\\"b\\\":\\\"b6698d38-149f-11ec-82a8-0242ac130003\\\"}],\\\"ind1\\\":\\\"1\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"780\\\":{\\\"subfields\\\":[{\\\"t\\\":\\\"Houston oil directory\\\"}],\\\"ind1\\\":\\\"0\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"785\\\":{\\\"subfields\\\":[{\\\"t\\\":\\\"SAIS review of international affairs\\\"},{\\\"x\\\":\\\"1945-4724\\\"}],\\\"ind1\\\":\\\"0\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"500\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"Adaptation of Xi xiang ji by Wang Shifu.\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"520\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"Ben shu miao shu le cui ying ying he zhang sheng wei zheng qu hun yin zi you li jin qu zhe jian xin zhi hou, zhong cheng juan shu de ai qing gu shi. jie lu le bao ban hun yin he feng jian li jiao de zui e.\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"999\\\":{\\\"subfields\\\":[{\\\"i\\\":\\\"4d4545df-b5ba-4031-a031-70b1c1b2fc5d\\\"}],\\\"ind1\\\":\\\"f\\\",\\\"ind2\\\":\\\"f\\\"}}]}\"" +
"}}");
HttpResponse<Buffer> resp = buildHttpResponseWithBuffer(buffer);
when(sourceStorageClient.postSourceStorageRecords(any())).thenReturn(Future.succeededFuture(resp));

DataImportEventPayload dataImportEventPayload = new DataImportEventPayload()
.withEventType(DI_INVENTORY_INSTANCE_CREATED.value())
.withContext(context)
.withCurrentNode(profileSnapshotWrapperWithNatureOfContentTerm.getChildSnapshotWrappers().get(0))
.withTenant(TENANT_ID)
.withOkapiUrl(mockServer.baseUrl())
.withToken(TOKEN)
.withJobExecutionId(UUID.randomUUID().toString())
.withOkapiUrl(mockServer.baseUrl());

CompletableFuture<DataImportEventPayload> future = createInstanceEventHandler.handle(dataImportEventPayload);
future.get(10, TimeUnit.SECONDS);
}

@Test(expected = Exception.class)
public void shouldNotProcessEventIfRecordContains999field() throws InterruptedException, ExecutionException, TimeoutException {
var recordId = UUID.randomUUID().toString();
Expand Down
Loading

0 comments on commit 2019a17

Please sign in to comment.