Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[MODDICORE-411] Map accepted values from Mapping params #357

Merged
merged 14 commits into from
Sep 30, 2024
Merged

[MODDICORE-411] Map accepted values from Mapping params #357

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
d13bd7b
[MODDICORE-411] Map accepted values from Mapping params
RomanChernetskyi Sep 13, 2024
75ec09c
fix HoldingsMapperTest
RomanChernetskyi Sep 13, 2024
6137740
Merge branch 'master' into MODDICORE-411
RomanChernetskyi Sep 20, 2024
20a30b2
Map orders mapping parameters
RomanChernetskyi Sep 23, 2024
0ef0c34
Merge branch 'master' into MODDICORE-411
RomanChernetskyi Sep 23, 2024
b9cf68b
add AcceptedValuesUtilTest
RomanChernetskyi Sep 23, 2024
7318cf9
Extend AcceptedValuesUtilTest
RomanChernetskyi Sep 23, 2024
0fe3f25
Format stat codes
RomanChernetskyi Sep 24, 2024
c6d0f4d
Fix mappingmanagerunittest
RomanChernetskyi Sep 24, 2024
d915674
Update kafka-junit dependency version
RomanChernetskyi Sep 25, 2024
1d5b1c2
Fix kafka tests
RomanChernetskyi Sep 25, 2024
38f9a95
Merge branch 'master' into MODDICORE-411
RomanChernetskyi Sep 25, 2024
dbe90aa
Remove unused imports
RomanChernetskyi Sep 25, 2024
0ca5bc1
refactor AcceptedValuesUtil
RomanChernetskyi Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@
<dependency>
<groupId>net.mguenther.kafka</groupId>
<artifactId>kafka-junit</artifactId>
<version>2.5.0</version>
<version>3.6.0</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
108 changes: 0 additions & 108 deletions src/main/java/org/folio/processing/mapping/MappingManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,8 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.folio.DataImportEventPayload;
import org.folio.Location;
import org.folio.MappingProfile;
import org.folio.Organization;
import org.folio.processing.exceptions.MappingException;
import org.folio.processing.mapping.defaultmapper.processor.parameters.MappingParameters;
import org.folio.processing.mapping.mapper.FactoryRegistry;
import org.folio.processing.mapping.mapper.Mapper;
import org.folio.processing.mapping.mapper.MappingContext;
Expand All @@ -17,12 +14,9 @@
import org.folio.processing.mapping.mapper.reader.ReaderFactory;
import org.folio.processing.mapping.mapper.writer.Writer;
import org.folio.processing.mapping.mapper.writer.WriterFactory;
import org.folio.rest.jaxrs.model.MappingRule;
import org.folio.rest.jaxrs.model.ProfileSnapshotWrapper;

import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;

import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE;

Expand All @@ -38,12 +32,6 @@
public final class MappingManager {
private static final Logger LOGGER = LogManager.getLogger(MappingManager.class);
private static final FactoryRegistry FACTORY_REGISTRY = new FactoryRegistry();
public static final String PERMANENT_LOCATION_ID = "permanentLocationId";
public static final String TEMPORARY_LOCATION_ID = "temporaryLocationId";
public static final String VENDOR_ID = "vendor";
private static final String MATERIAL_SUPPLIER = "materialSupplier";
private static final String ACCESS_PROVIDER = "accessProvider";
private static final String DONOR_ORGANIZATION_IDS = "donorOrganizationIds";

private MappingManager() {
}
Expand Down Expand Up @@ -71,10 +59,6 @@ public static DataImportEventPayload map(DataImportEventPayload eventPayload, Ma
mappingProfile = (MappingProfile) mappingProfileWrapper.getContent();
}

//Fix MODDICORE-128 (The system doesn't update acceptedLocation in mapping profiles after the location list is changed)
updateLocationsInMappingProfile(mappingProfile, mappingContext.getMappingParameters());
updateOrganizationsInMappingProfile(mappingProfile, mappingContext.getMappingParameters());

Reader reader = FACTORY_REGISTRY.createReader(mappingProfile.getIncomingRecordType());
Writer writer = FACTORY_REGISTRY.createWriter(mappingProfile.getExistingRecordType());

Expand All @@ -86,98 +70,6 @@ public static DataImportEventPayload map(DataImportEventPayload eventPayload, Ma
}
}

/**
* Fill Permanent and Temporary Locations in MappingProfile from {@code mappingParameters}
*
* @param mappingProfile - MappingProfile
* @param mappingParameters - mapping parameters
*/
private static void updateLocationsInMappingProfile(MappingProfile mappingProfile, MappingParameters mappingParameters) {
if ((mappingProfile.getMappingDetails() != null) && (mappingProfile.getMappingDetails().getMappingFields() != null)) {
HashMap<String, String> locations = getLocationsFromMappingParameters(mappingParameters);
if (!locations.isEmpty()) {
for (MappingRule mappingRule : mappingProfile.getMappingDetails().getMappingFields()) {
if ((mappingRule.getName() != null) && (mappingRule.getName().equals(PERMANENT_LOCATION_ID) || mappingRule.getName().equals(TEMPORARY_LOCATION_ID))) {
mappingRule.setAcceptedValues(locations);
}
}
}
}
}

/**
* Fill {@link Organization} accepted values in MappingProfile from {@code mappingParameters}
*
* @param mappingProfile - MappingProfile
* @param mappingParameters - mapping parameters
*/
private static void updateOrganizationsInMappingProfile(MappingProfile mappingProfile, MappingParameters mappingParameters) {
if (mappingParameters.getOrganizations() == null) {
return;
}

if ((mappingProfile.getMappingDetails() != null) && (mappingProfile.getMappingDetails().getMappingFields() != null)) {
HashMap<String, String> organizations = getOrganizationsFromMappingParameters(mappingParameters);
HashMap<String, String> donorOrganizations = getDonorOrganizationsFromMappingParameters(mappingParameters);
if (!organizations.isEmpty()) {
for (MappingRule mappingRule : mappingProfile.getMappingDetails().getMappingFields()) {
if (isOrganizationsAcceptedValuesNeeded(mappingRule)) {
mappingRule.setAcceptedValues(organizations);
} else if (DONOR_ORGANIZATION_IDS.equals(mappingRule.getName())) {
populateDonorOrganizations(mappingRule, donorOrganizations);
}
}
}
}
}

private static HashMap<String, String> getLocationsFromMappingParameters(MappingParameters mappingParameters) {
HashMap<String, String> locations = new HashMap<>();
for (Location location : mappingParameters.getLocations()) {
StringBuilder locationValue = new StringBuilder()
.append(location.getName())
.append(" (")
.append(location.getCode())
.append(")");
locations.put(location.getId(), String.valueOf(locationValue));
}
return locations;
}

private static HashMap<String, String> getOrganizationsFromMappingParameters(MappingParameters mappingParameters) {
HashMap<String, String> organizations = new HashMap<>();
for (Organization organization : mappingParameters.getOrganizations()) {
organizations.put(organization.getId(), formatOrganizationValue(organization));
}
return organizations;
}

private static HashMap<String, String> getDonorOrganizationsFromMappingParameters(MappingParameters mappingParameters) {
return mappingParameters.getOrganizations().stream()
.filter(organization -> Boolean.TRUE.equals(organization.getIsDonor()))
.collect(Collectors.toMap(Organization::getId, MappingManager::formatOrganizationValue, (a, b) -> b, HashMap::new));
}

private static String formatOrganizationValue(Organization organization) {
return new StringBuilder()
.append(organization.getCode())
.append(" (")
.append(organization.getId())
.append(")")
.toString();
}

private static boolean isOrganizationsAcceptedValuesNeeded(MappingRule mappingRule) {
return (mappingRule.getName() != null) && (mappingRule.getName().equals(VENDOR_ID)
|| mappingRule.getName().equals(MATERIAL_SUPPLIER) || mappingRule.getName().equals(ACCESS_PROVIDER));
}

private static void populateDonorOrganizations(MappingRule mappingRule, HashMap<String, String> donorOrganizations) {
mappingRule.getSubfields().stream()
.flatMap(subfieldMapping -> subfieldMapping.getFields().stream())
.forEach(rule -> rule.setAcceptedValues(donorOrganizations));
}

/**
* Registers reader factory
*
Expand Down
23 changes: 13 additions & 10 deletions src/main/java/org/folio/processing/mapping/mapper/reader/record/marc/MarcRecordReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
import static org.apache.commons.lang3.StringUtils.EMPTY;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static org.folio.processing.mapping.MappingManager.VENDOR_ID;
import static org.folio.processing.mapping.mapper.util.AcceptedValuesUtil.getAcceptedValues;
import static org.folio.processing.value.Value.ValueType.LIST;
import static org.folio.processing.value.Value.ValueType.MISSING;

Expand Down Expand Up @@ -85,6 +85,7 @@ public class MarcRecordReader implements Reader {
private static final List<String> NEEDS_VALIDATION_BY_ACCEPTED_VALUES = List.of("vendor", "materialSupplier", "accessProvider","relationshipId", "donorOrganizationIds");
private static final String STATISTICAL_CODE_ID_FIELD = "statisticalCodeId";
private static final String BLANK = "";
private static final String VENDOR_ID = "vendor";

private EntityType entityType;
private Map<String, AcceptedValuesMatcher> acceptedValuesMatchers;
Expand Down Expand Up @@ -199,7 +200,7 @@ private void processMARCExpression(boolean arrayValue, boolean isRepeatableField
List<String> marcValues = readValuesFromMarcRecord(expressionPart).stream().filter(m -> isNotBlank(m)).collect(Collectors.toList());
if (arrayValue || (isRepeatableField && marcValues.size() > 1)) {
if (!resultList.isEmpty() && marcValues.size() == resultList.size()) {
List<String> collectedValues = marcValues.stream().map(value -> getFromAcceptedValues(ruleExpression, value)).collect(Collectors.toList());
List<String> collectedValues = marcValues.stream().map(value -> getFromMappingParameters(ruleExpression, value)).collect(Collectors.toList());
List<String> tmpResultList = new ArrayList<>(resultList);
String concatenator = multipleStringBuilder.toString();
for (int i = 0; i < tmpResultList.size(); i++) {
Expand All @@ -208,22 +209,24 @@ private void processMARCExpression(boolean arrayValue, boolean isRepeatableField
}
} else {
// TODO This todo for cases where first subfields count not equals second subfields count
List<String> collectedValues = marcValues.stream().map(value -> getFromAcceptedValues(ruleExpression, value)).collect(Collectors.toList());
List<String> collectedValues = marcValues.stream().map(value -> getFromMappingParameters(ruleExpression, value)).collect(Collectors.toList());
resultList.addAll(collectedValues);
}
} else {
if (!marcValues.isEmpty()) {
sb.append(getFromAcceptedValues(ruleExpression, marcValues.get(0)));
sb.append(getFromMappingParameters(ruleExpression, marcValues.get(0)));
}
}
}

private String getFromAcceptedValues(MappingRule ruleExpression, String value) {
private String getFromMappingParameters(MappingRule ruleExpression, String value) {
AcceptedValuesMatcher acceptedValuesMatcher = Objects.isNull(ruleExpression.getName()) ? null
: acceptedValuesMatchers.get(ruleExpression.getName());

if (ruleExpression.getAcceptedValues() != null && !ruleExpression.getAcceptedValues().isEmpty()) {
for (Map.Entry<String, String> entry : ruleExpression.getAcceptedValues().entrySet()) {
Map<String, String> acceptedValues = getAcceptedValues(ruleExpression.getName(), mappingParameters);

if (!acceptedValues.isEmpty()) {
for (Map.Entry<String, String> entry : acceptedValues.entrySet()) {
if ((acceptedValuesMatcher != null && acceptedValuesMatcher.matches(entry.getValue(), value))
|| entry.getValue().equalsIgnoreCase(value) || equalsBasedOnBrackets(ruleExpression.getName(), entry.getValue(), value)) {
value = entry.getKey();
Expand All @@ -232,7 +235,7 @@ private String getFromAcceptedValues(MappingRule ruleExpression, String value) {
}
boolean needsValidationByAcceptedValues = NEEDS_VALIDATION_BY_ACCEPTED_VALUES.contains(String.valueOf(ruleExpression.getName()));

if (needsValidationByAcceptedValues && !ruleExpression.getAcceptedValues().containsKey(value)) {
if (needsValidationByAcceptedValues && !acceptedValues.containsKey(value)) {
return BLANK;
}

Expand Down Expand Up @@ -305,7 +308,7 @@ private int findFirstParenthesesIndex(String input, int startIndex) {
*/
private StringBuilder processStringExpression(MappingRule ruleExpression, boolean arrayValue, List<String> resultList, StringBuilder sb, StringBuilder multipleStringBuilder, String expressionPart) {
String value = expressionPart.replace(EXPRESSIONS_QUOTE, EMPTY);
value = getFromAcceptedValues(ruleExpression, value);
value = getFromMappingParameters(ruleExpression, value);
if (isNotEmpty(value)) {
if (arrayValue && resultList.isEmpty()) {
resultList.add(value);
Expand Down Expand Up @@ -406,7 +409,7 @@ private void retrieveValuesFromMarcRecord(List<String> repeatableStrings, Mappin

private String readRepeatableStringField(MappingRule mappingRule) {
String value = mappingRule.getValue().replace(EXPRESSIONS_QUOTE, EMPTY);
return getFromAcceptedValues(mappingRule, value);
return getFromMappingParameters(mappingRule, value);
}

private boolean shouldCreateItemPerRepeatedMarcField(Value.ValueType valueType, MappingRule mappingRule) {
Expand Down
Loading
Loading