Skip to content

Commit

Permalink
Fix wrong resource type for metadata documents.
Browse files Browse the repository at this point in the history 
Fix kit-data-manager#558
Fix kit-data-manager#560
  • Loading branch information
@VolkerHartmann
VolkerHartmann committed Aug 23, 2024
1 parent c6fe9e4 commit 77c79e5
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 31 deletions.
63 changes: 34 additions & 29 deletions src/main/java/edu/kit/datamanager/metastore2/util/DataResourceRecordUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,12 @@
import edu.kit.datamanager.metastore2.domain.DataRecord;
import edu.kit.datamanager.metastore2.domain.MetadataRecord;
import edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord;

import static edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord.SCHEMA_TYPE.JSON;
import static edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord.SCHEMA_TYPE.XML;

import edu.kit.datamanager.metastore2.domain.ResourceIdentifier;
import edu.kit.datamanager.metastore2.domain.ResourceIdentifier.IdentifierType;
import static edu.kit.datamanager.metastore2.domain.ResourceIdentifier.IdentifierType.INTERNAL;
import static edu.kit.datamanager.metastore2.domain.ResourceIdentifier.IdentifierType.URL;
import edu.kit.datamanager.metastore2.domain.SchemaRecord;
import edu.kit.datamanager.metastore2.domain.Url2Path;
import edu.kit.datamanager.metastore2.domain.oaipmh.MetadataFormat;
Expand All @@ -62,6 +62,7 @@
import edu.kit.datamanager.util.AuthenticationHelper;
import edu.kit.datamanager.util.ControllerUtils;
import io.swagger.v3.core.util.Json;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
Expand All @@ -87,6 +88,7 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;

import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -176,9 +178,9 @@ public static DataResource createDataResourceRecord4Schema(MetastoreConfiguratio
LOG.error(message);
throw new BadArgumentException(message);
}
// Check if id is lower case and URL encodable.
// and save as alternate identifier. (In case of
// upper letters in both versions (with and without
// Check if id is lower case and URL encodable.
// and save as alternate identifier. (In case of
// upper letters in both versions (with and without
// upper letters)
DataResourceRecordUtil.check4validSchemaId(metadataRecord);
// End of parameter checks
Expand Down Expand Up @@ -219,7 +221,7 @@ public static DataResource createDataResourceRecord4Schema(MetastoreConfiguratio
throw new UnprocessableEntityException(message);
}
}
// reload data resource
// reload data resource
metadataRecord = DataResourceRecordUtil.getRecordByIdAndVersion(applicationProperties, metadataRecord.getId(), Long.valueOf(metadataRecord.getVersion()));

return metadataRecord;
Expand Down Expand Up @@ -247,16 +249,9 @@ public static DataResource createDataResourceRecord4Metadata(MetastoreConfigurat
check4validId(metadataRecord, true);
}
// End of parameter checks
// validate schema document / determine type if not given
// validate schema document / determine or correct resource type
validateMetadataDocument(applicationProperties, metadataRecord, document);
// set internal parameters
if (metadataRecord.getResourceType() == null) {
LOG.trace("No mimetype set! Try to determine...");
if (document.getContentType() != null) {
LOG.trace("Set mimetype determined from document: '{}'", document.getContentType());
metadataRecord.getFormats().add(document.getContentType());
}
}

metadataRecord.setVersion(Long.toString(1));
// create record.
DataResource dataResource = metadataRecord;
Expand Down Expand Up @@ -989,10 +984,10 @@ public static Set<AclEntry> mergeAcl(Set<AclEntry> managed, Set<AclEntry> provid
provided = (provided == null) ? new HashSet<>() : provided;
if (!provided.isEmpty()) {
if (!provided.equals(managed)) {
// check for special access rights
// check for special access rights
// - only administrators are allowed to change ACL
checkAccessRights(managed, true);
// - at least principal has to remain as ADMIN
// - at least principal has to remain as ADMIN
checkAccessRights(provided, false);
LOG.trace("Updating record acl from {} to {}.", managed, provided);
managed = provided;
Expand Down Expand Up @@ -1041,16 +1036,16 @@ public static <T> T mergeEntry(String description, T managed, T provided, boolea
*
* @return Number of registered documents.
*/
public static long getNoOfDocuments() {
public static long getNoOfMetadataDocuments() {
// Search for resource type of MetadataSchemaRecord
Specification<DataResource> spec = ResourceTypeSpec.toSpecification(ResourceType.createResourceType(METADATA_SUFFIX, ResourceType.TYPE_GENERAL.MODEL));
Pageable pgbl = PageRequest.of(0, 1);
return queryDataResources(spec, pgbl).getTotalElements();
}

/**
* Return the number of ingested schema documents. If there are two versions of the
* same document this will be counted as one.
* Return the number of ingested schema documents. If there are two versions
* of the same document this will be counted as one.
*
* @return Number of registered documents.
*/
Expand Down Expand Up @@ -1152,7 +1147,6 @@ private static void saveNewDataRecord(DataRecord dataRecord) {
*
* @param aclEntries AclEntries of resource.
* @param currentAcl Check current ACL (true) or new one (false).
*
* @return Allowed (true) or not.
*/
public static boolean checkAccessRights(Set<AclEntry> aclEntries, boolean currentAcl) {
Expand Down Expand Up @@ -1276,6 +1270,13 @@ public static void check4RelatedResource(DataResource dataResource, RelatedIdent
}
}

/**
* Test if exactly one schema and one related resource exists. This method
* does NOT check the correctness of the references.
*
* @param dataResource Data resource of a metadata document.
* @throws BadArgumentException Related resources are not defined as expected.
*/
public static void validateRelatedResources4MetadataDocuments(DataResource dataResource) throws BadArgumentException {
int noOfRelatedData = 0;
int noOfRelatedSchemas = 0;
Expand Down Expand Up @@ -1662,12 +1663,11 @@ public static void validateMetadataDocument(MetastoreConfiguration metastoreProp
/**
* Gets SchemaRecord from identifier. Afterwards there should be a clean up.
*
* @see #cleanUp(edu.kit.datamanager.metastore2.domain.ResourceIdentifier,
* edu.kit.datamanager.metastore2.domain.SchemaRecord)
*
* @param identifier ResourceIdentifier of type INTERNAL or URL.
* @param version Version (may be null)
* @return schema record.
* @see #cleanUp(edu.kit.datamanager.metastore2.domain.ResourceIdentifier,
* edu.kit.datamanager.metastore2.domain.SchemaRecord)
*/
public static SchemaRecord getSchemaRecord(ResourceIdentifier identifier, Long version) {
LOG.trace("getSchemaRecord {},{}", identifier, version);
Expand Down Expand Up @@ -1889,7 +1889,8 @@ public static DataResource updateMetadataSchemaRecord(MetastoreConfiguration app
}

/**
* Validate metadata document with given schema.
* Validate metadata document with given schema. Determine type if not already
* given or check type.
*
* @param metastoreProperties Configuration for accessing services
* @param metadataRecord metadata of the document.
Expand Down Expand Up @@ -1925,6 +1926,10 @@ private static void validateMetadataDocument(MetastoreConfiguration metastorePro
try {
validateMetadataDocument(metastoreProperties, document, findByAlternateId);
validationSuccess = true;
// After successful validation set type for metadata document resource.
MetadataSchemaRecord.SCHEMA_TYPE type = findByAlternateId.getType();
metadataRecord.setResourceType(ResourceType.createResourceType(type + METADATA_SUFFIX, ResourceType.TYPE_GENERAL.MODEL));
//
} catch (Exception ex) {
String message = "Error validating document!";
LOG.error(message, ex);
Expand Down Expand Up @@ -2013,10 +2018,10 @@ public static Page<DataResource> queryDataResources(Specification spec, Pageable
} else {
LOG.trace("Query all data resources...");
}
LOG.trace("-----------------------------------------------");
LOG.trace("List '{}' of '{}' data resources in total!", records.getContent().size(), records.getTotalElements());
LOG.trace("-----------------------------------------------");
int itemNo = 1;
LOG.trace("-----------------------------------------------");
LOG.trace("List '{}' of '{}' data resources in total!", records.getContent().size(), records.getTotalElements());
LOG.trace("-----------------------------------------------");
int itemNo = 1;
for (DataResource item : records.getContent()) {
LOG.trace("#{} - '{}'", itemNo++, item);
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataControllerImplV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

import com.fasterxml.jackson.core.JsonParseException;
import static edu.kit.datamanager.entities.Identifier.IDENTIFIER_TYPE.INTERNAL;
import static edu.kit.datamanager.entities.Identifier.IDENTIFIER_TYPE.URL;

import edu.kit.datamanager.entities.PERMISSION;
import edu.kit.datamanager.entities.RepoUserRole;
import edu.kit.datamanager.entities.messaging.MetadataResourceMessage;
Expand Down Expand Up @@ -494,7 +494,7 @@ public void contribute(Info.Builder builder) {
Map<String, String> details = ActuatorUtil.testDirectory(basePath);

if (!details.isEmpty()) {
details.put("No of metadata documents", Long.toString(DataResourceRecordUtil.getNoOfDocuments()));
details.put("No of metadata documents", Long.toString(DataResourceRecordUtil.getNoOfMetadataDocuments()));
builder.withDetail("metadataRepo", details);
}
}
Expand Down
42 changes: 42 additions & 0 deletions src/test/java/edu/kit/datamanager/metastore2/test/MetadataControllerTestV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
import edu.kit.datamanager.repo.domain.ContentInformation;
import edu.kit.datamanager.repo.domain.DataResource;
import edu.kit.datamanager.repo.domain.RelatedIdentifier;
import edu.kit.datamanager.repo.domain.ResourceType;
import edu.kit.datamanager.repo.domain.Scheme;
import java.util.Locale;
import java.util.UUID;
Expand Down Expand Up @@ -772,6 +773,47 @@ public void testCreateRecordWithInvalidMetadata() throws Exception {
file(metadataFile)).andDo(print()).andExpect(status().isUnprocessableEntity()).andReturn();
}

@Test
public void testCreateRecordWithInvalidorEmptyResource() throws Exception {
String id = "testCreateRecordWithInvalidorEmptyResource";
String schemaId = SCHEMA_ID;
DataResource record = SchemaRegistryControllerTestV2.createDataResource4Document(id, schemaId);
ObjectMapper mapper = new ObjectMapper();
// empty resource type
record.setResourceType(null);

MockMultipartFile recordFile = new MockMultipartFile("record", "metadata-record.json", "application/json", mapper.writeValueAsString(record).getBytes());
MockMultipartFile metadataFile = new MockMultipartFile("document", "metadata.xml", "application/xml", DC_DOCUMENT.getBytes());

this.mockMvc.perform(MockMvcRequestBuilders.multipart(API_METADATA_PATH).
file(recordFile).
file(metadataFile)).andDo(print()).andExpect(status().isCreated()).andReturn();

// wrong resource type
id = "testCreateRecordWithWrongType";
record.setId(id);
record.getAlternateIdentifiers().clear();
record.setResourceType(ResourceType.createResourceType(DataResourceRecordUtil.XML_METADATA_TYPE));

recordFile = new MockMultipartFile("record", "metadata-record.json", "application/json", mapper.writeValueAsString(record).getBytes());
metadataFile = new MockMultipartFile("document", "metadata.xml", "application/xml", DC_DOCUMENT.getBytes());

this.mockMvc.perform(MockMvcRequestBuilders.multipart(API_METADATA_PATH).
file(recordFile).
file(metadataFile)).andDo(print()).andExpect(status().isCreated()).andReturn();

// wrong resource value
id = "testCreateRecordWithWrongValue";
record.setId(id);
record.getAlternateIdentifiers().clear();
record.setResourceType(ResourceType.createResourceType(DataResourceRecordUtil.XML_METADATA_TYPE + "invalid", ResourceType.TYPE_GENERAL.MODEL));
recordFile = new MockMultipartFile("record", "metadata-record.json", "application/json", mapper.writeValueAsString(record).getBytes());

this.mockMvc.perform(MockMvcRequestBuilders.multipart(API_METADATA_PATH).
file(recordFile).
file(metadataFile)).andDo(print()).andExpect(status().isCreated()).andReturn();
}

@Test
public void testCreateRecordWithoutRecord() throws Exception {
MockMultipartFile metadataFile = new MockMultipartFile("document", "metadata.xml", "application/xml", DC_DOCUMENT.getBytes());
Expand Down

0 comments on commit 77c79e5

Please sign in to comment.