Feat/MET-6002 Save record deletion reason

metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java

@@ -30,6 +30,7 @@ public final class RestEndpoints {
   // DEPUBLISHED RECORDS
   public static final String DEPUBLISH_RECORDIDS_DATASETID = "/depublish/record_ids/{datasetId}";
   public static final String DEPUBLISH_EXECUTE_DATASETID = "/depublish/execute/{datasetId}";
+  public static final String DEPUBLISH_REASONS = "/depublish/reasons";
 
   //AUTHENTICATION
   public static final String AUTHENTICATION_REGISTER = "/authentication/register";

metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/dataset/DepublishRecordId.java

@@ -6,6 +6,7 @@
 import dev.morphia.annotations.Index;
 import dev.morphia.annotations.IndexOptions;
 import dev.morphia.annotations.Indexes;
+import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
 import eu.europeana.metis.mongo.model.HasMongoObjectId;
 import java.time.Instant;
 import org.bson.types.ObjectId;
@@ -59,6 +60,11 @@ public enum DepublicationStatus {DEPUBLISHED, PENDING_DEPUBLICATION}
    **/
   private Instant depublicationDate;
 
+  /**
+   * Reason for depublication of record
+   */
+  private DepublicationReason depublicationReason;
+
   @Override
   public void setId(ObjectId id) {
     this.id = id;
@@ -100,4 +106,12 @@ public Instant getDepublicationDate() {
   public void setDepublicationDate(Instant depublicationDate) {
     this.depublicationDate = depublicationDate;
   }
+
+  public DepublicationReason getDepublicationReason() {
+    return depublicationReason;
+  }
+
+  public void setDepublicationReason(DepublicationReason depublicationReason) {
+    this.depublicationReason = depublicationReason;
+  }
 }

metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/rest/DepublishRecordIdView.java

@@ -3,6 +3,7 @@
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import eu.europeana.metis.core.common.JavaTimeSerialization.IsoInstantSerializer;
 import eu.europeana.metis.core.dataset.DepublishRecordId;
+import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
 import java.time.Instant;
 
 /**
@@ -12,6 +13,7 @@ public class DepublishRecordIdView {
 
   private final String recordId;
   private final DepublicationStatus depublicationStatus;
+  private final DepublicationReason depublicationReason;
 
   @JsonSerialize(using = IsoInstantSerializer.class)
   private final Instant depublicationDate;
@@ -25,6 +27,7 @@ public DepublishRecordIdView(DepublishRecordId record) {
     this.depublicationDate = record.getDepublicationDate();
     this.depublicationStatus = DepublicationStatus
         .convertFromModelToView(record.getDepublicationStatus());
+    this.depublicationReason = record.getDepublicationReason() == null ? DepublicationReason.UNKNOWN : record.getDepublicationReason();
   }
 
   public String getRecordId() {
@@ -39,6 +42,10 @@ public Instant getDepublicationDate() {
     return depublicationDate;
   }
 
+  public DepublicationReason getDepublicationReason() {
+    return depublicationReason;
+  }
+
   /**
    * The status of this record with regards to (de)publication.
    */

metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublicationReason.java

@@ -0,0 +1,22 @@
+package eu.europeana.metis.core.workflow.plugins;
+
+public enum DepublicationReason {
+
+  BROKEN_MEDIA_LINKS("Broken media links"),
+  GDPR("GDPR"),
+  LACK_OF_PERMISSIONS("Lack of permissions"),
+  SENSITIVE_CONTENT("Sensitive content"),
+  REMOVED_DATA_AT_SOURCE("Removed data at source"),
+  GENERIC("Generic"),
+  UNKNOWN("Unknown");
+
+  private final String valueAsString;
+
+  DepublicationReason(String valueAsString) {
+    this.valueAsString = valueAsString;
+  }
+
+  public String toString(){
+    return valueAsString;
+  }
+}

metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadata.java

@@ -17,6 +17,7 @@ public class DepublishPluginMetadata extends AbstractExecutablePluginMetadata {
   private static final ExecutablePluginType pluginType = ExecutablePluginType.DEPUBLISH;
   private boolean datasetDepublish;
   private Set<String> recordIdsToDepublish;
+  private DepublicationReason depublicationReason;
 
   public DepublishPluginMetadata() {
     //Required for json serialization
@@ -43,4 +44,12 @@ public Set<String> getRecordIdsToDepublish() {
   public void setRecordIdsToDepublish(Set<String> recordIdsToDepublish) {
     this.recordIdsToDepublish = new HashSet<>(recordIdsToDepublish);
   }
+
+  public void setDepublicationReason(DepublicationReason depublicationReason) {
+    this.depublicationReason = depublicationReason;
+  }
+
+  public DepublicationReason getDepublicationReason() {
+    return depublicationReason;
+  }
 }

metis-core/metis-core-common/src/test/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadataTest.java

@@ -0,0 +1,43 @@
+package eu.europeana.metis.core.workflow.plugins;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.Set;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+class DepublishPluginMetadataTest {
+
+  DepublishPluginMetadata depublishPluginMetadata = new DepublishPluginMetadata();
+
+  @Test
+  void getExecutablePluginType() {
+    assertEquals(ExecutablePluginType.DEPUBLISH, depublishPluginMetadata.getExecutablePluginType());
+  }
+
+  @Test
+  void isDatasetDepublish() {
+    final boolean datasetDepublish = true;
+    depublishPluginMetadata.setDatasetDepublish(datasetDepublish);
+    boolean actualDepublish = depublishPluginMetadata.isDatasetDepublish();
+    assertEquals(datasetDepublish, actualDepublish);
+  }
+
+  @Test
+  void getRecordIdsToDepublish() {
+    Set<String> recordIdsToDepublish = Set.of("t1r", "t2r", "t3r", "t4r", "t5r");
+    depublishPluginMetadata.setRecordIdsToDepublish(recordIdsToDepublish);
+    Set<String> actualDepublish = depublishPluginMetadata.getRecordIdsToDepublish();
+    assertEquals(recordIdsToDepublish, actualDepublish);
+  }
+
+  @ParameterizedTest
+  @EnumSource(DepublicationReason.class)
+  void getDepublicationReason(DepublicationReason depublicationReason) {
+
+    depublishPluginMetadata.setDepublicationReason(depublicationReason);
+
+    assertEquals(depublicationReason, depublishPluginMetadata.getDepublicationReason());
+  }
+}

metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/controller/DepublishRecordIdController.java

@@ -1,5 +1,6 @@
 package eu.europeana.metis.core.rest.controller;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import eu.europeana.metis.authentication.rest.client.AuthenticationClient;
 import eu.europeana.metis.authentication.user.MetisUserView;
 import eu.europeana.metis.core.exceptions.NoDatasetFoundException;
@@ -8,14 +9,18 @@
 import eu.europeana.metis.core.util.DepublishRecordIdSortField;
 import eu.europeana.metis.core.util.SortDirection;
 import eu.europeana.metis.core.workflow.WorkflowExecution;
+import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
 import eu.europeana.metis.exception.BadContentException;
 import eu.europeana.metis.exception.GenericMetisException;
 import eu.europeana.metis.exception.UserUnauthorizedException;
 import eu.europeana.metis.utils.CommonStringValues;
 import eu.europeana.metis.utils.RestEndpoints;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -77,11 +82,13 @@ public DepublishRecordIdController(DepublishRecordIdService depublishRecordIdSer
       MediaType.TEXT_PLAIN_VALUE})
   @ResponseStatus(HttpStatus.CREATED)
   public void createRecordIdsToBeDepublished(@RequestHeader("Authorization") String authorization,
-      @PathVariable("datasetId") String datasetId, @RequestBody String recordIdsInSeparateLines
+      @PathVariable("datasetId") String datasetId,
+      @RequestParam(value = "depublicationReason") DepublicationReason depublicationReason,
+      @RequestBody String recordIdsInSeparateLines
   ) throws GenericMetisException {
     final MetisUserView metisUserView = authenticationClient.getUserByAccessTokenInHeader(authorization);
     final int added = depublishRecordIdService
-        .addRecordIdsToBeDepublished(metisUserView, datasetId, recordIdsInSeparateLines);
+        .addRecordIdsToBeDepublished(metisUserView, datasetId, recordIdsInSeparateLines, depublicationReason);
     if (LOGGER.isInfoEnabled()) {
       LOGGER.info("{} Depublish record ids added to dataset with datasetId: {}", added,
           CRLF_PATTERN.matcher(datasetId).replaceAll(""));
@@ -107,9 +114,10 @@ public void createRecordIdsToBeDepublished(@RequestHeader("Authorization") Strin
   @ResponseStatus(HttpStatus.CREATED)
   public void createRecordIdsToBeDepublished(@RequestHeader("Authorization") String authorization,
       @PathVariable("datasetId") String datasetId,
+      @RequestParam(value = "depublicationReason") DepublicationReason depublicationReason,
       @RequestPart("depublicationFile") MultipartFile recordIdsFile
   ) throws GenericMetisException, IOException {
-    createRecordIdsToBeDepublished(authorization, datasetId,
+    createRecordIdsToBeDepublished(authorization, datasetId, depublicationReason,
         new String(recordIdsFile.getBytes(), StandardCharsets.UTF_8));
   }
 
@@ -215,12 +223,43 @@ public WorkflowExecution addDepublishWorkflowInQueueOfWorkflowExecutions(
       @RequestHeader("Authorization") String authorization,
       @PathVariable("datasetId") String datasetId,
       @RequestParam(value = "datasetDepublish", defaultValue = "" + true) boolean datasetDepublish,
+      @RequestParam(value = "depublicationReason") DepublicationReason depublicationReason,
       @RequestParam(value = "priority", defaultValue = "0") int priority,
       @RequestBody(required = false) String recordIdsInSeparateLines)
       throws GenericMetisException {
     MetisUserView metisUserView = authenticationClient.getUserByAccessTokenInHeader(authorization);
     return depublishRecordIdService
         .createAndAddInQueueDepublishWorkflowExecution(metisUserView, datasetId,
-            datasetDepublish, priority, recordIdsInSeparateLines);
+            datasetDepublish, priority, recordIdsInSeparateLines, depublicationReason);
+  }
+
+  /**
+   * API to return all possible values of depublication reasons
+   * @return All possible values of depublication reasons
+   */
+  @GetMapping(value = RestEndpoints.DEPUBLISH_REASONS, produces = {
+      MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE})
+  @ResponseStatus(HttpStatus.OK)
+  public List<DepublicationReasonView> getAllDepublicationReasons(){
+    return Arrays.stream(DepublicationReason.values()).filter(value -> value != DepublicationReason.UNKNOWN)
+                 .map(DepublicationReasonView::new).collect(Collectors.toList());
+  }
+
+  private static class DepublicationReasonView {
+
+    @JsonProperty("name")
+    private final String name;
+    @JsonProperty("valueAsString")
+    private final String valueAsString;
+
+    /**
+     * Instantiates a new DepublicationReason view.
+     *
+     * @param depublicationReason the depublication reason
+     */
+    DepublicationReasonView(DepublicationReason depublicationReason) {
+      this.name = depublicationReason.name();
+      this.valueAsString = depublicationReason.toString();
+    }
   }
 }

metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/DepublishRecordIdDao.java

@@ -17,6 +17,7 @@
 import eu.europeana.metis.core.rest.RequestLimits;
 import eu.europeana.metis.core.util.DepublishRecordIdSortField;
 import eu.europeana.metis.core.util.SortDirection;
+import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
 import eu.europeana.metis.exception.BadContentException;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -109,7 +110,7 @@ private Set<String> getNonExistingRecordIds(String datasetId, Set<String> record
    * @throws BadContentException In case adding the records would violate the maximum number of depublished records that each
    * dataset can have.
    */
-  public int createRecordIdsToBeDepublished(String datasetId, Set<String> candidateRecordIds)
+  public int createRecordIdsToBeDepublished(String datasetId, Set<String> candidateRecordIds, DepublicationReason depublicationReason)
       throws BadContentException {
 
     // Check list size: if this is too large we can throw exception regardless of what's in the database.
@@ -129,19 +130,20 @@ public int createRecordIdsToBeDepublished(String datasetId, Set<String> candidat
     }
 
     // Add the records and we're done.
-    addRecords(recordIdsToAdd, datasetId, DepublicationStatus.PENDING_DEPUBLICATION, null);
+    addRecords(recordIdsToAdd, datasetId, DepublicationStatus.PENDING_DEPUBLICATION, null, depublicationReason);
     return recordIdsToAdd.size();
   }
 
   void addRecords(Set<String> recordIdsToAdd, String datasetId,
-      DepublicationStatus depublicationStatus, Instant depublicationDate) {
+      DepublicationStatus depublicationStatus, Instant depublicationDate, DepublicationReason depublicationReason) {
     final List<DepublishRecordId> objectsToAdd = recordIdsToAdd.stream().map(recordId -> {
       final DepublishRecordId depublishRecordId = new DepublishRecordId();
       depublishRecordId.setId(new ObjectId());
       depublishRecordId.setDatasetId(datasetId);
       depublishRecordId.setRecordId(recordId);
       depublishRecordId.setDepublicationStatus(depublicationStatus);
       depublishRecordId.setDepublicationDate(depublicationDate);
+      depublishRecordId.setDepublicationReason(depublicationReason);
       return depublishRecordId;
     }).toList();
     retryableExternalRequestForNetworkExceptions(() -> {
@@ -328,7 +330,7 @@ private Query<DepublishRecordId> prepareQueryForDepublishRecordIds(String datase
    * {@link DepublicationStatus#PENDING_DEPUBLICATION}
    */
   public void markRecordIdsWithDepublicationStatus(String datasetId, Set<String> recordIds,
-      DepublicationStatus depublicationStatus, @Nullable Date depublicationDate) {
+      DepublicationStatus depublicationStatus, @Nullable Date depublicationDate, DepublicationReason depublicationReason) {
 
     // Check correctness of parameters
     if (Objects.isNull(depublicationStatus) || StringUtils.isBlank(datasetId)) {
@@ -353,7 +355,7 @@ public void markRecordIdsWithDepublicationStatus(String datasetId, Set<String> r
                                                    .filter(
                                                        date -> depublicationStatus != DepublicationStatus.PENDING_DEPUBLICATION)
                                                    .map(Date::toInstant).orElse(null);
-      addRecords(recordIdsToAdd, datasetId, depublicationStatus, depublicationInstant);
+      addRecords(recordIdsToAdd, datasetId, depublicationStatus, depublicationInstant, depublicationReason);
 
       // Compute the records to update - if there are none, we're done.
       recordIdsToUpdate = new HashSet<>(recordIds);

metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java

@@ -123,13 +123,13 @@ private void reinstateDepublishRecordIdsStatus(IndexToPublishPlugin indexPlugin,
         if (!CollectionUtils.isEmpty(publishedRecordIds)) {
           depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId,
               publishedRecordIds.stream().map(depublishedRecordIdsByFullId::get)
-                                .collect(Collectors.toSet()), DepublicationStatus.PENDING_DEPUBLICATION, null);
+                                .collect(Collectors.toSet()), DepublicationStatus.PENDING_DEPUBLICATION, null, null);
         }
       }
     } else {
       // reset de-publish status, pass null, all records will be de-published
       depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null,
-          DepublicationStatus.PENDING_DEPUBLICATION, null);
+          DepublicationStatus.PENDING_DEPUBLICATION, null, null);
     }
   }
 
@@ -178,7 +178,7 @@ private void depublishRecordPostProcess(DepublishPlugin depublishPlugin, String
                                                                    Collectors.mapping(Pair::getRight, Collectors.toSet())));
     successfulRecords.forEach((dataset, records) ->
         depublishRecordIdDao.markRecordIdsWithDepublicationStatus(dataset, records,
-            DepublicationStatus.DEPUBLISHED, new Date()));
+            DepublicationStatus.DEPUBLISHED, new Date(), null));
 
     // Set publication fitness to PARTIALLY FIT (if not set to the more severe UNFIT).
     final Dataset dataset = datasetDao.getDatasetByDatasetId(datasetId);
@@ -195,7 +195,7 @@ private void depublishDatasetPostProcess(String datasetId) {
 
     // Set all depublished records back to PENDING.
     depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null,
-        DepublicationStatus.PENDING_DEPUBLICATION, null);
+        DepublicationStatus.PENDING_DEPUBLICATION, null, null);
     // Find latest PUBLISH Type Plugin and set dataStatus to DELETED.
     final PluginWithExecutionId<MetisPlugin> latestSuccessfulPlugin = workflowExecutionDao
         .getLatestSuccessfulPlugin(datasetId, OrchestratorService.PUBLISH_TYPES);
@@ -238,4 +238,4 @@ void performPluginPostProcessing(AbstractExecutablePlugin<?> plugin, String data
     }
     LOGGER.info("Finished postprocessing of plugin {} in dataset {}.", pluginType, datasetId);
   }
-}
+}