From dcd5034b59f1d2a9e40f4139053e2353be2c6fde Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 26 May 2022 15:51:14 -0400 Subject: [PATCH 1/5] single-version semantics for archiving --- doc/sphinx-guides/source/api/native-api.rst | 2 ++ .../edu/harvard/iq/dataverse/DatasetPage.java | 4 +--- .../edu/harvard/iq/dataverse/api/Admin.java | 7 ++++++ .../harvard/iq/dataverse/api/Datasets.java | 23 +++++++++++++++++++ .../impl/AbstractSubmitToArchiveCommand.java | 16 +++++++++++++ .../iq/dataverse/util/ArchiverUtil.java | 23 +++++++++++++++++++ ....10.1.2__8605-support-archival-status.sql} | 0 7 files changed, 72 insertions(+), 3 deletions(-) rename src/main/resources/db/migration/{V5.10.1.0.2__8605-support-archival-status.sql => V5.10.1.2__8605-support-archival-status.sql} (100%) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d7046a43c7e..91ef488e9f8 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1911,6 +1911,8 @@ The body is a Json object that must contain a "status" which may be "success", " export JSON='{"status":"failure","message":"Something went wrong"}' curl -H "X-Dataverse-key: $API_TOKEN" -H "Content-Type:application/json" -X PUT "$SERVER_URL/api/datasets/submitDatasetVersionToArchive/$VERSION/status?persistentId=$PERSISTENT_IDENTIFIER" -d "$JSON" + +Note that if the configured archiver only supports archiving a single version, the call may return 409 CONFLICT if/when another version already has a non-null status. Delete the Archival Status of a Dataset By Version ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index d752c46d9a0..b9a34dac2ea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -5536,10 +5536,8 @@ public void refreshPaginator() { */ public void archiveVersion(Long id) { if (session.getUser() instanceof AuthenticatedUser) { - AuthenticatedUser au = ((AuthenticatedUser) session.getUser()); - DatasetVersion dv = datasetVersionService.retrieveDatasetVersionByVersionId(id).getDatasetVersion(); - String className = settingsService.getValueForKey(SettingsServiceBean.Key.ArchiverClassName); + String className = settingsWrapper.getValueForKey(SettingsServiceBean.Key.ArchiverClassName, null); AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, dvRequestService.getDataverseRequest(), dv); if (cmd != null) { try { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 78ec4a6edb5..f1f9c788f1e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1823,6 +1823,13 @@ public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @Pat String className = settingsService.getValueForKey(SettingsServiceBean.Key.ArchiverClassName); AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, dvRequestService.getDataverseRequest(), dv); if (cmd != null) { + if(ArchiverUtil.onlySingleVersionArchiving(cmd.getClass(), settingsService)) { + for (DatasetVersion version : ds.getVersions()) { + if ((dv != version) && version.getArchivalCopyLocation() != null) { + return error(Status.CONFLICT, "Dataset already archived."); + } + } + } new Thread(new Runnable() { public void run() { try { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index eac4a8f0d44..5545d61a3ad 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3346,6 +3346,13 @@ public Response setDatasetVersionToArchiveStatus(@PathParam("id") String dsid, if(dv==null) { return error(Status.NOT_FOUND, "Dataset version not found"); } + if (isSingleVersionArchiving()) { + for (DatasetVersion version : ds.getVersions()) { + if ((!dv.equals(version)) && (version.getArchivalCopyLocation() != null)) { + return error(Status.CONFLICT, "Dataset already archived."); + } + } + } dv.setArchivalCopyLocation(JsonUtil.prettyPrint(update)); dv = datasetversionService.merge(dv); @@ -3389,4 +3396,20 @@ public Response deleteDatasetVersionToArchiveStatus(@PathParam("id") String dsid return wr.getResponse(); } } + + private boolean isSingleVersionArchiving() { + String className = settingsService.getValueForKey(SettingsServiceBean.Key.ArchiverClassName, null); + if (className != null) { + Class clazz; + try { + clazz = Class.forName(className).asSubclass(AbstractSubmitToArchiveCommand.class); + return ArchiverUtil.onlySingleVersionArchiving(clazz, settingsService); + } catch (ClassNotFoundException e) { + logger.warning(":ArchiverClassName does not refer to a known Archiver"); + } catch (ClassCastException cce) { + logger.warning(":ArchiverClassName does not refer to an Archiver class"); + } + } + return false; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java index 4fa0961d134..2e8d33a61de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java @@ -1,7 +1,9 @@ package edu.harvard.iq.dataverse.engine.command.impl; +import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.SettingsWrapper; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -87,4 +89,18 @@ public String describe() { + version.getFriendlyVersionNumber()+")]"; } + public static boolean isArchivable(Dataset dataset, SettingsWrapper settingsWrapper) { + return true; + } + + //Check if the chosen archiver imposes single-version-only archiving - in a View context + public static boolean isSingleVersion(SettingsWrapper settingsWrapper) { + return false; + } + + //Check if the chosen archiver imposes single-version-only archiving - in the API + public static boolean isSingleVersion(SettingsServiceBean settingsService) { + return false; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java index fc97f972f5c..b21fc807574 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java @@ -1,11 +1,14 @@ package edu.harvard.iq.dataverse.util; import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.logging.Logger; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; /** * Simple class to reflectively get an instance of the desired class for @@ -35,4 +38,24 @@ public static AbstractSubmitToArchiveCommand createSubmitToArchiveCommand(String } return null; } + + public static boolean onlySingleVersionArchiving(Class clazz, SettingsServiceBean settingsService) { + Method m; + try { + m = clazz.getMethod("isSingleVersion", SettingsServiceBean.class); + Object[] params = { settingsService }; + return (Boolean) m.invoke(null, params); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (SecurityException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + return (AbstractSubmitToArchiveCommand.isSingleVersion(settingsService)); + } } diff --git a/src/main/resources/db/migration/V5.10.1.0.2__8605-support-archival-status.sql b/src/main/resources/db/migration/V5.10.1.2__8605-support-archival-status.sql similarity index 100% rename from src/main/resources/db/migration/V5.10.1.0.2__8605-support-archival-status.sql rename to src/main/resources/db/migration/V5.10.1.2__8605-support-archival-status.sql From 3acaa45c8d97ea12c401c547936babc0dd0b5001 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 26 May 2022 15:53:36 -0400 Subject: [PATCH 2/5] rename flyway script --- ...val-status.sql => V5.10.1.3__8605-support-archival-status.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.10.1.2__8605-support-archival-status.sql => V5.10.1.3__8605-support-archival-status.sql} (100%) diff --git a/src/main/resources/db/migration/V5.10.1.2__8605-support-archival-status.sql b/src/main/resources/db/migration/V5.10.1.3__8605-support-archival-status.sql similarity index 100% rename from src/main/resources/db/migration/V5.10.1.2__8605-support-archival-status.sql rename to src/main/resources/db/migration/V5.10.1.3__8605-support-archival-status.sql From c74f9b68cb719abe669be63024b3419fc9678d9e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 20 Jul 2022 13:19:26 -0400 Subject: [PATCH 3/5] handle name and dataset/version changes from 8696 review --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 9a52907f3c9..15b3cd2b9db 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3343,8 +3343,8 @@ public Response setDatasetVersionArchivalStatus(@PathParam("id") String datasetI return error(Status.NOT_FOUND, "Dataset version not found"); } if (isSingleVersionArchiving()) { - for (DatasetVersion version : ds.getVersions()) { - if ((!dv.equals(version)) && (version.getArchivalCopyLocation() != null)) { + for (DatasetVersion version : dsv.getDataset().getVersions()) { + if ((!dsv.equals(version)) && (version.getArchivalCopyLocation() != null)) { return error(Status.CONFLICT, "Dataset already archived."); } } From cb6148d32e28246800d9ad1de3775f7694a1651b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 20 Jul 2022 13:49:36 -0400 Subject: [PATCH 4/5] change instance to installation per review request --- doc/sphinx-guides/source/api/native-api.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index ab904a25ac6..69cc4b7f11f 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1245,7 +1245,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/24/privateUrl -If Anonymized Access has been enabled on a Dataverse instance (see the :ref:`:AnonymizedFieldTypeNames` setting), an optional 'anonymizedAccess' query parameter is allowed. +If Anonymized Access has been enabled on a Dataverse installation (see the :ref:`:AnonymizedFieldTypeNames` setting), an optional 'anonymizedAccess' query parameter is allowed. Setting anonymizedAccess=true in your call will create a PrivateURL that only allows an anonymized view of the Dataset (see :ref:`privateurl`). .. code-block:: bash @@ -1303,7 +1303,7 @@ When adding a file to a dataset, you can optionally specify the following: - Whether or not the file is restricted. - Whether or not the file skips :doc:`tabular ingest `. If the ``tabIngest`` parameter is not specified, it defaults to ``true``. -Note that when a Dataverse instance is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. +Note that when a Dataverse installation is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. In the curl example below, all of the above are specified but they are optional. @@ -1878,7 +1878,7 @@ The API call requires a Json body that includes the list of the fileIds that the Get the Archival Status of a Dataset By Version ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Archiving is an optional feature that may be configured for a Dataverse instance. When that is enabled, this API call be used to retrieve the status. Note that this requires "superuser" credentials. +Archiving is an optional feature that may be configured for a Dataverse installation. When that is enabled, this API call be used to retrieve the status. Note that this requires "superuser" credentials. GET /api/datasets/$dataset-id/$version/archivalStatus returns the archival status of the specified dataset version. @@ -1896,7 +1896,7 @@ The response is a JSON object that will contain a "status" which may be "success Set the Archival Status of a Dataset By Version ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Archiving is an optional feature that may be configured for a Dataverse instance. When that is enabled, this API call be used to set the status. Note that this is intended to be used by the archival system and requires "superuser" credentials. +Archiving is an optional feature that may be configured for a Dataverse installation. When that is enabled, this API call be used to set the status. Note that this is intended to be used by the archival system and requires "superuser" credentials. PUT /api/datasets/$dataset-id/$version/archivalStatus sets the archival status of the specified dataset version. @@ -1917,7 +1917,7 @@ Note that if the configured archiver only supports archiving a single version, t Delete the Archival Status of a Dataset By Version ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Archiving is an optional feature that may be configured for a Dataverse instance. When that is enabled, this API call be used to delete the status. Note that this is intended to be used by the archival system and requires "superuser" credentials. +Archiving is an optional feature that may be configured for a Dataverse installation. When that is enabled, this API call be used to delete the status. Note that this is intended to be used by the archival system and requires "superuser" credentials. DELETE /api/datasets/$dataset-id/$version/archivalStatus deletes the archival status of the specified dataset version. @@ -2135,7 +2135,7 @@ Replacing Files Replace an existing file where ``ID`` is the database id of the file to replace or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires the ``file`` to be passed as well as a ``jsonString`` expressing the new metadata. Note that metadata such as description, directoryLabel (File Path) and tags are not carried over from the file being replaced. -Note that when a Dataverse instance is configured to use S3 storage with direct upload enabled, there is API support to send a replacement file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. +Note that when a Dataverse installation is configured to use S3 storage with direct upload enabled, there is API support to send a replacement file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. A curl example using an ``ID`` From aa165dead1c25feac4d9037fe474029a1b6331f6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 28 Jul 2022 11:21:53 -0400 Subject: [PATCH 5/5] remove obsolete script --- .../db/migration/V5.10.1.3__8605-support-archival-status.sql | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 src/main/resources/db/migration/V5.10.1.3__8605-support-archival-status.sql diff --git a/src/main/resources/db/migration/V5.10.1.3__8605-support-archival-status.sql b/src/main/resources/db/migration/V5.10.1.3__8605-support-archival-status.sql deleted file mode 100644 index cf708ad0ea9..00000000000 --- a/src/main/resources/db/migration/V5.10.1.3__8605-support-archival-status.sql +++ /dev/null @@ -1,2 +0,0 @@ -UPDATE datasetversion SET archivalCopyLocation = CONCAT('{"status":"success", "message":"', archivalCopyLocation,'"}') where archivalCopyLocation is not null and not archivalCopyLocation='Attempted'; -UPDATE datasetversion SET archivalCopyLocation = CONCAT('{"status":"failure", "message":"Attempted"}') where archivalCopyLocation='Attempted';