From 86ad64b33ac55972b9e1180b69b9629db21116bb Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 20 Aug 2024 17:32:40 -0400 Subject: [PATCH 01/20] fix/workaround for when PID provider is down --- .../CuratePublishedDatasetVersionCommand.java | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index f83041d87bd..6a4c78d4f4f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.datavariable.VarGroup; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; @@ -20,7 +19,6 @@ import edu.harvard.iq.dataverse.DataFileCategory; import edu.harvard.iq.dataverse.DatasetVersionDifference; -import java.util.Collection; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -167,33 +165,41 @@ public Dataset execute(CommandContext ctxt) throws CommandException { DeleteDatasetVersionCommand cmd; cmd = new DeleteDatasetVersionCommand(getRequest(), savedDataset); - ctxt.engine().submit(cmd); - // Running the command above reindexes the dataset, so we don't need to do it - // again in here. + cmd.execute(ctxt); // And update metadata at PID provider - ctxt.engine().submit( - new UpdateDvObjectPIDMetadataCommand(savedDataset, getRequest())); - - //And the exported metadata files try { - ExportService instance = ExportService.getInstance(); - instance.exportAllFormats(getDataset()); - } catch (ExportException ex) { - // Just like with indexing, a failure to export is not a fatal condition. - logger.log(Level.WARNING, "Curate Published DatasetVersion: exception while exporting metadata files:{0}", ex.getMessage()); + ctxt.engine().submit( + new UpdateDvObjectPIDMetadataCommand(savedDataset, getRequest())); + } catch (CommandException ex) { + //Make this non-fatal as after the DeleteDatasetVersionCommand, we can't roll back - for some reason no datasetfields remain in the DB + //(The old version doesn't need them and the new version doesn't get updated to include them?) + logger.log(Level.WARNING, "Curate Published DatasetVersion: exception while updating PID metadata:{0}", ex.getMessage()); } - - // Update so that getDataset() in updateDatasetUser will get the up-to-date copy // (with no draft version) setDataset(savedDataset); updateDatasetUser(ctxt); - - - return savedDataset; } + @Override + public boolean onSuccess(CommandContext ctxt, Object r) { + boolean retVal = true; + Dataset d = (Dataset) r; + + ctxt.index().asyncIndexDataset(d, true); + + // And the exported metadata files + try { + ExportService instance = ExportService.getInstance(); + instance.exportAllFormats(d); + } catch (ExportException ex) { + // Just like with indexing, a failure to export is not a fatal condition. + retVal = false; + logger.log(Level.WARNING, "Curate Published DatasetVersion: exception while exporting metadata files:{0}", ex.getMessage()); + } + return retVal; + } } From 1abf9d9988d806b67d73963bfacec78cbf47d5c3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 21 Aug 2024 15:10:00 -0400 Subject: [PATCH 02/20] add section on links for .rst files #8598 --- .../source/contributor/documentation.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index ebaf111aa4b..9506063b5fa 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -150,6 +150,18 @@ If the page is written in Markdown (.md), use this form: :depth: 3 ``` +### Links + +Getting links right with .rst files can be tricky. + +#### Custom Titles + +You can use a custom title when linking to a document like this: + + :doc:`Custom title ` + +See also + ### Images A good documentation is just like a website enhanced and upgraded by adding high quality and self-explanatory images. Often images depict a lot of written text in a simple manner. Within our Sphinx docs, you can add them in two ways: a) add a PNG image directly and include or b) use inline description languages like GraphViz (current only option). From 8503a3288bb0e5d5e4ff6fedb8aa1923ee096f94 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 22 Aug 2024 10:47:11 -0400 Subject: [PATCH 03/20] use DATAVERSE_SITEURL instead of _CT_ version #10756 The _CT_ version is not needed now that we have upgraded to Payara 6.2024.6 in #10495 which included this fix: https://github.com/payara/Payara/pull/6550 --- doc/sphinx-guides/source/container/running/demo.rst | 2 -- docker/compose/demo/compose.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/container/running/demo.rst b/doc/sphinx-guides/source/container/running/demo.rst index 0508639c616..f9642347558 100644 --- a/doc/sphinx-guides/source/container/running/demo.rst +++ b/doc/sphinx-guides/source/container/running/demo.rst @@ -124,8 +124,6 @@ Some JVM options can be configured as environment variables. For example, you ca We are in the process of making more JVM options configurable as environment variables. Look for the term "MicroProfile Config" in under :doc:`/installation/config` in the Installation Guide to know if you can use them this way. -Please note that for a few environment variables (the ones that start with ``%ct`` in :download:`microprofile-config.properties <../../../../../src/main/resources/META-INF/microprofile-config.properties>`), you have to prepend ``_CT_`` to make, for example, ``_CT_DATAVERSE_SITEURL``. We are working on a fix for this in https://github.com/IQSS/dataverse/issues/10285. - There is a final way to configure JVM options that we plan to deprecate once all JVM options have been converted to MicroProfile Config. Look for "magic trick" under "tunables" at :doc:`../app-image` for more information. Database Settings diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index e6ffc9f392a..33e7b52004b 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -9,7 +9,7 @@ services: restart: on-failure user: payara environment: - _CT_DATAVERSE_SITEURL: "https://demo.example.org" + DATAVERSE_SITEURL: "https://demo.example.org" DATAVERSE_DB_HOST: postgres DATAVERSE_DB_PASSWORD: secret DATAVERSE_DB_USER: dataverse From 3a9568e5a458157dab514e0a42d71fac70aaea33 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 23 Aug 2024 13:42:34 -0400 Subject: [PATCH 04/20] curate it test updates --- .../harvard/iq/dataverse/api/DatasetsIT.java | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 4f1ee1717c1..8bb33552fdd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3632,11 +3632,36 @@ public void testCuratePublishedDatasetVersionCommand() throws IOException { UtilIT.publishDatasetViaNativeApi(datasetId, "updatecurrent", apiToken).then().assertThat().statusCode(OK.getStatusCode()); + //Check that the dataset contains the updated metadata (which includes the name Spruce) Response getDatasetJsonAfterUpdate = UtilIT.nativeGet(datasetId, apiToken); - getDatasetJsonAfterUpdate.prettyPrint(); + assertTrue(getDatasetJsonAfterUpdate.prettyPrint().contains("Spruce")); getDatasetJsonAfterUpdate.then().assertThat() .statusCode(OK.getStatusCode()); + //Check that the draft version is gone + Response getDraft1 = UtilIT.getDatasetVersion(updatedContent, DS_VERSION_DRAFT, apiToken); + getDraft1.then().assertThat() + .statusCode(NOT_FOUND.getStatusCode()); + + + //Also test a terms change + String jsonLDTerms = "{'https://dataverse.org/schema/core#dataAccessPlace':'Somewhere'}"; + Response updateTerms = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, jsonLDTerms, true); + updateTerms.then().assertThat() + .statusCode(OK.getStatusCode()); + + //Verify the new term is there + Response jsonLDResponse = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + assertTrue(jsonLDResponse.prettyPrint().contains("Somewhere")); + jsonLDResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + //And that the draft is gone + Response getDraft2 = UtilIT.getDatasetVersion(updatedContent, DS_VERSION_DRAFT, apiToken); + getDraft2.then().assertThat() + .statusCode(NOT_FOUND.getStatusCode()); + + } /** From c93c8bd72d1f08699cab978b17d0f3c4b2ba8403 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 23 Aug 2024 13:46:52 -0400 Subject: [PATCH 05/20] minimal fix for changing terms issue --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 6a4c78d4f4f..03987aa2f60 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -12,6 +12,7 @@ import edu.harvard.iq.dataverse.util.DatasetFieldUtil; import edu.harvard.iq.dataverse.workflows.WorkflowComment; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.DataFile; @@ -19,6 +20,8 @@ import edu.harvard.iq.dataverse.DataFileCategory; import edu.harvard.iq.dataverse.DatasetVersionDifference; + +import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -56,6 +59,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { DatasetVersion newVersion = getDataset().getOrCreateEditVersion(); // Copy metadata from draft version to latest published version updateVersion.setDatasetFields(newVersion.initDatasetFields()); + newVersion.setDatasetFields(new ArrayList()); From 7c6e78341517feb57809492f24ce91cc7249c431 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 23 Aug 2024 15:06:18 -0400 Subject: [PATCH 06/20] Add RO-Crate to guides #10744 (Also add Croissant) --- doc/release-notes/10744-ro-crate-docs.md | 3 +++ doc/sphinx-guides/source/installation/advanced.rst | 5 ++++- doc/sphinx-guides/source/user/dataset-management.rst | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 doc/release-notes/10744-ro-crate-docs.md diff --git a/doc/release-notes/10744-ro-crate-docs.md b/doc/release-notes/10744-ro-crate-docs.md new file mode 100644 index 00000000000..9d52b4578b4 --- /dev/null +++ b/doc/release-notes/10744-ro-crate-docs.md @@ -0,0 +1,3 @@ +## RO-Crate Support (Metadata Export) + +Dataverse now supports [RO-Crate](https://www.researchobject.org/ro-crate/) in the sense that dataset metadata can be exported in that format. This functionality is not available out of the box but you can enable one or more RO-Crate exporters from the [list of external exporters](https://preview.guides.gdcc.io/en/develop/installation/advanced.html#inventory-of-external-exporters). See also #10744. diff --git a/doc/sphinx-guides/source/installation/advanced.rst b/doc/sphinx-guides/source/installation/advanced.rst index 03dc8d447c4..bee289ecd5b 100644 --- a/doc/sphinx-guides/source/installation/advanced.rst +++ b/doc/sphinx-guides/source/installation/advanced.rst @@ -136,7 +136,10 @@ Use the :ref:`dataverse.spi.exporters.directory` configuration option to specify Inventory of External Exporters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -For a list of external exporters, see the README at https://github.com/gdcc/dataverse-exporters +For a list of external exporters, see the README at https://github.com/gdcc/dataverse-exporters. To highlight a few: + +- Croissant +- RO-Crate Developing New Exporters ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index 9bf070ba0ee..6852b60575b 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -36,7 +36,10 @@ Once a dataset has been published, its metadata can be exported in a variety of - OpenAIRE - Schema.org JSON-LD -Additional formats can be enabled. See :ref:`inventory-of-external-exporters` in the Installation Guide. +Additional formats can be enabled. See :ref:`inventory-of-external-exporters` in the Installation Guide. To highlight a few: + +- Croissant +- RO-Crate Each of these metadata exports contains the metadata of the most recently published version of the dataset. From e67f53590f9bb8a59e2d1224000f58602fe97ab1 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 23 Aug 2024 16:35:53 -0400 Subject: [PATCH 07/20] don't use DeleteDV command could allow PID update to be fatal again when provider service is down (not tested so left as is) --- .../CuratePublishedDatasetVersionCommand.java | 88 ++++++++++++------- .../privateurl/PrivateUrlServiceBean.java | 2 +- 2 files changed, 58 insertions(+), 32 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 03987aa2f60..fd272b76e1c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -17,11 +17,12 @@ import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.RoleAssignment; import edu.harvard.iq.dataverse.DataFileCategory; import edu.harvard.iq.dataverse.DatasetVersionDifference; - import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -51,6 +52,9 @@ public Dataset execute(CommandContext ctxt) throws CommandException { if (!getUser().isSuperuser()) { throw new IllegalCommandException("Only superusers can curate published dataset versions", this); } + Dataset savedDataset = null; + // Merge the dataset into our JPA context + setDataset(ctxt.em().merge(getDataset())); ctxt.permissions().checkEditDatasetLock(getDataset(), getRequest(), this); // Invariant: Dataset has no locks preventing the update @@ -61,22 +65,21 @@ public Dataset execute(CommandContext ctxt) throws CommandException { updateVersion.setDatasetFields(newVersion.initDatasetFields()); newVersion.setDatasetFields(new ArrayList()); - - // final DatasetVersion editVersion = getDataset().getEditVersion(); DatasetFieldUtil.tidyUpFields(updateVersion.getDatasetFields(), true); - // Merge the new version into our JPA context - ctxt.em().merge(updateVersion); - TermsOfUseAndAccess oldTerms = updateVersion.getTermsOfUseAndAccess(); TermsOfUseAndAccess newTerms = newVersion.getTermsOfUseAndAccess(); newTerms.setDatasetVersion(updateVersion); updateVersion.setTermsOfUseAndAccess(newTerms); - //Put old terms on version that will be deleted.... - newVersion.setTermsOfUseAndAccess(oldTerms); - - //Validate metadata and TofA conditions + // Clear unnecessary terms relationships .... + newVersion.setTermsOfUseAndAccess(null); + oldTerms.setDatasetVersion(null); + // Without this there's a db exception related to the oldTerms being referenced + // by the datasetversion table at the flush around line 212 + ctxt.em().flush(); + + // Validate metadata and TofA conditions validateOrDie(updateVersion, isValidateLenient()); //Also set the fileaccessrequest boolean on the dataset to match the new terms @@ -89,19 +92,20 @@ public Dataset execute(CommandContext ctxt) throws CommandException { updateVersion.getWorkflowComments().addAll(newComments); } - // we have to merge to update the database but not flush because // we don't want to create two draft versions! - Dataset tempDataset = ctxt.em().merge(getDataset()); - + Dataset tempDataset = getDataset(); updateVersion = tempDataset.getLatestVersionForCopy(); // Look for file metadata changes and update published metadata if needed List pubFmds = updateVersion.getFileMetadatas(); int pubFileCount = pubFmds.size(); int newFileCount = tempDataset.getOrCreateEditVersion().getFileMetadatas().size(); - /* The policy for this command is that it should only be used when the change is a 'minor update' with no file changes. - * Nominally we could call .isMinorUpdate() for that but we're making the same checks as we go through the update here. + /* + * The policy for this command is that it should only be used when the change is + * a 'minor update' with no file changes. Nominally we could call + * .isMinorUpdate() for that but we're making the same checks as we go through + * the update here. */ if (pubFileCount != newFileCount) { logger.severe("Draft version of dataset: " + tempDataset.getId() + " has: " + newFileCount + " while last published version has " + pubFileCount); @@ -110,7 +114,10 @@ public Dataset execute(CommandContext ctxt) throws CommandException { Long thumbId = null; if(tempDataset.getThumbnailFile()!=null) { thumbId = tempDataset.getThumbnailFile().getId(); - }; + } + + // Note - Curate allows file metadata changes but not adding/deleting files. If + // that ever changes, this command needs to be updated. for (FileMetadata publishedFmd : pubFmds) { DataFile dataFile = publishedFmd.getDataFile(); FileMetadata draftFmd = dataFile.getLatestFileMetadata(); @@ -157,34 +164,53 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // Update modification time on the published version and the dataset updateVersion.setLastUpdateTime(getTimestamp()); tempDataset.setModificationTime(getTimestamp()); - ctxt.em().merge(updateVersion); - Dataset savedDataset = ctxt.em().merge(tempDataset); - - // Flush before calling DeleteDatasetVersion which calls - // PrivateUrlServiceBean.getPrivateUrlFromDatasetId() that will query the DB and - // fail if our changes aren't there - ctxt.em().flush(); + newVersion = ctxt.em().merge(newVersion); + savedDataset = ctxt.em().merge(tempDataset); // Now delete draft version - DeleteDatasetVersionCommand cmd; - cmd = new DeleteDatasetVersionCommand(getRequest(), savedDataset); - cmd.execute(ctxt); + ctxt.em().remove(newVersion); + + Iterator dvIt = savedDataset.getVersions().iterator(); + while (dvIt.hasNext()) { + DatasetVersion dv = dvIt.next(); + if (dv.isDraft()) { + dvIt.remove(); + break; // We've removed the draft version, no need to continue iterating + } + } + + savedDataset = ctxt.em().merge(savedDataset); + ctxt.em().flush(); + + RoleAssignment ra = ctxt.privateUrl().getPrivateUrlRoleAssignmentFromDataset(savedDataset); + if (ra != null) { + ctxt.roles().revoke(ra); + } // And update metadata at PID provider try { ctxt.engine().submit( - new UpdateDvObjectPIDMetadataCommand(savedDataset, getRequest())); + new UpdateDvObjectPIDMetadataCommand(savedDataset, getRequest())); } catch (CommandException ex) { - //Make this non-fatal as after the DeleteDatasetVersionCommand, we can't roll back - for some reason no datasetfields remain in the DB - //(The old version doesn't need them and the new version doesn't get updated to include them?) + // Make this non-fatal? This can be corrected by running the update PID API + // later, but who will look in the log? + // With the change to not use the DeleteDatasetVersionCommand above and other + // fixes, this error may now cleanly restore the initial state + // with the draft and last published versions unchanged. logger.log(Level.WARNING, "Curate Published DatasetVersion: exception while updating PID metadata:{0}", ex.getMessage()); } - // Update so that getDataset() in updateDatasetUser will get the up-to-date copy - // (with no draft version) + // Update so that getDataset() in updateDatasetUser() will get the up-to-date + // copy (with no draft version) setDataset(savedDataset); + updateDatasetUser(ctxt); + // ToDo - see if there are other DatasetVersionUser entries unique to the draft + // version that should be moved to the last published version + // As this command is intended for minor fixes, often done by the person pushing + // the update-current-version button, this is probably a minor issue. + return savedDataset; } diff --git a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlServiceBean.java index 9e5879106e4..01710e06f8f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlServiceBean.java @@ -96,7 +96,7 @@ private RoleAssignment getRoleAssignmentFromPrivateUrlToken(String privateUrlTok * * @todo This might be a good place for Optional. */ - private RoleAssignment getPrivateUrlRoleAssignmentFromDataset(Dataset dataset) { + public RoleAssignment getPrivateUrlRoleAssignmentFromDataset(Dataset dataset) { if (dataset == null) { return null; } From c2365416bbe73b23397c5221560e98432168ffdf Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 23 Aug 2024 16:58:19 -0400 Subject: [PATCH 08/20] release note/suggested fix in older versions --- .../10797-update-current-version-bug-fix.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/release-notes/10797-update-current-version-bug-fix.md diff --git a/doc/release-notes/10797-update-current-version-bug-fix.md b/doc/release-notes/10797-update-current-version-bug-fix.md new file mode 100644 index 00000000000..2cfaf69cad3 --- /dev/null +++ b/doc/release-notes/10797-update-current-version-bug-fix.md @@ -0,0 +1,11 @@ +A significant bug in the superuser-only "Update-Current-Version" publication was found and fixed in this release. If the Update-Current-Version option was used when changes were made to the dataset Terms (rather than to dataset metadata), or if the PID provider service was down/returned an error, the update would fail and render the dataset unusable and require restoration from a backup. The fix in this release allows the update to succeed in both of these cases and redesigns the functionality such that any unknown issues should not make the dataset unusable (i.e. the error would be reported and the dataset would remain in its current state with the last-published version as it was and changes still in the draft version.) + +Users of earlier Dataverse releases are encouraged to alert their superusers to this issue. Those who wish to disable this functionality have two options: +* Change the dataset.updateRelease entry in the Bundle.properties file (or local language version) to "Do Not Use" or similar (doesn't disable but alerts superusers to the issue), or +* Edit the dataset.xhtml file to remove the lines + + + + + +, delete the contents of the generated and osgi-cache directories in the Dataverse Payara domain, and restart the Payara server. From 10aa234de7d54a91f3d320b32ce96f1b2a360358 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 26 Aug 2024 10:30:35 -0400 Subject: [PATCH 09/20] update "how to make a PR" doc with current practices --- .../source/developers/version-control.rst | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 8648c8ce2a0..127955a44ea 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -137,17 +137,29 @@ Make a Pull Request ~~~~~~~~~~~~~~~~~~~ Make a pull request to get approval to merge your changes into the develop branch. -If the pull request notes indicate that release notes are necessary, the workflow can then verify the existence of a corresponding file and respond with a 'thank you!' message. On the other hand, if no release notes are detected, the contributor can be gently reminded of their absence. Please see :doc:`making-releases` for guidance on writing release notes. -Note that once a pull request is created, we'll remove the corresponding issue from our kanban board so that we're only tracking one card. -Feedback on the pull request template we use is welcome! Here's an example of a pull request for issue #3827: https://github.com/IQSS/dataverse/pull/3827 +Feedback on the pull request template we use is welcome! + +Here's an example of a pull request for issue #9729: https://github.com/IQSS/dataverse/pull/10474 + +Replace Issue with Pull Request +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the pull request closes an issue that has been prioritized, someone from the core team will do the following: + +- Move the open issue to the "Done" column of the `project board`_. We do this to track only one card, the pull request, on the project board. Merging the pull request will close the issue because we use the "closes #1234" `keyword `_ . +- Copy all labels from the issue to the pull request with the exception of the "size" label. +- Add a size label to the pull request that reflects the amount of review and QA time needed. +- Move the pull request to the "Ready for Review" column. + +.. _project board: https://github.com/orgs/IQSS/projects/34 Make Sure Your Pull Request Has Been Advanced to Code Review ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Now that you've made your pull request, your goal is to make sure it appears in the "Code Review" column at https://github.com/orgs/IQSS/projects/34. +Now that you've made your pull request, your goal is to make sure it appears in the "Code Review" column on the `project board`_. -Look at https://github.com/IQSS/dataverse/blob/master/CONTRIBUTING.md for various ways to reach out to developers who have enough access to the GitHub repo to move your issue and pull request to the "Code Review" column. +Look at :ref:`getting-help-developers` for various ways to reach out to developers who have enough access to the GitHub repo to move your issue and pull request to the "Code Review" column. Summary of Git commands ~~~~~~~~~~~~~~~~~~~~~~~ From 86f95719e53423dfc05b1b1442745d13b385cb08 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 26 Aug 2024 16:43:05 -0400 Subject: [PATCH 10/20] Clarify code comment. --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index fd272b76e1c..e6e8279a314 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -193,11 +193,12 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.engine().submit( new UpdateDvObjectPIDMetadataCommand(savedDataset, getRequest())); } catch (CommandException ex) { - // Make this non-fatal? This can be corrected by running the update PID API - // later, but who will look in the log? + // The try/catch makes this non-fatal. Should it be non-fatal - it's different from what we do in publish? + // This can be corrected by running the update PID API later, but who will look in the log? // With the change to not use the DeleteDatasetVersionCommand above and other // fixes, this error may now cleanly restore the initial state - // with the draft and last published versions unchanged. + // with the draft and last published versions unchanged, but this has not yet bee tested. + // (Alternately this could move to onSuccess if we intend it to stay non-fatal.) logger.log(Level.WARNING, "Curate Published DatasetVersion: exception while updating PID metadata:{0}", ex.getMessage()); } // Update so that getDataset() in updateDatasetUser() will get the up-to-date From 192585e917f4d88b4ce1b531c4dc8af9e8d6e2a1 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 26 Aug 2024 18:14:33 -0400 Subject: [PATCH 11/20] typos --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 8bb33552fdd..92032438962 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3639,7 +3639,7 @@ public void testCuratePublishedDatasetVersionCommand() throws IOException { .statusCode(OK.getStatusCode()); //Check that the draft version is gone - Response getDraft1 = UtilIT.getDatasetVersion(updatedContent, DS_VERSION_DRAFT, apiToken); + Response getDraft1 = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken); getDraft1.then().assertThat() .statusCode(NOT_FOUND.getStatusCode()); @@ -3657,7 +3657,7 @@ public void testCuratePublishedDatasetVersionCommand() throws IOException { .statusCode(OK.getStatusCode()); //And that the draft is gone - Response getDraft2 = UtilIT.getDatasetVersion(updatedContent, DS_VERSION_DRAFT, apiToken); + Response getDraft2 = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken); getDraft2.then().assertThat() .statusCode(NOT_FOUND.getStatusCode()); From 9464873b622c6dccea8f2f2f7451e3e682d7644c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 27 Aug 2024 09:14:13 -0400 Subject: [PATCH 12/20] remove deprecated call to makeSuperuser (and trigger tests) --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 92032438962..8d3841482a7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3626,7 +3626,7 @@ public void testCuratePublishedDatasetVersionCommand() throws IOException { UtilIT.publishDatasetViaNativeApi(datasetId, "updatecurrent", apiToken).then().assertThat().statusCode(FORBIDDEN.getStatusCode()); - Response makeSuperUser = UtilIT.makeSuperUser(username); + Response makeSuperUser = UtilIT.setSuperuserStatus(username, true); //should work after making super user From f276930000bb1f8a92caf93f6a4f08adaf1f9d11 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 27 Aug 2024 17:36:46 -0400 Subject: [PATCH 13/20] #10807 fix GB fragment --- src/main/webapp/guestbook-terms-popup-fragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml index d53c4bf4709..6eec4f100b1 100644 --- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml +++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml @@ -48,7 +48,7 @@ - From 3674a6f86c9df19ae0aac57ec4e3ebcee513fd01 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 28 Aug 2024 14:40:11 -0400 Subject: [PATCH 14/20] change server used in test to avoid bad cert --- .../iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java index 2c0e0a5c6b7..c57fa71a340 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java @@ -41,7 +41,7 @@ public class RemoteOverlayAccessIOTest { public void setUp() { System.setProperty("dataverse.files.test.type", "remote"); System.setProperty("dataverse.files.test.label", "testOverlay"); - System.setProperty("dataverse.files.test.base-url", "https://demo.dataverse.org/resources"); + System.setProperty("dataverse.files.test.base-url", "https://data.qdr.syr.edu/resources"); System.setProperty("dataverse.files.test.base-store", "file"); System.setProperty("dataverse.files.test.download-redirect", "true"); System.setProperty("dataverse.files.test.remote-store-name", "DemoDataCorp"); From b2b023a6c6c7a33214c80cd5cd3421c5735feb31 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 29 Aug 2024 10:34:42 -0400 Subject: [PATCH 15/20] fix test jsonld --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 8d3841482a7..9bebb22e321 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3645,7 +3645,7 @@ public void testCuratePublishedDatasetVersionCommand() throws IOException { //Also test a terms change - String jsonLDTerms = "{'https://dataverse.org/schema/core#dataAccessPlace':'Somewhere'}"; + String jsonLDTerms = "{\"https://dataverse.org/schema/core#fileTermsOfAccess\":{\"https://dataverse.org/schema/core#dataAccessPlace\":\"Somewhere\"}}"; Response updateTerms = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, jsonLDTerms, true); updateTerms.then().assertThat() .statusCode(OK.getStatusCode()); From 9f300752f3156d9c6c782dfcb514342dc4aafaeb Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 29 Aug 2024 11:53:56 -0400 Subject: [PATCH 16/20] another test fix --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 9bebb22e321..7476117d67d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3650,6 +3650,11 @@ public void testCuratePublishedDatasetVersionCommand() throws IOException { updateTerms.then().assertThat() .statusCode(OK.getStatusCode()); + //Run Update-Current Version again + + UtilIT.publishDatasetViaNativeApi(datasetId, "updatecurrent", apiToken).then().assertThat().statusCode(OK.getStatusCode()); + + //Verify the new term is there Response jsonLDResponse = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); assertTrue(jsonLDResponse.prettyPrint().contains("Somewhere")); From 92d3839f619f1534fcb705522755b04a4657f371 Mon Sep 17 00:00:00 2001 From: landreev Date: Tue, 3 Sep 2024 16:07:45 -0400 Subject: [PATCH 17/20] A Fix for the Dataset Thumbnail Bug on Publish, 10819 (#10820) * This is the fix for the bug in its entirety. #10819 * Making dataset thumbnail auto-selection optional (but enabled by default) #10819 * release note for the PR. #10819 * added a size == 0 check to DatasetUtil.getThumbnail (copied from #10687) #10819 * Added instructions on how to fix the files and datasets affected by the bug to the release note. #10819 * 2 extra words added to the release note #10819 * droped TIF from the error message in the thumbnail widgets, since it's not supported. (#10819) --- .../10819-publish-thumbnail-bug.md | 6 + .../edu/harvard/iq/dataverse/Dataset.java | 5 + .../dataverse/DatasetVersionServiceBean.java | 140 +++++++++--------- .../iq/dataverse/dataset/DatasetUtil.java | 8 +- .../iq/dataverse/settings/FeatureFlags.java | 10 ++ src/main/java/propertyFiles/Bundle.properties | 4 +- src/main/webapp/dataset.xhtml | 2 +- 7 files changed, 102 insertions(+), 73 deletions(-) create mode 100644 doc/release-notes/10819-publish-thumbnail-bug.md diff --git a/doc/release-notes/10819-publish-thumbnail-bug.md b/doc/release-notes/10819-publish-thumbnail-bug.md new file mode 100644 index 00000000000..46c9875a6ef --- /dev/null +++ b/doc/release-notes/10819-publish-thumbnail-bug.md @@ -0,0 +1,6 @@ +The initial release of the Dataverse v6.3 introduced a bug where publishing would break the dataset thumbnail, which in turn broke the rendering of the parent Collection ("dataverse") page. This problem was fixed in the PR 10820. + +This bug fix will prevent this from happening in the future, but does not fix any existing broken links. To restore any broken thumbnails caused by this bug, you can call the http://localhost:8080/api/admin/clearThumbnailFailureFlag API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the http://localhost:8080/api/admin/clearThumbnailFailureFlag/id to clear the flag for individual files. Calling the former, batch API is recommended. + +Additionally, the same PR made it possible to turn off the feature that automatically selects of one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by raising the feature flag `-Ddataverse.feature.disable-dataset-thumbnail-autoselect=true`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. + diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index eaf406d01bf..98766dca447 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -6,6 +6,7 @@ import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitations; import edu.harvard.iq.dataverse.makedatacount.DatasetMetrics; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Timestamp; @@ -206,6 +207,10 @@ public Dataset(boolean isHarvested) { StorageUse storageUse = new StorageUse(this); this.setStorageUse(storageUse); } + + if (FeatureFlags.DISABLE_DATASET_THUMBNAIL_AUTOSELECT.enabled()) { + this.setUseGenericThumbnail(true); + } } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java index 8ddc9545763..762319884b9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java @@ -9,6 +9,7 @@ import static edu.harvard.iq.dataverse.batch.jobs.importer.filesystem.FileRecordJobListener.SEP; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import edu.harvard.iq.dataverse.search.SolrSearchResult; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.MarkupChecker; @@ -807,36 +808,11 @@ public Long getThumbnailByVersionId(Long versionId) { return null; } - Long thumbnailFileId; - - // First, let's see if there are thumbnails that have already been - // generated: - try { - thumbnailFileId = (Long) em.createNativeQuery("SELECT df.id " - + "FROM datafile df, filemetadata fm, datasetversion dv, dvobject o " - + "WHERE dv.id = " + versionId + " " - + "AND df.id = o.id " - + "AND fm.datasetversion_id = dv.id " - + "AND fm.datafile_id = df.id " - + "AND df.restricted = false " - + "AND df.embargo_id is null " - + "AND df.retention_id is null " - + "AND o.previewImageAvailable = true " - + "ORDER BY df.id LIMIT 1;").getSingleResult(); - } catch (Exception ex) { - thumbnailFileId = null; - } - - if (thumbnailFileId != null) { - logger.fine("DatasetVersionService,getThumbnailByVersionid(): found already generated thumbnail for version " + versionId + ": " + thumbnailFileId); - assignDatasetThumbnailByNativeQuery(versionId, thumbnailFileId); - return thumbnailFileId; - } - - if (!systemConfig.isThumbnailGenerationDisabledForImages()) { - // OK, let's try and generate an image thumbnail! - long imageThumbnailSizeLimit = systemConfig.getThumbnailSizeLimitImage(); + if (!FeatureFlags.DISABLE_DATASET_THUMBNAIL_AUTOSELECT.enabled()) { + Long thumbnailFileId; + // First, let's see if there are thumbnails that have already been + // generated: try { thumbnailFileId = (Long) em.createNativeQuery("SELECT df.id " + "FROM datafile df, filemetadata fm, datasetversion dv, dvobject o " @@ -844,63 +820,89 @@ public Long getThumbnailByVersionId(Long versionId) { + "AND df.id = o.id " + "AND fm.datasetversion_id = dv.id " + "AND fm.datafile_id = df.id " - + "AND o.previewimagefail = false " + "AND df.restricted = false " + "AND df.embargo_id is null " + "AND df.retention_id is null " - + "AND df.contenttype LIKE 'image/%' " - + "AND NOT df.contenttype = 'image/fits' " - + "AND df.filesize < " + imageThumbnailSizeLimit + " " - + "ORDER BY df.filesize ASC LIMIT 1;").getSingleResult(); + + "AND o.previewImageAvailable = true " + + "ORDER BY df.id LIMIT 1;").getSingleResult(); } catch (Exception ex) { thumbnailFileId = null; } if (thumbnailFileId != null) { - logger.fine("obtained file id: " + thumbnailFileId); - DataFile thumbnailFile = datafileService.find(thumbnailFileId); - if (thumbnailFile != null) { - if (datafileService.isThumbnailAvailable(thumbnailFile)) { - assignDatasetThumbnailByNativeQuery(versionId, thumbnailFileId); - return thumbnailFileId; + logger.fine("DatasetVersionService,getThumbnailByVersionid(): found already generated thumbnail for version " + versionId + ": " + thumbnailFileId); + assignDatasetThumbnailByNativeQuery(versionId, thumbnailFileId); + return thumbnailFileId; + } + + if (!systemConfig.isThumbnailGenerationDisabledForImages()) { + // OK, let's try and generate an image thumbnail! + long imageThumbnailSizeLimit = systemConfig.getThumbnailSizeLimitImage(); + + try { + thumbnailFileId = (Long) em.createNativeQuery("SELECT df.id " + + "FROM datafile df, filemetadata fm, datasetversion dv, dvobject o " + + "WHERE dv.id = " + versionId + " " + + "AND df.id = o.id " + + "AND fm.datasetversion_id = dv.id " + + "AND fm.datafile_id = df.id " + + "AND o.previewimagefail = false " + + "AND df.restricted = false " + + "AND df.embargo_id is null " + + "AND df.retention_id is null " + + "AND df.contenttype LIKE 'image/%' " + + "AND NOT df.contenttype = 'image/fits' " + + "AND df.filesize < " + imageThumbnailSizeLimit + " " + + "ORDER BY df.filesize ASC LIMIT 1;").getSingleResult(); + } catch (Exception ex) { + thumbnailFileId = null; + } + + if (thumbnailFileId != null) { + logger.fine("obtained file id: " + thumbnailFileId); + DataFile thumbnailFile = datafileService.find(thumbnailFileId); + if (thumbnailFile != null) { + if (datafileService.isThumbnailAvailable(thumbnailFile)) { + assignDatasetThumbnailByNativeQuery(versionId, thumbnailFileId); + return thumbnailFileId; + } } } } - } - // And if that didn't work, try the same thing for PDFs: - if (!systemConfig.isThumbnailGenerationDisabledForPDF()) { - // OK, let's try and generate an image thumbnail! - long imageThumbnailSizeLimit = systemConfig.getThumbnailSizeLimitPDF(); - try { - thumbnailFileId = (Long) em.createNativeQuery("SELECT df.id " - + "FROM datafile df, filemetadata fm, datasetversion dv, dvobject o " - + "WHERE dv.id = " + versionId + " " - + "AND df.id = o.id " - + "AND fm.datasetversion_id = dv.id " - + "AND fm.datafile_id = df.id " - + "AND o.previewimagefail = false " - + "AND df.restricted = false " - + "AND df.embargo_id is null " - + "AND df.retention_id is null " - + "AND df.contenttype = 'application/pdf' " - + "AND df.filesize < " + imageThumbnailSizeLimit + " " - + "ORDER BY df.filesize ASC LIMIT 1;").getSingleResult(); - } catch (Exception ex) { - thumbnailFileId = null; - } + // And if that didn't work, try the same thing for PDFs: + if (!systemConfig.isThumbnailGenerationDisabledForPDF()) { + // OK, let's try and generate an image thumbnail! + long imageThumbnailSizeLimit = systemConfig.getThumbnailSizeLimitPDF(); + try { + thumbnailFileId = (Long) em.createNativeQuery("SELECT df.id " + + "FROM datafile df, filemetadata fm, datasetversion dv, dvobject o " + + "WHERE dv.id = " + versionId + " " + + "AND df.id = o.id " + + "AND fm.datasetversion_id = dv.id " + + "AND fm.datafile_id = df.id " + + "AND o.previewimagefail = false " + + "AND df.restricted = false " + + "AND df.embargo_id is null " + + "AND df.retention_id is null " + + "AND df.contenttype = 'application/pdf' " + + "AND df.filesize < " + imageThumbnailSizeLimit + " " + + "ORDER BY df.filesize ASC LIMIT 1;").getSingleResult(); + } catch (Exception ex) { + thumbnailFileId = null; + } - if (thumbnailFileId != null) { - DataFile thumbnailFile = datafileService.find(thumbnailFileId); - if (thumbnailFile != null) { - if (datafileService.isThumbnailAvailable(thumbnailFile)) { - assignDatasetThumbnailByNativeQuery(versionId, thumbnailFileId); - return thumbnailFileId; + if (thumbnailFileId != null) { + DataFile thumbnailFile = datafileService.find(thumbnailFileId); + if (thumbnailFile != null) { + if (datafileService.isThumbnailAvailable(thumbnailFile)) { + assignDatasetThumbnailByNativeQuery(versionId, thumbnailFileId); + return thumbnailFileId; + } } } } } - return null; } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index 4c84384b271..cacd409b365 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -113,13 +113,19 @@ public static List getThumbnailCandidates(Dataset dataset, boo * * @param dataset * @param datasetVersion - * @return + * @param size of the requested thumbnail + * @return DatasetThumbnail object, or null if not available */ public static DatasetThumbnail getThumbnail(Dataset dataset, DatasetVersion datasetVersion, int size) { if (dataset == null) { return null; } + if (size == 0) { + // Size 0 will fail (and set the failure flag) and should never be sent + logger.warning("getThumbnail called with size 0"); + return null; + } StorageIO dataAccess = null; try{ diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 021977ff8c6..2bfda69247a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -91,6 +91,16 @@ public enum FeatureFlags { * @since Dataverse 6.3 */ DISABLE_RETURN_TO_AUTHOR_REASON("disable-return-to-author-reason"), + /** + * This flag disables the feature that automatically selects one of the + * DataFile thumbnails in the dataset/version as the dedicated thumbnail + * for the dataset. + * + * @apiNote Raise flag by setting + * "dataverse.feature.enable-dataset-thumbnail-autoselect" + * @since Dataverse 6.4 + */ + DISABLE_DATASET_THUMBNAIL_AUTOSELECT("disable-dataset-thumbnail-autoselect"), ; final String flag; diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 4fc7ce88a7f..b53f0e84400 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1064,7 +1064,7 @@ dataverse.theme.logo.imageFooter=Footer Image dataverse.theme.logo.image.title=The logo or image file you wish to display in the header of this dataverse. dataverse.theme.logo.image.footer=The logo or image file you wish to display in the footer of this dataverse. dataverse.theme.logo.image.uploadNewFile=Upload New File -dataverse.theme.logo.image.invalidMsg=The image could not be uploaded. Please try again with a JPG, TIF, or PNG file. +dataverse.theme.logo.image.invalidMsg=The image could not be uploaded. Please try again with a JPG or PNG file. dataverse.theme.logo.image.uploadImgFile=Upload Image File dataverse.theme.logo.format.title=The shape for the logo or image file you upload for this dataverse. dataverse.theme.logo.alignment.title=Where the logo or image should display in the header or footer. @@ -2166,7 +2166,7 @@ dataset.thumbnailsAndWidget.thumbnailImage.selectAvailable.title=Select a thumbn dataset.thumbnailsAndWidget.thumbnailImage.uploadNew=Upload New File dataset.thumbnailsAndWidget.thumbnailImage.uploadNew.title=Upload an image file as your dataset thumbnail, which will be stored separately from the data files that belong to your dataset. dataset.thumbnailsAndWidget.thumbnailImage.upload=Upload Image -dataset.thumbnailsAndWidget.thumbnailImage.upload.invalidMsg=The image could not be uploaded. Please try again with a JPG, TIF, or PNG file. +dataset.thumbnailsAndWidget.thumbnailImage.upload.invalidMsg=The image could not be uploaded. Please try again with a JPG or PNG file. dataset.thumbnailsAndWidget.thumbnailImage.alt=Thumbnail image selected for dataset dataset.thumbnailsAndWidget.success=Dataset thumbnail updated. dataset.thumbnailsAndWidget.removeThumbnail=Remove Thumbnail diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 4cece74b67a..0118e6decb2 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -86,7 +86,7 @@ - + From 05962ef949d57f7d5e398d5e2e147f4d00c9ddc3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 4 Sep 2024 10:24:43 -0400 Subject: [PATCH 18/20] add details to error messages (#10813) --- .../errorhandlers/WebApplicationExceptionHandler.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java index e67e91e63c9..af9aeffa1c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java @@ -18,6 +18,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.lang3.StringUtils; + /** * Catches all types of web application exceptions like NotFoundException, etc etc and handles them properly. */ @@ -49,7 +51,14 @@ public Response toResponse(WebApplicationException ex) { } else if ((ex.getMessage() + "").toLowerCase().startsWith("no permission to download file")) { jrb.message(BundleUtil.getStringFromBundle("access.api.exception.metadata.restricted.no.permission")); } else { - jrb.message("Bad Request. The API request cannot be completed with the parameters supplied. Please check your code for typos, or consult our API guide at http://guides.dataverse.org."); + String msg = ex.getMessage(); + msg = StringUtils.isEmpty(msg) + ? "Bad Request. The API request cannot be completed with the parameters supplied. Please check your code for typos, or consult our API guide at http://guides.dataverse.org." + : "Bad Request. The API request cannot be completed with the parameters supplied. Details: " + + msg + + " - please check your code for typos, or consult our API guide at http://guides.dataverse.org."; + + jrb.message(msg); jrb.request(request); } break; From 3834bbf7cdb04e59e31ee789db79cb015bb161db Mon Sep 17 00:00:00 2001 From: Guillermo Portas Date: Wed, 4 Sep 2024 15:25:20 +0100 Subject: [PATCH 19/20] Fix addDataverse expected request body structure (#10802) * Fixed: MetadataBlockServiceBean to check for not excluded fields in input levels * Changed: using queries for obtaining dataset field types based on displaying conditions * Refactor: json printer method for MetadataBlock * Added: IT test case for list metadata blocks testing field with include=false and displayOnCreate=true property * Fixed: removed condition in MetadataBlockServiceBean * Added: release notes for #10741 * Fixed: displayOnCreate query logic * Fixed: excluding conditionally required fields when display-on-create is true * Fixed: query predicate for required-in-dataverse field condition * Fixed: addDataverse API facetIds field json structure * Added: docs #10800 --- .../10800-add-dataverse-request-json-fix.md | 1 + .../edu/harvard/iq/dataverse/api/Dataverses.java | 7 ++++--- .../edu/harvard/iq/dataverse/api/DataversesIT.java | 13 ++++++++++--- .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 6 +++--- 4 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 doc/release-notes/10800-add-dataverse-request-json-fix.md diff --git a/doc/release-notes/10800-add-dataverse-request-json-fix.md b/doc/release-notes/10800-add-dataverse-request-json-fix.md new file mode 100644 index 00000000000..ddd6c388ec6 --- /dev/null +++ b/doc/release-notes/10800-add-dataverse-request-json-fix.md @@ -0,0 +1 @@ +Fixed the "addDataverse" API endpoint (/dataverses/{id} POST) expected request JSON structure to parse facetIds as described in the docs. \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index ed2a8db5e06..75e3456ab27 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -143,16 +143,17 @@ public Response addDataverse(@Context ContainerRequestContext crc, String body, JsonObject metadataBlocksJson = newDataverseJson.getJsonObject("metadataBlocks"); List inputLevels = null; List metadataBlocks = null; + List facetList = null; if (metadataBlocksJson != null) { JsonArray inputLevelsArray = metadataBlocksJson.getJsonArray("inputLevels"); inputLevels = inputLevelsArray != null ? parseInputLevels(inputLevelsArray, newDataverse) : null; JsonArray metadataBlockNamesArray = metadataBlocksJson.getJsonArray("metadataBlockNames"); metadataBlocks = metadataBlockNamesArray != null ? parseNewDataverseMetadataBlocks(metadataBlockNamesArray) : null; - } - JsonArray facetIdsArray = newDataverseJson.getJsonArray("facetIds"); - List facetList = facetIdsArray != null ? parseFacets(facetIdsArray) : null; + JsonArray facetIdsArray = metadataBlocksJson.getJsonArray("facetIds"); + facetList = facetIdsArray != null ? parseFacets(facetIdsArray) : null; + } if (!parentIdtf.isEmpty()) { Dataverse owner = findDataverseOrDie(parentIdtf); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index d682e4ade98..f3c74f028e4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -26,6 +26,7 @@ import static jakarta.ws.rs.core.Response.Status.*; import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.hasItemInArray; import static org.junit.jupiter.api.Assertions.*; @@ -998,16 +999,22 @@ public void testAddDataverse() { String actualMetadataBlockName = listMetadataBlocksResponse.then().extract().path("data[0].name"); assertEquals(actualMetadataBlockName, "citation"); + // Assert root facets are configured + String[] expectedRootFacetIds = {"authorName", "subject", "keywordValue", "dateOfDeposit"}; + Response listDataverseFacetsResponse = UtilIT.listDataverseFacets(testDataverseAlias, apiToken); + List actualFacetNames = listDataverseFacetsResponse.then().extract().path("data"); + assertThat("Facet names should match expected root facet ids", actualFacetNames, containsInAnyOrder(expectedRootFacetIds)); + // With optional input levels and facet ids String[] testInputLevelNames = {"geographicCoverage", "country"}; - String[] testFacetIds = {"authorName", "authorAffiliation"}; + String[] testFacetIds = {"language", "contributorName"}; String[] testMetadataBlockNames = {"citation", "geospatial"}; testDataverseAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; createSubDataverseResponse = UtilIT.createSubDataverse(testDataverseAlias, null, apiToken, "root", testInputLevelNames, testFacetIds, testMetadataBlockNames); createSubDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); - // Assert facets are configured - Response listDataverseFacetsResponse = UtilIT.listDataverseFacets(testDataverseAlias, apiToken); + // Assert custom facets are configured + listDataverseFacetsResponse = UtilIT.listDataverseFacets(testDataverseAlias, apiToken); String actualFacetName1 = listDataverseFacetsResponse.then().extract().path("data[0]"); String actualFacetName2 = listDataverseFacetsResponse.then().extract().path("data[1]"); assertNotEquals(actualFacetName1, actualFacetName2); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 8f1fcdf57eb..110dc7e570e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -398,16 +398,16 @@ static Response createSubDataverse(String alias, String category, String apiToke metadataBlocksObjectBuilder.add("metadataBlockNames", metadataBlockNamesArrayBuilder); } - objectBuilder.add("metadataBlocks", metadataBlocksObjectBuilder); - if (facetIds != null) { JsonArrayBuilder facetIdsArrayBuilder = Json.createArrayBuilder(); for(String facetId : facetIds) { facetIdsArrayBuilder.add(facetId); } - objectBuilder.add("facetIds", facetIdsArrayBuilder); + metadataBlocksObjectBuilder.add("facetIds", facetIdsArrayBuilder); } + objectBuilder.add("metadataBlocks", metadataBlocksObjectBuilder); + JsonObject dvData = objectBuilder.build(); Response createDataverseResponse = given() .body(dvData.toString()).contentType(ContentType.JSON) From d4b9260e22694c9ff7530551e087d4ba386a39c3 Mon Sep 17 00:00:00 2001 From: landreev Date: Wed, 4 Sep 2024 11:09:03 -0400 Subject: [PATCH 20/20] A one line fix for #10821 - ? (#10823) --- src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index dab0ff43fcf..d3926742b57 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -851,6 +851,7 @@ public Dataset setNonDatasetFileAsThumbnail(Dataset dataset, InputStream inputSt } dataset = DatasetUtil.persistDatasetLogoToStorageAndCreateThumbnails(dataset, inputStream); dataset.setThumbnailFile(null); + dataset.setUseGenericThumbnail(false); return merge(dataset); }