Merge remote-tracking branch 'IQSS/develop' into IQSS-4894-Open_Graph…

…_metadata
ErykKul · Apr 23, 2024 · 1e81b44 · 1e81b44
2 parents e35f835 + 9bda7dd
commit 1e81b44
Show file tree

Hide file tree

Showing 126 changed files with 2,097 additions and 661 deletions.
diff --git a/.env b/.env
@@ -1,5 +1,5 @@
 APP_IMAGE=gdcc/dataverse:unstable
-POSTGRES_VERSION=13
+POSTGRES_VERSION=16
 DATAVERSE_DB_USER=dataverse
 SOLR_VERSION=9.3.0
 SKIP_DEPLOY=0
diff --git a/.github/workflows/maven_cache_management.yml b/.github/workflows/maven_cache_management.yml
@@ -0,0 +1,101 @@
+name: Maven Cache Management
+
+on:
+    # Every push to develop should trigger cache rejuvenation (dependencies might have changed)
+    push:
+        branches:
+            - develop
+    # According to https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy
+    # all caches are deleted after 7 days of no access. Make sure we rejuvenate every 7 days to keep it available.
+    schedule:
+        - cron: '23 2 * * 0' # Run for 'develop' every Sunday at 02:23 UTC (3:23 CET, 21:23 ET)
+    # Enable manual cache management
+    workflow_dispatch:
+    # Delete branch caches once a PR is merged
+    pull_request:
+        types:
+            - closed
+
+env:
+    COMMON_CACHE_KEY: "dataverse-maven-cache"
+    COMMON_CACHE_PATH: "~/.m2/repository"
+
+jobs:
+    seed:
+        name: Drop and Re-Seed Local Repository
+        runs-on: ubuntu-latest
+        if: ${{ github.event_name != 'pull_request' }}
+        permissions:
+            # Write permission needed to delete caches
+            # See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id
+            actions: write
+            contents: read
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+            - name: Determine Java version from Parent POM
+              run: echo "JAVA_VERSION=$(grep '<target.java.version>' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV}
+            - name: Set up JDK ${{ env.JAVA_VERSION }}
+              uses: actions/setup-java@v4
+              with:
+                  java-version: ${{ env.JAVA_VERSION }}
+                  distribution: temurin
+            - name: Seed common cache
+              run: |
+                  mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins
+            # This non-obvious order is due to the fact that the download via Maven above will take a very long time (7-8 min).
+            # Jobs should not be left without a cache. Deleting and saving in one go leaves only a small chance for a cache miss.
+            - name: Drop common cache
+              run: |
+                  gh extension install actions/gh-actions-cache                  
+                  echo "🛒 Fetching list of cache keys"
+                  cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -B develop | cut -f 1 )
+                  
+                  ## Setting this to not fail the workflow while deleting cache keys. 
+                  set +e
+                  echo "🗑️ Deleting caches..."
+                  for cacheKey in $cacheKeys
+                  do
+                      gh actions-cache delete $cacheKey -R ${{ github.repository }} -B develop --confirm
+                  done
+                  echo "✅ Done"
+              env:
+                  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            - name: Save the common cache
+              uses: actions/cache@v4
+              with:
+                  path: ${{ env.COMMON_CACHE_PATH }}
+                  key: ${{ env.COMMON_CACHE_KEY }}
+                  enableCrossOsArchive: true
+
+    # Let's delete feature branch caches once their PR is merged - we only have 10 GB of space before eviction kicks in
+    deplete:
+        name: Deplete feature branch caches
+        runs-on: ubuntu-latest
+        if: ${{ github.event_name == 'pull_request' }}
+        permissions:
+            # `actions:write` permission is required to delete caches
+            # See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id
+            actions: write
+            contents: read
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+            - name: Cleanup caches
+              run: |
+                  gh extension install actions/gh-actions-cache
+
+                  BRANCH=refs/pull/${{ github.event.pull_request.number }}/merge
+                  echo "🛒 Fetching list of cache keys"
+                  cacheKeysForPR=$(gh actions-cache list -R ${{ github.repository }} -B $BRANCH | cut -f 1 )
+                
+                  ## Setting this to not fail the workflow while deleting cache keys. 
+                  set +e
+                  echo "🗑️ Deleting caches..."
+                  for cacheKey in $cacheKeysForPR
+                  do
+                      gh actions-cache delete $cacheKey -R ${{ github.repository }} -B $BRANCH --confirm
+                  done
+                  echo "✅ Done"
+              env:
+                  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/doc/release-notes/10022_upload_redirect_without_tagging.md b/doc/release-notes/10022_upload_redirect_without_tagging.md
@@ -0,0 +1,5 @@
+If your S3 store does not support tagging and gives an error if you configure direct uploads, you can disable the tagging by using the ``dataverse.files.<id>.disable-tagging`` JVM option. For more details see https://dataverse-guide--10029.org.readthedocs.build/en/10029/developers/big-data-support.html#s3-tags #10022 and #10029.
+
+## New config options
+
+- dataverse.files.<id>.disable-tagging
diff --git a/doc/release-notes/10101-qa-guide.md b/doc/release-notes/10101-qa-guide.md
diff --git a/doc/release-notes/10117-api-metrics-add-user-accounts-number.md b/doc/release-notes/10117-api-metrics-add-user-accounts-number.md
diff --git a/doc/release-notes/10155-datasets-can-download-at-least-one-file.md b/doc/release-notes/10155-datasets-can-download-at-least-one-file.md
diff --git a/doc/release-notes/10202-extend-getVersionFiles-api-to-include-total-file-count.md b/doc/release-notes/10202-extend-getVersionFiles-api-to-include-total-file-count.md
diff --git a/doc/release-notes/10216-metadatablocks.md b/doc/release-notes/10216-metadatablocks.md
diff --git a/doc/release-notes/10238-container-demo.md b/doc/release-notes/10238-container-demo.md
diff --git a/doc/release-notes/10240-file-citation.md b/doc/release-notes/10240-file-citation.md
diff --git a/doc/release-notes/10242-add-feature-dv-api b/doc/release-notes/10242-add-feature-dv-api
@@ -0,0 +1 @@
+New api endpoints have been added to allow you to add or remove featured collections from a dataverse collection.
diff --git a/doc/release-notes/10280-get-file-api-extension.md b/doc/release-notes/10280-get-file-api-extension.md
diff --git a/doc/release-notes/10286-return-owner-added-to-get-apis.md b/doc/release-notes/10286-return-owner-added-to-get-apis.md
diff --git a/doc/release-notes/10297-metadata-api-fix.md b/doc/release-notes/10297-metadata-api-fix.md
diff --git a/doc/release-notes/10316_cvoc_http_headers.md b/doc/release-notes/10316_cvoc_http_headers.md
@@ -0,0 +1,5 @@
+You are now able to add HTTP request headers required by the External Vocabulary Services you are implementing.
+
+A combined documentation can be found on pull request [#10404](https://github.com/IQSS/dataverse/pull/10404).
+
+For more information, see issue [#10316](https://github.com/IQSS/dataverse/issues/10316) and pull request [gddc/dataverse-external-vocab-support#19](https://github.com/gdcc/dataverse-external-vocab-support/pull/19).
diff --git a/doc/release-notes/10318-uningest-and-reingest.md b/doc/release-notes/10318-uningest-and-reingest.md
diff --git a/doc/release-notes/10330-api-change-latest-version-status.md b/doc/release-notes/10330-api-change-latest-version-status.md
@@ -0,0 +1 @@
+The API endpoint for getting the Dataset version has been extended to include latestVersionPublishingStatus.
diff --git a/doc/release-notes/10338-expose-and-sort-publish-status-facet.md b/doc/release-notes/10338-expose-and-sort-publish-status-facet.md
diff --git a/doc/release-notes/10339-workflow.md b/doc/release-notes/10339-workflow.md
@@ -0,0 +1,3 @@
+The computational workflow metadata block has been updated to present a clickable link for the External Code Repository URL field.
+
+Release notes should include the usual instructions, for those who have installed this optional block, to update the computational_workflow block. (PR#10441)
diff --git a/doc/release-notes/10342-assign-roles-without-privilege-escalation.md b/doc/release-notes/10342-assign-roles-without-privilege-escalation.md
diff --git a/doc/release-notes/10360-binder-redirect.md b/doc/release-notes/10360-binder-redirect.md
diff --git a/doc/release-notes/10381-index-after-publish.md b/doc/release-notes/10381-index-after-publish.md
diff --git a/doc/release-notes/10382-optional-croissant-exporter.md b/doc/release-notes/10382-optional-croissant-exporter.md
diff --git a/doc/release-notes/10389-metadatablocks-api-extension.md b/doc/release-notes/10389-metadatablocks-api-extension.md
@@ -0,0 +1,6 @@
+New optional query parameters added to ``api/metadatablocks`` and ``api/dataverses/{id}/metadatablocks`` endpoints:
+
+- ``returnDatasetFieldTypes``: Whether or not to return the dataset field types present in each metadata block. If not set, the default value is false.
+- ``onlyDisplayedOnCreate``: Whether or not to return only the metadata blocks that are displayed on dataset creation. If ``returnDatasetFieldTypes`` is true, only the dataset field types shown on dataset creation will be returned within each metadata block. If not set, the default value is false.
+
+Added new ``displayOnCreate`` field to the MetadataBlock and DatasetFieldType payloads.
diff --git a/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md b/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md
@@ -0,0 +1,4 @@
+For scenarios involving API calls related to large datasets (Numerous files, for example: ~10k) it has been optimized:
+
+- The search API endpoint.
+- The permission checking logic present in PermissionServiceBean.
diff --git a/doc/release-notes/10424-new-api-for-mdc.md b/doc/release-notes/10424-new-api-for-mdc.md
diff --git a/doc/release-notes/10464-add-name-harvesting-client-facet.md b/doc/release-notes/10464-add-name-harvesting-client-facet.md
@@ -0,0 +1,3 @@
+The Metadata Source facet has been updated to show the name of the harvesting client rather than grouping all such datasets under 'harvested'
+
+TODO: for the v6.13 release note: Please add a full re-index using http://localhost:8080/api/admin/index to the upgrade instructions. 
diff --git a/doc/release-notes/10468-doc-datalad-integration.md b/doc/release-notes/10468-doc-datalad-integration.md
@@ -0,0 +1 @@
+DataLad has been integrated with Dataverse. For more information, see https://dataverse-guide--10470.org.readthedocs.build/en/10470/admin/integrations.html#datalad
diff --git a/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md b/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md
@@ -0,0 +1,22 @@
+The Dataverse object returned by /api/dataverses has been extended to include "isReleased": {boolean}.
+```javascript
+{
+    "status": "OK",
+        "data": {
+        "id": 32,
+            "alias": "dv6f645bb5",
+            "name": "dv6f645bb5",
+            "dataverseContacts": [
+            {
+                "displayOrder": 0,
+                "contactEmail": "54180268@mailinator.com"
+            }
+        ],
+            "permissionRoot": true,
+            "dataverseType": "UNCATEGORIZED",
+            "ownerId": 1,
+            "creationDate": "2024-04-12T18:05:59Z",
+            "isReleased": true
+    }
+}
+```
diff --git a/doc/release-notes/3437-new-index-api-added.md b/doc/release-notes/3437-new-index-api-added.md
diff --git a/doc/release-notes/3623-multipid.md b/doc/release-notes/3623-multipid.md
diff --git a/doc/release-notes/3702-return-to-author.md b/doc/release-notes/3702-return-to-author.md
diff --git a/doc/release-notes/5645-geospatial-props-nslong-fix.md b/doc/release-notes/5645-geospatial-props-nslong-fix.md
diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
@@ -247,7 +247,7 @@ Upgrading requires a maintenance window and downtime. Please plan ahead, create
 
 These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.0.
 
-0\. These instructions assume that you are upgrading from 6.0. If you are running an earlier version, the only safe way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to 5.14. 
+0\. These instructions assume that you are upgrading from 6.0. If you are running an earlier version, the only safe way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to 6.1. 
 
 If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user.
 
@@ -288,6 +288,7 @@ As noted above, deployment of the war file might take several minutes due a data
 
 6a\. Update Citation Metadata Block (to make Alternative Title repeatable)
 
+- `wget https://github.com/IQSS/dataverse/releases/download/v6.1/citation.tsv`
 - `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
 
 7\. Upate Solr schema.xml to allow multiple Alternative Titles to be used. See specific instructions below for those installations without custom metadata blocks (7a) and those with  custom metadata blocks  (7b).