diff --git a/.env b/.env index d5cffcec0aa..9d604630073 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ APP_IMAGE=gdcc/dataverse:unstable -POSTGRES_VERSION=16 +POSTGRES_VERSION=17 DATAVERSE_DB_USER=dataverse SOLR_VERSION=9.3.0 -SKIP_DEPLOY=0 \ No newline at end of file +SKIP_DEPLOY=0 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7e6995d76d9..3dba7d52109 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -14,7 +14,7 @@ Thank you for contributing to the Dataverse Project through the creation of a bu WARNING: If this is a security issue it should be reported privately to security@dataverse.org More information on bug issues and contributions can be found in the "Contributing to Dataverse" page: -https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md#bug-reportsissues +https://guides.dataverse.org/en/latest/contributor/index.html Please fill out as much of the template as you can. Start below this comment section. @@ -44,7 +44,6 @@ Start below this comment section. **Any related open or closed issues to this bug report?** - **Screenshots:** No matter the issue, screenshots are always welcome. @@ -53,3 +52,7 @@ To add a screenshot, please use one of the following formats and/or methods desc * https://help.github.com/en/articles/file-attachments-on-issues-and-pull-requests * + + +**Are you thinking about creating a pull request for this issue?** +Help is always welcome, is this bug something you or your organization plan to fix? diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index d6248537418..7365cb4317c 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,7 +1,7 @@ --- name: Feature request about: Suggest an idea or new feature for the Dataverse software! -title: 'Feature Request/Idea:' +title: 'Feature Request:' labels: 'Type: Feature' assignees: '' @@ -11,7 +11,7 @@ assignees: '' Thank you for contributing to the Dataverse Project through the creation of a feature request! More information on ideas/feature requests and contributions can be found in the "Contributing to Dataverse" page: -https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md#ideasfeature-requests +https://guides.dataverse.org/en/latest/contributor/index.html Please fill out as much of the template as you can. Start below this comment section. @@ -34,3 +34,6 @@ Start below this comment section. **Any open or closed issues related to this feature request?** + +**Are you thinking about creating a pull request for this feature?** +Help is always welcome, is this feature something you or your organization plan to implement? diff --git a/.github/ISSUE_TEMPLATE/idea_proposal.md b/.github/ISSUE_TEMPLATE/idea_proposal.md new file mode 100644 index 00000000000..8cb6c7bfafe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/idea_proposal.md @@ -0,0 +1,40 @@ +--- +name: Idea proposal +about: Propose a new idea for discussion to improve the Dataverse software! +title: 'Suggestion:' +labels: 'Type: Suggestion' +assignees: '' + +--- + + + +**Overview of the Suggestion** + + +**What kind of user is the suggestion intended for?** +(Example users roles: API User, Curator, Depositor, Guest, Superuser, Sysadmin) + + +**What inspired this idea?** + + +**What existing behavior do you want changed?** + + +**Any brand new behavior do you want to add to Dataverse?** + + +**Any open or closed issues related to this suggestion?** + + +**Are you thinking about creating a pull request for this issue?** +Help is always welcome, is this idea something you or your organization plan to implement? diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b57aa23fc0f..f2a779bbf21 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,7 +2,7 @@ **Which issue(s) this PR closes**: -Closes # +- Closes # **Special notes for your reviewer**: diff --git a/.github/actions/setup-maven/action.yml b/.github/actions/setup-maven/action.yml new file mode 100644 index 00000000000..4cf09f34231 --- /dev/null +++ b/.github/actions/setup-maven/action.yml @@ -0,0 +1,37 @@ +--- +name: "Setup Maven and Caches" +description: "Determine Java version and setup Maven, including necessary caches." +inputs: + git-reference: + description: 'The git reference (branch/tag) to check out' + required: false + default: '${{ github.ref }}' + pom-paths: + description: "List of paths to Maven POM(s) for cache dependency setup" + required: false + default: 'pom.xml' +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.git-reference }} + - name: Determine Java version by reading the Maven property + shell: bash + run: | + echo "JAVA_VERSION=$(grep '' ${GITHUB_WORKSPACE}/modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" | tee -a ${GITHUB_ENV} + - name: Set up JDK ${{ env.JAVA_VERSION }} + id: setup-java + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: 'temurin' + cache: 'maven' + cache-dependency-path: ${{ inputs.pom-paths }} + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache + path: ~/.m2/repository diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..6325029dac1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# Set update schedule for GitHub Actions +# https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/keeping-your-actions-up-to-date-with-dependabot + +version: 2 +updates: + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions daily + interval: "daily" diff --git a/.github/workflows/check_property_files.yml b/.github/workflows/check_property_files.yml new file mode 100644 index 00000000000..505310aab35 --- /dev/null +++ b/.github/workflows/check_property_files.yml @@ -0,0 +1,32 @@ +name: "Properties Check" +on: + pull_request: + paths: + - "src/**/*.properties" + - "scripts/api/data/metadatablocks/*" +jobs: + duplicate_keys: + name: Duplicate Keys + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run duplicates detection script + shell: bash + run: tests/check_duplicate_properties.sh + + metadata_blocks_properties: + name: Metadata Blocks Properties + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup GraalVM + Native Image + uses: graalvm/setup-graalvm@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + java-version: '21' + distribution: 'graalvm-community' + - name: Setup JBang + uses: jbangdev/setup-jbang@main + - name: Run metadata block properties verification script + shell: bash + run: tests/verify_mdb_properties.sh diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index c86d284e74b..c3f9e7bdc0d 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -20,14 +20,14 @@ jobs: if: ${{ github.repository_owner == 'IQSS' }} steps: # Checkout the pull request code as when merged - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: "17" distribution: 'adopt' - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} @@ -87,7 +87,7 @@ jobs: :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name. # Leave a note when things have gone sideways - - uses: peter-evans/create-or-update-comment@v3 + - uses: peter-evans/create-or-update-comment@v4 if: ${{ failure() }} with: issue-number: ${{ github.event.client_payload.pull_request.number }} diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index b3e247e376c..afb4f6f874b 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -5,6 +5,12 @@ on: # We are deliberately *not* running on push events here to avoid double runs. # Instead, push events will trigger from the base image and maven unit tests via workflow_call. workflow_call: + inputs: + base-image-ref: + type: string + description: "Reference of the base image to build on in full qualified form [/]/:" + required: false + default: "gdcc/base:unstable" pull_request: branches: - develop @@ -16,7 +22,6 @@ on: env: IMAGE_TAG: unstable - BASE_IMAGE_TAG: unstable REGISTRY: "" # Empty means default to Docker Hub PLATFORMS: "linux/amd64,linux/arm64" MASTER_BRANCH_TAG: alpha @@ -33,20 +38,24 @@ jobs: if: ${{ github.repository_owner == 'IQSS' }} steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up JDK - uses: actions/setup-java@v3 + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: - java-version: "17" - distribution: temurin - cache: maven + pom-paths: | + pom.xml + modules/container-configbaker/pom.xml + modules/dataverse-parent/pom.xml + + # TODO: Add a filter step here, that avoids building the image if this is a PR and there are other files touched than declared above. + # Use https://github.com/dorny/paths-filter to solve this. This will ensure we do not run this twice if this workflow + # will be triggered by the other workflows already (base image or java changes) + # To become a part of #10618. - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) install # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false") @@ -59,15 +68,15 @@ jobs: if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: peter-evans/dockerhub-description@v3 + - uses: actions/checkout@v4 + - uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} repository: gdcc/dataverse short-description: "Dataverse Application Container Image providing the executable" readme-filepath: ./src/main/docker/README.md - - uses: peter-evans/dockerhub-description@v3 + - uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -106,11 +115,13 @@ jobs: if: needs.check-secrets.outputs.available == 'true' && ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: - java-version: "17" - distribution: temurin + pom-paths: | + pom.xml + modules/container-configbaker/pom.xml + modules/dataverse-parent/pom.xml # Depending on context, we push to different targets. Login accordingly. - if: github.event_name != 'pull_request' @@ -146,11 +157,13 @@ jobs: run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) install - name: Deploy multi-arch application and configbaker container image run: > mvn - -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} + -Dapp.image.tag=${{ env.IMAGE_TAG }} + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }} -P ct deploy diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index b938851f816..c2340576c78 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -1,99 +1,130 @@ --- -name: Base Container Image +name: Container Images Releasing on: push: + tags: + - 'v[6-9].**' branches: - 'develop' - - 'master' + # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' + - '!modules/container-base/src/backports/**' + - '!modules/container-base/README.md' - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' - pull_request: - branches: - - 'develop' - - 'master' - paths: - - 'modules/container-base/**' - - 'modules/dataverse-parent/pom.xml' - - '.github/workflows/container_base_push.yml' - schedule: - - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC + + # These TODOs are left for #10618 + # TODO: we are missing a workflow_call option here, so we can trigger this flow from pr comments and maven tests (keep the secrets availability in mind!) + # TODO: we are missing a pull_request option here (filter for stuff that would trigger the maven runs!) so we can trigger preview builds for them when coming from the main repo (keep the secrets availability in mind!) env: - IMAGE_TAG: unstable PLATFORMS: linux/amd64,linux/arm64 + DEVELOPMENT_BRANCH: develop jobs: build: - name: Build image + name: Base Image runs-on: ubuntu-latest permissions: contents: read packages: read - strategy: - matrix: - jdk: [ '17' ] # Only run in upstream repo - avoid unnecessary runs in forks if: ${{ github.repository_owner == 'IQSS' }} + outputs: + base-image-ref: ${{ steps.finalize.outputs.base-image-ref }} steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: - java-version: ${{ matrix.jdk }} - distribution: 'adopt' - - name: Cache Maven packages - uses: actions/cache@v3 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - - name: Build base container image with local architecture - run: mvn -f modules/container-base -Pct package + pom-paths: modules/container-base/pom.xml - # Run anything below only if this is not a pull request. - # Accessing, pushing tags etc. to DockerHub will only succeed in upstream because secrets. - - - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} - name: Push description to DockerHub - uses: peter-evans/dockerhub-description@v3 + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + - name: Log in to the Container registry + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: gdcc/base - short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" - readme-filepath: ./modules/container-base/README.md - - if: ${{ github.event_name != 'pull_request' }} - name: Log in to the Container registry - uses: docker/login-action@v2 + # In case this is a push to develop, we care about buildtime. + # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. + - name: Setup SSH agent + if: ${{ github.event_name != 'schedule' }} + uses: webfactory/ssh-agent@v0.9.0 with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - if: ${{ github.event_name != 'pull_request' }} - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v2 - - name: Re-set image tag based on branch - if: ${{ github.ref_name == 'master' }} - run: echo "IMAGE_TAG=alpha" >> $GITHUB_ENV - - if: ${{ github.event_name != 'pull_request' }} - name: Deploy multi-arch base container image to Docker Hub - run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }} -Ddocker.platforms=${{ env.PLATFORMS }} + ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} + - name: Provide the known hosts key and the builder config + if: ${{ github.event_name != 'schedule' }} + run: | + echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts + mkdir -p modules/container-base/target/buildx-state/buildx/instances + cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF + { "Name": "maven", + "Driver": "docker-container", + "Dynamic": false, + "Nodes": [{"Name": "maven0", + "Endpoint": "unix:///var/run/docker.sock", + "Platforms": [{"os": "linux", "architecture": "amd64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}, + {"Name": "maven1", + "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", + "Platforms": [{"os": "linux", "architecture": "arm64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}]} + EOF + + # Determine the base image name we are going to use from here on + - name: Determine base image name + run: | + if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + else + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + fi + - name: Calculate revision number for immutable tag (on release branches only) + if: ${{ github.ref_name != env.DEVELOPMENT_BRANCH }} + id: revision-tag + uses: ./.github/actions/get-image-revision + with: + image-ref: ${{ env.BASE_IMAGE }} + tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision=" + - name: Configure update of "latest" tag for development branch + id: develop-tag + if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} + run: | + echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" + + - name: Deploy multi-arch base container image to Docker Hub + id: build + run: | + mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ + -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} + + - name: Determine appropriate base image ref for app image + id: finalize + run: | + if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then + echo "base-image-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT" + else + echo "base-image-ref=gdcc/base:${{ steps.revision-tag.outputs.revision-tag }}" | tee -a "$GITHUB_OUTPUT" + fi + push-app-img: name: "Rebase & Publish App Image" permissions: contents: read packages: write pull-requests: write - needs: build - # We do not release a new base image for pull requests, so do not trigger. - if: ${{ github.event_name != 'pull_request' }} - uses: ./.github/workflows/container_app_push.yml secrets: inherit + needs: + - build + uses: ./.github/workflows/container_app_push.yml + with: + base-image-ref: ${{ needs.build.outputs.base-image-ref }} diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml new file mode 100644 index 00000000000..986fe25cdf5 --- /dev/null +++ b/.github/workflows/container_maintenance.yml @@ -0,0 +1,119 @@ +--- +name: Container Images Scheduled Maintenance + +on: + # TODO: think about adding a (filtered) push event trigger here in case we change the patches + # --- + # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) + workflow_dispatch: + inputs: + force_build: + type: boolean + required: false + default: false + description: "Build and deploy even if no newer Java images or package updates are found." + schedule: + - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC + +env: + PLATFORMS: linux/amd64,linux/arm64 + NUM_PAST_RELEASES: 3 + +jobs: + build: + name: Base Image Matrix Build + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + # Only run in upstream repo - avoid unnecessary runs in forks + if: ${{ github.repository_owner == 'IQSS' }} + outputs: + supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} + rebuilt_base_images: ${{ steps.execute.outputs.rebuilt_base_images }} + + steps: + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop + with: + pom-paths: modules/container-base/pom.xml + + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU for multi-arch builds + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} + + # Discover the releases we want to maintain + - name: Discover maintained releases + id: discover + run: | + echo "FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 )" | tee -a "$GITHUB_ENV" + DEVELOPMENT_BRANCH=$( curl -f -sS https://api.github.com/repos/${{ github.repository }} | jq -r '.default_branch' ) + echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" | tee -a "$GITHUB_ENV" + echo "branches=$( curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq -r " .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, \"${DEVELOPMENT_BRANCH}\" " | tr "\n" " " )" | tee -a "${GITHUB_OUTPUT}" + + # Execute matrix build for the discovered branches + - name: Execute build matrix script + id: execute + run: | + .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} + + # TODO: Use the needs.build.outputs.rebuilt_base_images with fromJSON() to create a matrix job. + # Must be a single rank matrix (vector), the branch and base image tag information ships as "branch=tag" string + # Will be part of working on #10618, app image versioned tags. + #push-app-img: + # name: "Rebase & Publish App Image" + # permissions: + # contents: read + # packages: write + # pull-requests: write + # secrets: inherit + # needs: + # - build + # strategy: + # fail-fast: false + # matrix: + # branch: ${{ fromJson(needs.discover.outputs.branches) }} + # uses: ./.github/workflows/container_app_push.yml + # with: + # branch: ${{ matrix.branch }} + + hub-description: + name: Push description to DockerHub + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Render README + id: render + run: | + TAGS_JSON='${{ needs.build.outputs.supported_tag_matrix }}' + echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | + while IFS= read -r branch; do + echo \ + "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ + "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/modules/container-base/src/main/docker/Dockerfile)," \ + "[Patches](https://github.com/IQSS/dataverse/blob/develop/modules/container-base/src/backports/${branch}))" \ + | tee -a "${GITHUB_WORKSPACE}/tags.md" + done + sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags.md" "./modules/container-base/README.md" + + - name: Push description to DockerHub + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: gdcc/base + short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" + readme-filepath: ./modules/container-base/README.md \ No newline at end of file diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index 028f0140cc9..eca8416732a 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -5,14 +5,18 @@ on: branches: - develop +concurrency: + group: deploy-beta-testing + cancel-in-progress: false + jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: '17' @@ -32,7 +36,7 @@ jobs: run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV - name: Upload war artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: built-app path: ./target/${{ env.war_file }} @@ -42,10 +46,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download war artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: built-app path: ./ diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml index 992f30f2872..fa3a876c418 100644 --- a/.github/workflows/guides_build_sphinx.yml +++ b/.github/workflows/guides_build_sphinx.yml @@ -10,7 +10,7 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: OdumInstitute/sphinx-action@master + - uses: actions/checkout@v4 + - uses: uncch-rdmc/sphinx-action@master with: docs-folder: "doc/sphinx-guides/" diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 4ad4798bc64..45180ea7aec 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -30,10 +30,11 @@ jobs: continue-on-error: ${{ matrix.experimental }} runs-on: ubuntu-latest steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.jdk }} distribution: temurin @@ -56,7 +57,7 @@ jobs: # Upload the built war file. For download, it will be wrapped in a ZIP by GitHub. # See also https://github.com/actions/upload-artifact#zipped-artifact-downloads - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: dataverse-java${{ matrix.jdk }}.war path: target/dataverse*.war @@ -66,7 +67,7 @@ jobs: - run: | tar -cvf java-builddir.tar target tar -cvf java-m2-selection.tar ~/.m2/repository/io/gdcc/dataverse-* - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: java-artifacts path: | @@ -95,17 +96,18 @@ jobs: # status: "Experimental" continue-on-error: ${{ matrix.experimental }} steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.jdk }} distribution: temurin cache: maven # Get the build output from the unit test job - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: java-artifacts - run: | @@ -117,7 +119,7 @@ jobs: # Wrap up and send to coverage job - run: tar -cvf java-reportdir.tar target/site - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: java-reportdir path: java-reportdir.tar @@ -128,16 +130,17 @@ jobs: needs: integration-test name: Coverage Report Submission steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: java-version: '17' distribution: temurin cache: maven # Get the build output from the integration test job - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: java-reportdir - run: tar -xvf java-reportdir.tar @@ -156,6 +159,11 @@ jobs: # NOTE: this may be extended with adding a report to the build output, leave a comment, send to Sonarcloud, ... + # TODO: Add a filter step here, that avoids calling the app image release workflow if there are changes to the base image. + # Use https://github.com/dorny/paths-filter to solve this. Will require and additional job or adding to integration-test job. + # This way we ensure that we're not running the app image flow with a non-matching base image. + # To become a part of #10618. + push-app-img: name: Publish App Image permissions: diff --git a/.github/workflows/pr_comment_commands.yml b/.github/workflows/pr_comment_commands.yml index 5ff75def623..06b11b1ac5b 100644 --- a/.github/workflows/pr_comment_commands.yml +++ b/.github/workflows/pr_comment_commands.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Dispatch - uses: peter-evans/slash-command-dispatch@v3 + uses: peter-evans/slash-command-dispatch@v4 with: # This token belongs to @dataversebot and has sufficient scope. token: ${{ secrets.GHCR_TOKEN }} diff --git a/.github/workflows/reviewdog_checkstyle.yml b/.github/workflows/reviewdog_checkstyle.yml index 90a0dd7d06b..804b04f696a 100644 --- a/.github/workflows/reviewdog_checkstyle.yml +++ b/.github/workflows/reviewdog_checkstyle.yml @@ -10,7 +10,7 @@ jobs: name: Checkstyle job steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Run check style uses: nikitasavinov/checkstyle-action@master with: diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh new file mode 100755 index 00000000000..370988b9812 --- /dev/null +++ b/.github/workflows/scripts/maintenance-job.sh @@ -0,0 +1,180 @@ +#!/bin/bash + +# A matrix-like job to maintain a number of releases as well as the latest snap of Dataverse. + +# PREREQUISITES: +# - You have Java, Maven, QEMU and Docker all setup and ready to go +# - You obviously checked out the develop branch, otherwise you'd not be executing this script +# - You added all the branch names you want to run maintenance for as arguments +# Optional, but recommended: +# - You added a DEVELOPMENT_BRANCH env var to your runner/job env with the name of the development branch +# - You added a FORCE_BUILD=0|1 env var to indicate if the base image build should be forced +# - You added a PLATFORMS env var with all the target platforms you want to build for + +# NOTE: +# This script is a culmination of Github Action steps into a single script. +# The reason to put all of this in here is due to the complexity of the Github Action and the limitation of the +# matrix support in Github actions, where outputs cannot be aggregated or otherwise used further. + +set -euo pipefail + +# Get all the inputs +# If not within a runner, just print to stdout (duplicating the output in case of tee usage, but that's ok for testing) +GITHUB_OUTPUT=${GITHUB_OUTPUT:-"/proc/self/fd/1"} +GITHUB_ENV=${GITHUB_ENV:-"/proc/self/fd/1"} +GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-"$(pwd)"} +GITHUB_SERVER_URL=${GITHUB_SERVER_URL:-"https://github.com"} +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-"IQSS/dataverse"} + +MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" + +DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" +FORCE_BUILD="${FORCE_BUILD:-"0"}" +PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" + +# Setup and validation +if [[ -z "$*" ]]; then + >&2 echo "You must give a list of branch names as arguments" + exit 1; +fi + +source "$( dirname "$0" )/utils.sh" + +# Delete old stuff if present +rm -rf "$MAINTENANCE_WORKSPACE" +mkdir -p "$MAINTENANCE_WORKSPACE" + +# Store the image tags we maintain in this array (same order as branches array!) +# This list will be used to build the support matrix within the Docker Hub image description +SUPPORTED_ROLLING_TAGS=() +# Store the tags of base images we are actually rebuilding to base new app images upon +# Takes the from "branch-name=base-image-ref" +REBUILT_BASE_IMAGES=() + +for BRANCH in "$@"; do + echo "::group::Running maintenance for $BRANCH" + + # 0. Determine if this is a development branch and the most current release + IS_DEV=0 + if [[ "$BRANCH" = "$DEVELOPMENT_BRANCH" ]]; then + IS_DEV=1 + fi + IS_CURRENT_RELEASE=0 + if [[ "$BRANCH" = $( curl -f -sS "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" | jq -r '.[0].tag_name' ) ]]; then + IS_CURRENT_RELEASE=1 + fi + + # 1. Let's get the maintained sources + git clone -c advice.detachedHead=false --depth 1 --branch "$BRANCH" "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" "$MAINTENANCE_WORKSPACE/$BRANCH" + # Switch context + cd "$MAINTENANCE_WORKSPACE/$BRANCH" + + # 2. Now let's apply the patches (we have them checked out in $GITHUB_WORKSPACE, not necessarily in this local checkout) + echo "Checking for patches..." + if [[ -d ${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH ]]; then + echo "Applying patches now." + find "${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + fi + + # 3. Determine the base image ref (/:) + BASE_IMAGE_REF="" + # For the dev branch we want to full flexi stack tag, to detect stack upgrades requiring new build + if (( IS_DEV )); then + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) + else + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) + fi + echo "Determined BASE_IMAGE_REF=$BASE_IMAGE_REF from Maven" + + # 4. Check for Temurin image updates + JAVA_IMAGE_REF=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout ) + echo "Determined JAVA_IMAGE_REF=$JAVA_IMAGE_REF from Maven" + NEWER_JAVA_IMAGE=0 + if check_newer_parent "$JAVA_IMAGE_REF" "$BASE_IMAGE_REF"; then + NEWER_JAVA_IMAGE=1 + fi + + # 5. Check for package updates in base image + PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" + echo "Determined installed packages=\"$PKGS\" from Maven" + NEWER_PKGS=0 + # Don't bother with package checks if the java image is newer already + if ! (( NEWER_JAVA_IMAGE )); then + if check_newer_pkgs "$BASE_IMAGE_REF" "$PKGS"; then + NEWER_PKGS=1 + fi + fi + + # 6. Get current immutable revision tag if not on the dev branch + REV=$( current_revision "$BASE_IMAGE_REF" ) + CURRENT_REV_TAG="${BASE_IMAGE_REF#*:}-r$REV" + NEXT_REV_TAG="${BASE_IMAGE_REF#*:}-r$(( REV + 1 ))" + + # 7. Let's put together what tags we want added to this build run + TAG_OPTIONS="" + if ! (( IS_DEV )); then + TAG_OPTIONS="-Dbase.image=$BASE_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" + # In case of the current release, add the "latest" tag as well. + if (( IS_CURRENT_RELEASE )); then + TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" + fi + else + UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image.tag -Dbase.image.tag.suffix="" -q -DforceStdout ) + TAG_OPTIONS="-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=$UPCOMING_TAG" + + # For the dev branch we only have rolling tags and can add them now already + SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\", \"${BASE_IMAGE_REF#*:}\"]") + fi + echo "Determined these additional Maven tag options: $TAG_OPTIONS" + + # 8. Let's build the base image if necessary + NEWER_IMAGE=0 + if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then + mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + NEWER_IMAGE=1 + # Save the information about the immutable or rolling tag we just built + if ! (( IS_DEV )); then + REBUILT_BASE_IMAGES+=("$BRANCH=${BASE_IMAGE_REF%:*}:$NEXT_REV_TAG") + else + REBUILT_BASE_IMAGES+=("$BRANCH=$BASE_IMAGE_REF") + fi + else + echo "No rebuild necessary, we're done here." + fi + + # 9. Add list of rolling and immutable tags for release builds + if ! (( IS_DEV )); then + RELEASE_TAGS_LIST="[" + if (( IS_CURRENT_RELEASE )); then + RELEASE_TAGS_LIST+="\"latest\", " + fi + RELEASE_TAGS_LIST+="\"${BASE_IMAGE_REF#*:}\", " + if (( NEWER_IMAGE )); then + RELEASE_TAGS_LIST+="\"$NEXT_REV_TAG\"]" + else + RELEASE_TAGS_LIST+="\"$CURRENT_REV_TAG\"]" + fi + SUPPORTED_ROLLING_TAGS+=("${RELEASE_TAGS_LIST}") + fi + + echo "::endgroup::" +done + +# Built the output which base images have actually been rebuilt as JSON +REBUILT_IMAGES="[" +for IMAGE in "${REBUILT_BASE_IMAGES[@]}"; do + REBUILT_IMAGES+=" \"$IMAGE\" " +done +REBUILT_IMAGES+="]" +echo "rebuilt_base_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" + +# Built the supported rolling tags matrix as JSON +SUPPORTED_TAGS="{" +for (( i=0; i < ${#SUPPORTED_ROLLING_TAGS[@]} ; i++ )); do + j=$((i+1)) + SUPPORTED_TAGS+="\"${!j}\": ${SUPPORTED_ROLLING_TAGS[$i]}" + (( i < ${#SUPPORTED_ROLLING_TAGS[@]}-1 )) && SUPPORTED_TAGS+=", " +done +SUPPORTED_TAGS+="}" +echo "supported_tag_matrix=$SUPPORTED_TAGS" | tee -a "$GITHUB_OUTPUT" diff --git a/.github/workflows/scripts/utils.sh b/.github/workflows/scripts/utils.sh new file mode 100644 index 00000000000..987b58d8bb5 --- /dev/null +++ b/.github/workflows/scripts/utils.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +set -euo pipefail + +function check_newer_parent() { + PARENT_IMAGE="$1" + # Get namespace, default to "library" if not found + PARENT_IMAGE_NS="${PARENT_IMAGE%/*}" + if [[ "$PARENT_IMAGE_NS" = "${PARENT_IMAGE}" ]]; then + PARENT_IMAGE_NS="library" + fi + PARENT_IMAGE_REPO="${PARENT_IMAGE%:*}" + PARENT_IMAGE_TAG="${PARENT_IMAGE#*:}" + + PARENT_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${PARENT_IMAGE_NS}/repositories/${PARENT_IMAGE_REPO}/tags/${PARENT_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$PARENT_IMAGE_LAST_UPDATE" = "null" ]]; then + echo "::error title='Invalid PARENT Image'::Could not find ${PARENT_IMAGE} in the registry" + exit 1 + fi + + DERIVED_IMAGE="$2" + # Get namespace, default to "library" if not found + DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" + if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then + DERIVED_IMAGE_NS="library" + fi + DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" + DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" + + DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$PARENT_IMAGE_LAST_UPDATE" ]]; then + echo "Parent image $PARENT_IMAGE has a newer release ($PARENT_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + return 0 + else + echo "Parent image $PARENT_IMAGE ($PARENT_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + return 1 + fi +} + +function check_newer_pkgs() { + IMAGE="$1" + PKGS="$2" + + docker run --rm -u 0 "${IMAGE}" sh -c "apt update >/dev/null 2>&1 && apt install -s ${PKGS}" | tee /proc/self/fd/2 | grep -q "0 upgraded" + STATUS=$? + + if [[ $STATUS -eq 0 ]]; then + echo "Base image $IMAGE has no updates for our custom installed packages" + return 1 + else + echo "Base image $IMAGE needs updates for our custom installed packages" + return 0 + fi + + # TODO: In a future version of this script, we might want to include checking for other security updates, + # not just updates to the packages we installed. + # grep security /etc/apt/sources.list > /tmp/security.list + # apt-get update -oDir::Etc::Sourcelist=/tmp/security.list + # apt-get dist-upgrade -y -oDir::Etc::Sourcelist=/tmp/security.list -oDir::Etc::SourceParts=/bin/false -s + +} + +function current_revision() { + IMAGE="$1" + IMAGE_NS_REPO="${IMAGE%:*}" + IMAGE_TAG="${IMAGE#*:}" + + if [[ "$IMAGE_TAG" = "$IMAGE_NS_REPO" ]]; then + >&2 echo "You must provide an image reference in the format [/]:" + exit 1 + fi + + case "$IMAGE_NS_REPO" in + */*) :;; # namespace/repository syntax, leave as is + *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax + esac + + # Without such a token we may run into rate limits + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. + # token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) + + ALL_TAGS="$( + i=0 + while [ $? == 0 ]; do + i=$((i+1)) + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. + # RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + RESULT=$( curl -s "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then + # If we run into an error on the first attempt, that means we have a problem. + if [[ "$i" == "1" ]]; then + >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" + exit 2 + # Otherwise it will just mean we reached the last page already + else + break + fi + else + echo "$RESULT" | jq -r '."results"[]["name"]' + # DEBUG: + #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' + fi + done + )" + + # Note: if a former tag could not be found, it just might not exist already. Start new series with rev 0 + echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 || echo "-1" +} diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 56f7d648dc4..fb9cf5a0a1f 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -21,7 +21,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: shellcheck uses: reviewdog/action-shellcheck@v1 with: diff --git a/.github/workflows/shellspec.yml b/.github/workflows/shellspec.yml index 227a74fa00f..cc09992edac 100644 --- a/.github/workflows/shellspec.yml +++ b/.github/workflows/shellspec.yml @@ -19,35 +19,18 @@ jobs: steps: - name: Install shellspec run: curl -fsSL https://git.io/shellspec | sh -s ${{ env.SHELLSPEC_VERSION }} --yes - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Run Shellspec run: | cd tests/shell shellspec - shellspec-centos7: - name: "CentOS 7" + shellspec-rocky9: + name: "RockyLinux 9" runs-on: ubuntu-latest container: - image: centos:7 + image: rockylinux/rockylinux:9 steps: - - uses: actions/checkout@v2 - - name: Install shellspec - run: | - curl -fsSL https://github.com/shellspec/shellspec/releases/download/${{ env.SHELLSPEC_VERSION }}/shellspec-dist.tar.gz | tar -xz -C /usr/share - ln -s /usr/share/shellspec/shellspec /usr/bin/shellspec - - name: Install dependencies - run: yum install -y ed - - name: Run shellspec - run: | - cd tests/shell - shellspec - shellspec-rocky8: - name: "RockyLinux 8" - runs-on: ubuntu-latest - container: - image: rockylinux/rockylinux:8 - steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install shellspec run: | curl -fsSL https://github.com/shellspec/shellspec/releases/download/${{ env.SHELLSPEC_VERSION }}/shellspec-dist.tar.gz | tar -xz -C /usr/share @@ -64,7 +47,7 @@ jobs: steps: - name: Install shellspec run: curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Run Shellspec run: | cd tests/shell diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml index 8ad74b3e4bb..6398edca412 100644 --- a/.github/workflows/spi_release.yml +++ b/.github/workflows/spi_release.yml @@ -37,15 +37,15 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'pull_request' && needs.check-secrets.outputs.available == 'true' steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: java-version: '17' distribution: 'adopt' server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} @@ -63,12 +63,12 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && needs.check-secrets.outputs.available == 'true' steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: java-version: '17' distribution: 'adopt' - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} @@ -76,7 +76,7 @@ jobs: # Running setup-java again overwrites the settings.xml - IT'S MANDATORY TO DO THIS SECOND SETUP!!! - name: Set up Maven Central Repository - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '17' distribution: 'adopt' diff --git a/conf/keycloak/test-realm.json b/conf/keycloak/test-realm.json index efe71cc5d29..2e5ed1c4d69 100644 --- a/conf/keycloak/test-realm.json +++ b/conf/keycloak/test-realm.json @@ -45,287 +45,411 @@ "quickLoginCheckMilliSeconds" : 1000, "maxDeltaTimeSeconds" : 43200, "failureFactor" : 30, - "roles" : { - "realm" : [ { - "id" : "075daee1-5ab2-44b5-adbf-fa49a3da8305", - "name" : "uma_authorization", - "description" : "${role_uma_authorization}", - "composite" : false, - "clientRole" : false, - "containerId" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", - "attributes" : { } - }, { - "id" : "b4ff9091-ddf9-4536-b175-8cfa3e331d71", - "name" : "default-roles-test", - "description" : "${role_default-roles}", - "composite" : true, - "composites" : { - "realm" : [ "offline_access", "uma_authorization" ], - "client" : { - "account" : [ "view-profile", "manage-account" ] - } + "roles": { + "realm": [ + { + "id": "075daee1-5ab2-44b5-adbf-fa49a3da8305", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} }, - "clientRole" : false, - "containerId" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", - "attributes" : { } - }, { - "id" : "e6d31555-6be6-4dee-bc6a-40a53108e4c2", - "name" : "offline_access", - "description" : "${role_offline-access}", - "composite" : false, - "clientRole" : false, - "containerId" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", - "attributes" : { } - } ], - "client" : { - "realm-management" : [ { - "id" : "1955bd12-5f86-4a74-b130-d68a8ef6f0ee", - "name" : "impersonation", - "description" : "${role_impersonation}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "1109c350-9ab1-426c-9876-ef67d4310f35", - "name" : "view-authorization", - "description" : "${role_view-authorization}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "980c3fd3-1ae3-4b8f-9a00-d764c939035f", - "name" : "query-users", - "description" : "${role_query-users}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "5363e601-0f9d-4633-a8c8-28cb0f859b7b", - "name" : "query-groups", - "description" : "${role_query-groups}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "59aa7992-ad78-48db-868a-25d6e1d7db50", - "name" : "realm-admin", - "description" : "${role_realm-admin}", - "composite" : true, - "composites" : { - "client" : { - "realm-management" : [ "impersonation", "view-authorization", "query-users", "query-groups", "manage-clients", "manage-realm", "view-identity-providers", "query-realms", "manage-authorization", "manage-identity-providers", "manage-users", "view-users", "view-realm", "create-client", "view-clients", "manage-events", "query-clients", "view-events" ] + { + "id": "b4ff9091-ddf9-4536-b175-8cfa3e331d71", + "name": "default-roles-test", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ], + "client": { + "account": [ + "view-profile", + "manage-account" + ] } }, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "112f53c2-897d-4c01-81db-b8dc10c5b995", - "name" : "manage-clients", - "description" : "${role_manage-clients}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "c7f57bbd-ef32-4a64-9888-7b8abd90777a", - "name" : "manage-realm", - "description" : "${role_manage-realm}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "8885dac8-0af3-45af-94ce-eff5e801bb80", - "name" : "view-identity-providers", - "description" : "${role_view-identity-providers}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "2673346c-b0ef-4e01-8a90-be03866093af", - "name" : "manage-authorization", - "description" : "${role_manage-authorization}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "b7182885-9e57-445f-8dae-17c16eb31b5d", - "name" : "manage-identity-providers", - "description" : "${role_manage-identity-providers}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "ba7bfe0c-cb07-4a47-b92c-b8132b57e181", - "name" : "manage-users", - "description" : "${role_manage-users}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "13a8f0fc-647d-4bfe-b525-73956898e550", - "name" : "query-realms", - "description" : "${role_query-realms}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "ef4c57dc-78c2-4f9a-8d2b-0e97d46fc842", - "name" : "view-realm", - "description" : "${role_view-realm}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "2875da34-006c-4b7f-bfc8-9ae8e46af3a2", - "name" : "view-users", - "description" : "${role_view-users}", - "composite" : true, - "composites" : { - "client" : { - "realm-management" : [ "query-users", "query-groups" ] + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} + }, + { + "id": "131ff85b-0c25-491b-8e13-dde779ec0854", + "name": "admin", + "description": "", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "impersonation", + "view-authorization", + "query-users", + "manage-realm", + "view-identity-providers", + "manage-authorization", + "view-clients", + "manage-events", + "query-clients", + "view-events", + "query-groups", + "realm-admin", + "manage-clients", + "query-realms", + "manage-identity-providers", + "manage-users", + "view-users", + "view-realm", + "create-client" + ], + "broker": [ + "read-token" + ], + "account": [ + "delete-account", + "manage-consent", + "view-consent", + "view-applications", + "view-groups", + "manage-account-links", + "view-profile", + "manage-account" + ] } }, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "c8c8f7dc-876b-4263-806f-3329f7cd5fd3", - "name" : "create-client", - "description" : "${role_create-client}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "21b84f90-5a9a-4845-a7ba-bbd98ac0fcc4", - "name" : "view-clients", - "description" : "${role_view-clients}", - "composite" : true, - "composites" : { - "client" : { - "realm-management" : [ "query-clients" ] - } + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} + }, + { + "id": "e6d31555-6be6-4dee-bc6a-40a53108e4c2", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "1955bd12-5f86-4a74-b130-d68a8ef6f0ee", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} }, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "6fd64c94-d663-4501-ad77-0dcf8887d434", - "name" : "manage-events", - "description" : "${role_manage-events}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "b321927a-023c-4d2a-99ad-24baf7ff6d83", - "name" : "query-clients", - "description" : "${role_query-clients}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "2fc21160-78de-457b-8594-e5c76cde1d5e", - "name" : "view-events", - "description" : "${role_view-events}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - } ], - "test" : [ ], - "security-admin-console" : [ ], - "admin-cli" : [ ], - "account-console" : [ ], - "broker" : [ { - "id" : "07ee59b5-dca6-48fb-83d4-2994ef02850e", - "name" : "read-token", - "description" : "${role_read-token}", - "composite" : false, - "clientRole" : true, - "containerId" : "b57d62bb-77ff-42bd-b8ff-381c7288f327", - "attributes" : { } - } ], - "account" : [ { - "id" : "17d2f811-7bdf-4c73-83b4-1037001797b8", - "name" : "view-applications", - "description" : "${role_view-applications}", - "composite" : false, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "d1ff44f9-419e-42fd-98e8-1add1169a972", - "name" : "delete-account", - "description" : "${role_delete-account}", - "composite" : false, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "14c23a18-ae2d-43c9-b0c0-aaf6e0c7f5b0", - "name" : "manage-account-links", - "description" : "${role_manage-account-links}", - "composite" : false, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "6fbe58af-d2fe-4d66-95fe-a2e8a818cb55", - "name" : "view-profile", - "description" : "${role_view-profile}", - "composite" : false, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "bdfd02bc-6f6a-47d2-82bc-0ca52d78ff48", - "name" : "manage-consent", - "description" : "${role_manage-consent}", - "composite" : true, - "composites" : { - "client" : { - "account" : [ "view-consent" ] - } + { + "id": "1109c350-9ab1-426c-9876-ef67d4310f35", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} }, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "782f3b0c-a17b-4a87-988b-1a711401f3b0", - "name" : "manage-account", - "description" : "${role_manage-account}", - "composite" : true, - "composites" : { - "client" : { - "account" : [ "manage-account-links" ] - } + { + "id": "980c3fd3-1ae3-4b8f-9a00-d764c939035f", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "5363e601-0f9d-4633-a8c8-28cb0f859b7b", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "59aa7992-ad78-48db-868a-25d6e1d7db50", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "impersonation", + "view-authorization", + "query-users", + "query-groups", + "manage-clients", + "manage-realm", + "view-identity-providers", + "query-realms", + "manage-authorization", + "manage-identity-providers", + "manage-users", + "view-users", + "view-realm", + "create-client", + "view-clients", + "manage-events", + "query-clients", + "view-events" + ] + } + }, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "112f53c2-897d-4c01-81db-b8dc10c5b995", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "c7f57bbd-ef32-4a64-9888-7b8abd90777a", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "8885dac8-0af3-45af-94ce-eff5e801bb80", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "2673346c-b0ef-4e01-8a90-be03866093af", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "b7182885-9e57-445f-8dae-17c16eb31b5d", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "ba7bfe0c-cb07-4a47-b92c-b8132b57e181", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} }, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "8a3bfe15-66d9-4f3d-83ac-801d682d42b0", - "name" : "view-consent", - "description" : "${role_view-consent}", - "composite" : false, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - } ] + { + "id": "13a8f0fc-647d-4bfe-b525-73956898e550", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "ef4c57dc-78c2-4f9a-8d2b-0e97d46fc842", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "2875da34-006c-4b7f-bfc8-9ae8e46af3a2", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "c8c8f7dc-876b-4263-806f-3329f7cd5fd3", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "21b84f90-5a9a-4845-a7ba-bbd98ac0fcc4", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "6fd64c94-d663-4501-ad77-0dcf8887d434", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "b321927a-023c-4d2a-99ad-24baf7ff6d83", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "2fc21160-78de-457b-8594-e5c76cde1d5e", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + } + ], + "test": [], + "security-admin-console": [], + "admin-cli": [], + "account-console": [], + "broker": [ + { + "id": "07ee59b5-dca6-48fb-83d4-2994ef02850e", + "name": "read-token", + "description": "${role_read-token}", + "composite": false, + "clientRole": true, + "containerId": "b57d62bb-77ff-42bd-b8ff-381c7288f327", + "attributes": {} + } + ], + "account": [ + { + "id": "17d2f811-7bdf-4c73-83b4-1037001797b8", + "name": "view-applications", + "description": "${role_view-applications}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "f5918d56-bd4d-4035-8fa7-8622075ed690", + "name": "view-groups", + "description": "${role_view-groups}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "d1ff44f9-419e-42fd-98e8-1add1169a972", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "14c23a18-ae2d-43c9-b0c0-aaf6e0c7f5b0", + "name": "manage-account-links", + "description": "${role_manage-account-links}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "6fbe58af-d2fe-4d66-95fe-a2e8a818cb55", + "name": "view-profile", + "description": "${role_view-profile}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "bdfd02bc-6f6a-47d2-82bc-0ca52d78ff48", + "name": "manage-consent", + "description": "${role_manage-consent}", + "composite": true, + "composites": { + "client": { + "account": [ + "view-consent" + ] + } + }, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "782f3b0c-a17b-4a87-988b-1a711401f3b0", + "name": "manage-account", + "description": "${role_manage-account}", + "composite": true, + "composites": { + "client": { + "account": [ + "manage-account-links" + ] + } + }, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "8a3bfe15-66d9-4f3d-83ac-801d682d42b0", + "name": "view-consent", + "description": "${role_view-consent}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + } + ] } }, "groups" : [ { @@ -409,7 +533,7 @@ } ], "disableableCredentialTypes" : [ ], "requiredActions" : [ ], - "realmRoles" : [ "default-roles-test" ], + "realmRoles" : [ "default-roles-test", "admin" ], "notBefore" : 0, "groups" : [ "/admins" ] }, { diff --git a/conf/solr/schema.xml b/conf/solr/schema.xml index 5dde750573d..f4121de97c1 100644 --- a/conf/solr/schema.xml +++ b/conf/solr/schema.xml @@ -142,6 +142,7 @@ + @@ -166,6 +167,8 @@ + + @@ -200,11 +203,14 @@ + + + @@ -232,6 +238,7 @@ + @@ -590,6 +599,7 @@ + diff --git a/doc/release-notes/ 11065-extend-search-api-to-include-type-counts.md b/doc/release-notes/ 11065-extend-search-api-to-include-type-counts.md new file mode 100644 index 00000000000..0ba188c8637 --- /dev/null +++ b/doc/release-notes/ 11065-extend-search-api-to-include-type-counts.md @@ -0,0 +1 @@ +The JSON payload of the search endpoint has been extended to include total_count_per_object_type for types: dataverse, dataset, and files when the search parameter "&show_type_counts=true" is passed in. diff --git a/doc/release-notes/10735-search-dataset-name-and-dataset-citation-different.md b/doc/release-notes/10735-search-dataset-name-and-dataset-citation-different.md new file mode 100644 index 00000000000..6a6b2008772 --- /dev/null +++ b/doc/release-notes/10735-search-dataset-name-and-dataset-citation-different.md @@ -0,0 +1,4 @@ + +### Search files Bug fix + +dataset-citation was displaying DRAFT version instead of latest released version diff --git a/doc/release-notes/10959-oidc-api-auth-ext.md b/doc/release-notes/10959-oidc-api-auth-ext.md new file mode 100644 index 00000000000..04ee2099f68 --- /dev/null +++ b/doc/release-notes/10959-oidc-api-auth-ext.md @@ -0,0 +1,14 @@ +Extends the OIDC API auth mechanism (available through feature flag ``api-bearer-auth``) to properly handle cases +where ``BearerTokenAuthMechanism`` successfully validates the token but cannot identify any Dataverse user because there +is no account associated with the token. + +To register a new user who has authenticated via an OIDC provider, a new endpoint has been +implemented (``/users/register``). A feature flag named ``api-bearer-auth-provide-missing-claims`` has been implemented +to allow +sending missing user claims in the request JSON. This is useful when the identity provider does not supply the necessary +claims. However, this flag will only be considered if the ``api-bearer-auth`` feature flag is enabled. If the latter is +not enabled, the ``api-bearer-auth-provide-missing-claims`` flag will be ignored. + +A feature flag named ``api-bearer-auth-handle-tos-acceptance-in-idp`` has been implemented. When enabled, it specifies +that Terms of Service acceptance is managed by the identity provider, eliminating the need to explicitly include the +acceptance in the user registration request JSON. diff --git a/doc/release-notes/11027-extend-datasets-files-from-search-api.md b/doc/release-notes/11027-extend-datasets-files-from-search-api.md new file mode 100644 index 00000000000..7b20daeeb0f --- /dev/null +++ b/doc/release-notes/11027-extend-datasets-files-from-search-api.md @@ -0,0 +1,22 @@ +### Feature to extend Search API for SPA + +Added new fields to search results type=files + +For Files: +- restricted: boolean +- canDownloadFile: boolean ( from file user permission) +- categories: array of string "categories" would be similar to what it is in metadata api. +For tabular files: +- tabularTags: array of string for example,{"tabularTags" : ["Event", "Genomics", "Geospatial"]} +- variables: number/int shows how many variables we have for the tabular file +- observations: number/int shows how many observations for the tabular file + + + +New fields added to solr schema.xml (Note: upgrade instructions will need to include instructions for schema.xml): + + + + + +See https://github.com/IQSS/dataverse/issues/11027 diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md new file mode 100644 index 00000000000..979fd16bf9e --- /dev/null +++ b/doc/release-notes/6.4-release-notes.md @@ -0,0 +1,526 @@ +# Dataverse 6.4 + +Please note: To read these instructions in full, please go to https://github.com/IQSS/dataverse/releases/tag/v6.4 rather than the list of releases, which will cut them off. + +This release brings new features, enhancements, and bug fixes to Dataverse. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project. + +## Release Highlights + +New features in Dataverse 6.4: + +- Enhanced DataCite Metadata, including "Relation Type" +- All ISO 639-3 languages are now supported +- There is now a button for "Unlink Dataset" +- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time +- Datasets can now have types such as "software" or "workflow" +- Croissant support +- RO-Crate support +- and more! Please see below. + +New client library: + +- Rust + +This release also fixes two important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: + +- "Update Current Version" can cause metadata loss +- Publishing breaks designated dataset thumbnail, messes up collection page + +Additional details on the above as well as many more features and bug fixes included in the release are described below. Read on! + +## Features Added + +### Enhanced DataCite Metadata, Including "Relation Type" + +Within the "Related Publication" field, a new subfield has been added called "Relation Type" that allows for the most common [values](https://datacite-metadata-schema.readthedocs.io/en/4.5/appendices/appendix-1/relationType/) recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. + +Dataverse now supports the [DataCite v4.5 schema](http://schema.datacite.org/meta/kernel-4/). Additional metadata is now being sent to DataCite including metadata about related publications and files in the dataset. Improved metadata is being sent including how PIDs (ORCID, ROR, DOIs, etc.), license/terms, geospatial, and other metadata are represented. The enhanced metadata will automatically be sent to DataCite when datasets are created and published. Additionally, after publication, you can inspect what was sent by looking at the DataCite XML export. + +The additions are in rough alignment with the OpenAIRE XML export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. For details see #10632, #10615 and the [design document](https://docs.google.com/document/d/1JzDo9UOIy9dVvaHvtIbOI8tFU6bWdfDfuQvWWpC0tkA/edit?usp=sharing) referenced there. + +Multiple backward incompatible changes and bug fixes have been made to API calls (three of four of which were not documented) related to updating PID target URLs and metadata at the provider service: +- [Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) +- [Update Target URL for all Published Datasets at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) +- [Update Metadata for a Published Dataset at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) +- [Update Metadata for all Published Datasets at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) + +### Full List of ISO 639-3 Languages Now Supported + +The controlled vocabulary values list for the metadata field "Language" in the citation block has now been extended to include roughly 7920 ISO 639-3 values. + +Some of the language entries in the pre-6.4 list correspond to "macro languages" in ISO-639-3 and admins/users may wish to update to use the corresponding individual language entries from ISO-639-3. As these cases are expected to be rare (they do not involve major world languages), finding them is not covered in the release notes. Anyone who desires help in this area is encouraged to reach out to the Dataverse community via any of the standard communication channels. + +ISO 639-3 codes were downloaded from [sil.org](https://iso639-3.sil.org/code_tables/download_tables#Complete%20Code%20Tables:~:text=iso%2D639%2D3_Code_Tables_20240415.zip) and the file used for merging with the existing citation.tsv was "iso-639-3.tab". See also #8578 and #10762. + +### Unlink Dataset Button + +A new "Unlink Dataset" button has been added to the dataset page to allow a user to unlink a dataset from a collection. To unlink a dataset the user must have permission to link the dataset. Additionally, the [existing API](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#unlink-a-dataset) for unlinking datasets has been updated to no longer require superuser access as the "Publish Dataset" permission is now enough. See also #10583 and #10689. + +### Pre-Publish File DOI Reservation + +Dataverse installations using DataCite as a persistent identifier (PID) provider (or other providers that support reserving PIDs) will be able to reserve PIDs for files when they are uploaded (rather than at publication time). Note that reserving file DOIs can slow uploads with large numbers of files so administrators may need to adjust timeouts (specifically any Apache "``ProxyPass / ajp://localhost:8009/ timeout=``" setting in the recommended Dataverse configuration). + +### Initial Support for Dataset Types + +Out of the box, all datasets now have the type "dataset" but superusers can add additional types. At this time the type of a dataset can only be set at creation time via API. The types "dataset", "software", and "workflow" (just those three, for now) will be sent to DataCite (as `resourceTypeGeneral`) when the dataset is published. + +For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-management.html#dataset-types), #10517 and #10694. Please note that this feature is highly experimental and is expected to [evolve](https://github.com/IQSS/dataverse-pm/issues/307). + +### Croissant Support (Metadata Export) + +A new metadata export format called [Croissant](https://github.com/mlcommons/croissant) is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. + +For more about the Croissant exporter, including installation instructions, see . See also #10341, #10533, and [discussion](https://groups.google.com/g/dataverse-community/c/JI8HPgGarr8/m/DqEIkiwlAgAJ) on the mailing list. + +Please note: the Croissant exporter works best with Dataverse 6.2 and higher (where it updates the content of `` as [described](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) in the guides) but can be used with 6.0 and higher (to get the export functionality). + +### RO-Crate Support (Metadata Export) + +Dataverse now supports [RO-Crate](https://www.researchobject.org/ro-crate/) as a metadata export format. This functionality is not available out of the box, but you can enable one or more RO-Crate exporters from the [list of external exporters](https://guides.dataverse.org/en/6.4/installation/advanced.html#inventory-of-external-exporters). See also #10744 and #10796. + +### Rust API Client Library + +An Dataverse API client library for the Rust programming language is now available at https://github.com/gdcc/rust-dataverse and has been added to the [list of client libraries](https://guides.dataverse.org/en/6.4/api/client-libraries.html) in the API Guide. See also #10758. + +### Collection Thumbnail Logo for Featured Collections + +Collections can now have a thumbnail logo that is displayed when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection as explained in [the guides](https://guides.dataverse.org/en/6.4/user/dataverse-management.html#theme). See also #10291 and #10433. + +### Saved Searches Can Be Deleted + +Saved searches can now be deleted via API. See the [Saved Search](https://guides.dataverse.org/en/6.4/api/native-api.html#saved-search) section of the API Guide, #9317 and #10198. + +### Notification Email Improvement + +When notification emails are sent the part of the closing that says "contact us for support at" will now show the support email address (`dataverse.mail.support-email`), when configured, instead of the default system email address. Using the system email address here was particularly problematic when it was a "noreply" address. See also #10287 and #10504. + +### Ability to Disable Automatic Thumbnail Selection + +It is now possible to turn off the feature that automatically selects one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by enabling the [feature flag](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) `dataverse.feature.disable-dataset-thumbnail-autoselect`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. See also #10820. + +### More Flexible PermaLinks + +The configuration setting `dataverse.pid.*.permalink.base-url`, which is used for PermaLinks, has been updated to support greater flexibility. Previously, the string `/citation?persistentId=` was automatically appended to the configured base URL. With this update, the base URL will now be used exactly as configured, without any automatic additions. See also #10775. + +### Globus Async Framework + +A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See `globus-use-experimental-async-framework` under [Feature Flags](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://guides.dataverse.org/en/6.4/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide. See also #10623 and #10781. + +### CVoc (Controlled Vocabulary): Allow ORCID and ROR to Be Used Together in Author Field + +Changes in Dataverse and updates to the ORCID and ROR external vocabulary scripts support deploying these for the citation block author field (and others). See also #10711, #10712, and . + +### Development on Windows + +New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See [the guides](https://guides.dataverse.org/en/6.4/developers/windows.html), #10606, and #10608. + +### Experimental Crossref PID (DOI) Provider + +Crossref can now be used as a PID (DOI) provider, but this feature is experimental. Please provide feedback through the usual channels. See also the [guides](https://guides.dataverse.org/en/6.4/installation/config.html#crossref-specific-settings), #8581, and #10806. + +### Improved JSON Schema Validation for Datasets + +JSON Schema validation has been enhanced with checks for required and allowed child objects as well as type checking for field types including `primitive`, `compound` and `controlledVocabulary`. More user-friendly error messages help pinpoint the issues in the dataset JSON. See [Retrieve a Dataset JSON Schema for a Collection](https://guides.dataverse.org/en/6.4/api/native-api.html#retrieve-a-dataset-json-schema-for-a-collection) in the API Guide, #10169, and #10543. + +### Counter Processor 1.05 Support (Make Data Count) + +Counter Processor 1.05 is now supported for use with Make Data Count. If you are running Counter Processor, you should reinstall/reconfigure it as described in the latest guides. Note that Counter Processor 1.05 requires Python 3, so you will need to follow the full Counter Processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse. See also #10479. + +### Version Tags for Container Base Images + +With this release we introduce a detailed maintenance workflow for our container images. As output of the [Containerization Working Group](https://ct.gdcc.io), the community takes another step towards production ready containers available directly from the core project. + +The maintenance workflow regularly updates the [Container Base Image](https://guides.dataverse.org/en/6.4/container/base-image.html), which contains the operating system, Java, Payara, and tools and libraries required by the Dataverse application. Shipping these rolling releases as well as immutable revisions is the foundation for secure and reliable [Dataverse Application Container](https://guides.dataverse.org/en/6.4/container/app-image.html) images. See also #10478 and #10827. + +## Bugs Fixed + +### Update Current Version + +A significant bug in the superuser-only [Update Current Version](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#make-metadata-updates-without-changing-dataset-version) publication option was fixed. If the "Update Current Version" option was used when changes were made to the dataset terms (rather than to dataset metadata) or if the PID provider service was down or returned an error, the update would fail and render the dataset unusable and require restoration from a backup. The fix in this release allows the update to succeed in both of these cases and redesigns the functionality such that any unknown issues should not make the dataset unusable (i.e. the error would be reported and the dataset would remain in its current state with the last-published version as it was and changes still in the draft version.) + +If you do not plan to upgrade to Dataverse 6.4 right away, you are encouraged to alert your superusers to this issue (see [this post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ)). Here are some workarounds for pre-6.4 versions: + +* Change the "dataset.updateRelease" entry in the Bundle.properties file (or local language version) to "Do Not Use" or similar (this doesn't disable the button but alerts superusers to the issue), or +* Edit the dataset.xhtml file to remove the lines below, delete the contents of the generated and osgi-cache directories in the Dataverse Payara domain, and restart the Payara server. This will remove the "Update Current Version" from the UI. + +``` + + + +``` + +Again, the workarounds above are only for pre-6.4 versions. The bug has been fixed in Dataverse 6.4. See also #10797. + +### Broken Thumbnails + +Dataverse 6.3 introduced a bug where publishing would break the dataset thumbnail, which in turn broke the rendering of the parent collection (dataverse) page. + +This bug has been fixed but any existing broken thumbnails must be fixed manually. See "clearThumbnailFailureFlag" in the upgrade instructions below. + +Additionally, it is now possible to turn off the feature that automatically selects of one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by raising the feature flag `-Ddataverse.feature.disable-dataset-thumbnail-autoselect=true`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. + +See also #10819, #10820, and [the post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list. + +### No License, No Terms of Use + +When datasets have neither a license nor custom terms of use, the dataset page will now indicate this. Also, these datasets will no longer be indexed as having custom terms. See also #8796, #10513, and #10614. + +### CC0 License Bug Fix + +At a high level, some datasets have been mislabeled as "Custom License" when they should have been "CC0 1.0". This has been corrected. + +In Dataverse 5.10, datasets with only "CC0 Waiver" in the "termsofuse" field were converted to "Custom License" (instead of the CC0 1.0 license) through a SQL migration script (see #10634). On deployment of Dataverse 6.4, a new SQL migration script will be run automatically to correct this, changing these datasets to CC0. You can review the script in #10634, which only affect the following datasets: + +- The existing "Terms of Use" must be equal to "This dataset is made available under a Creative Commons CC0 license with the following additional/modified terms and conditions: CC0 Waiver" (this was set in #10634). +- The following terms fields must be empty: Confidentiality Declaration, Special Permissions, Restrictions, Citation Requirements, Depositor Requirements, Conditions, and Disclaimer. +- The license ID must not be assigned. + +The script will set the license ID to that of the CC0 1.0 license and remove the contents of "termsofuse" field. See also #9081 and #10634. + +### Remap oai_dc Export and Harvesting Format Fields: dc:type and dc:date + +The `oai_dc` export and harvesting format has had the following fields remapped: + +- dc:type was mapped to the field "Kind of Data". Now it is hard-coded to the word "Dataset". +- dc:date was mapped to the field "Production Date" when available and otherwise to "Publication Date". Now it is mapped the field "Publication Date" or the field used for the citation date, if set (see [Set Citation Date Field Type for a Dataset](https://guides.dataverse.org/en/6.4/api/native-api.html#set-citation-date-field-type-for-a-dataset)). + +In order for these changes to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api) should be run (mentioned below). See #8129 and #10737. + +### Zip File No Longer Misdetected as Shapefile (Hidden Directories) + +When detecting files types, Dataverse would previously detect a zip file as a shapefile if it contained [markers of a shapefile](https://guides.dataverse.org/en/6.4/developers/geospatial.html) in hidden directories. These hidden directories are now ignored when deciding if a zip file is a shapefile or not. See also #8945 and #10627. + +### External Controlled Vocabulary + +This release fixes a bug (introduced in v6.3) in the external controlled vocabulary mechanism that could cause indexing to fail (with a NullPointerException) when a script is configured for one child field and no other child fields were managed. See also #10869 and #10870. + +### Valid JSON in Error Response + +When any `ApiBlockingFilter` policy applies to a request, the JSON in the body of the error response is now valid JSON. See also #10085. + +### Docker Container Base Image Security and Compatibility + +- Switch "wait-for" to "wait4x", aligned with the Configbaker Image +- Update "jattach" to v2.2 +- Install AMD64 / ARM64 versions of tools as necessary +- Run base image as unprivileged user by default instead of `root` - this was an oversight from OpenShift changes +- Linux User, Payara Admin and Domain Master passwords: + - Print hints about default, public knowledge passwords in place for + - Enable replacing these passwords at container boot time +- Enable building with updates Temurin JRE image based on Ubuntu 24.04 LTS +- Fix entrypoint script troubles with pre- and postboot script files +- Unify location of files at CONFIG_DIR=/opt/payara/config, avoid writing to other places + +See also #10508, #10672 and #10722. + +### Cleanup of Temp Directories + +In this release we addressed an issue where copies of files uploaded via the UI were left in one specific temp directory (`.../domain1/uploads` by default). We would like to remind all the installation admins that it is strongly recommended to have some automated (and aggressive) cleanup mechanisms in place for all the temp directories used by Dataverse. For example, at Harvard/IQSS we have the following configuration for the PrimeFaces uploads directory above: (note that, even with this fix in place, PrimeFaces will be leaving a large number of small log files in that location) + +Instead of the default location (`.../domain1/uploads`) we use a directory on a dedicated partition, outside of the filesystem where Dataverse is installed, via the following JVM option: + +``` +-Ddataverse.files.uploads=/uploads/web +``` + +and we have a dedicated cronjob that runs every 30 minutes and deletes everything older than 2 hours in that directory: + +``` +15,45 * * * * /bin/find /uploads/web/ -mmin +119 -type f -name "upload*" -exec rm -f {} \; > /dev/null 2>&1 +``` + +### Trailing Commas in Author Name Now Permitted + +When an author name ended in a comma (e.g. `Smith,` or `Smith, `), the dataset page was broken after publishing (a "500" error page was presented to the user). The underlying issue causing the JSON-LD Schema.org output on the page to break was fixed. See #10343 and #10776. + +## API Updates + +### Search API: affiliation, parentDataverseName, image_url, etc. + +The Search API (`/api/search`) response now includes additional fields, depending on the type. + +For collections (dataverses): + +- "affiliation" +- "parentDataverseName" +- "parentDataverseIdentifier" +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "Darwin's Finches", + ... + "affiliation": "Dataverse.org", + "parentDataverseName": "Root", + "parentDataverseIdentifier": "root", + "image_url":"/api/access/dvCardImage/{identifier}" +(etc, etc) +``` + +For datasets: + +- "image_url" (optional) + +```javascript +"items": [ + { + ... + "image_url": "http://localhost:8080/api/datasets/2/logo" + ... +(etc, etc) +``` + +For files: + +- "releaseOrCreateDate" +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "test.png", + ... + "releaseOrCreateDate": "2016-05-10T12:53:39Z", + "image_url":"/api/access/datafile/42?imageThumb=true" +(etc, etc) +``` + +These examples are also shown in the [Search API](https://guides.dataverse.org/en/6.4/api/search.html) section of the API Guide. + +The image_url field was already part of the SolrSearchResult JSON (and incorrectly appeared in Search API documentation), but it wasn't returned by the API because it was appended only after the Solr query was executed in the SearchIncludeFragment of JSF (the old/current UI framework). Now, the field is set in SearchServiceBean, ensuring it is always returned by the API when an image is available. + +The Solr schema.xml file has been updated to include a new field called "dvParentAlias" for supporting the new response field "parentDataverseIdentifier". See upgrade instructions below. + +See also #10810 and #10811. + +### Search API: publicationStatuses + +The Search API (`/api/search`) response will now include publicationStatuses in the JSON response as long as the list is not empty. + +Example: + +```javascript +"items": [ + { + "name": "Darwin's Finches", + ... + "publicationStatuses": [ + "Unpublished", + "Draft" + ], +(etc, etc) +``` + +See also #10733 and #10738. + +### Search Facet Information Exposed + +A new endpoint `/api/datasetfields/facetables` lists all facetable dataset fields defined in the installation, as described in [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-facetable-dataset-fields). + +A new optional query parameter "returnDetails" added to `/api/dataverses/{identifier}/facets/` endpoint to include detailed information of each DataverseFacet, as described in [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-facets-configured-for-a-dataverse-collection). See also #10726 and #10727. + +### User Permissions on Collections + +A new endpoint at `/api/dataverses/{identifier}/userPermissions` for obtaining the user permissions on a collection (dataverse) has been added. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#get-user-permissions-on-a-dataverse), #10749 and #10751. + +### addDataverse Extended + +The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended to allow adding metadata blocks, input levels and facet IDs at creation time, as the Dataverse page in create mode does in JSF. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#create-a-dataverse-collection), #10633 and #10644. + +### Metadata Blocks and Display on Create + +The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. + +The fields "depositor" and "dateOfDeposit" in the citation.tsv metadata block file have been updated to have the property "displayOnCreate" set to TRUE. In practice, only the API is affected because the UI has special logic that already shows these fields when datasets are created. See also and #10850 and #10884. + +### Feature Flags Can Be Listed + +It is now possible to list all feature flags and see if they are enabled or not. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags) and #10732. + +## Settings Added + +The following settings have been added: + +- dataverse.feature.disable-dataset-thumbnail-autoselect +- dataverse.feature.globus-use-experimental-async-framework +- dataverse.files.globus-monitoring-server +- dataverse.pid.*.crossref.url +- dataverse.pid.*.crossref.rest-api-url +- dataverse.pid.*.crossref.username +- dataverse.pid.*.crossref.password +- dataverse.pid.*.crossref.depositor +- dataverse.pid.*.crossref.depositor-email + +## Backward Incompatible Changes + +- The oai_dc export format has changed. See the "Remap oai_dc" section above. +- Several APIs related to DataCite have changed. See "More and Better Data Sent to DataCite" above. + +## Complete List of Changes + +For the complete list of code changes in this release, see the [6.4 milestone](https://github.com/IQSS/dataverse/issues?q=milestone%3A6.4+is%3Aclosed) in GitHub. + +## Getting Help + +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/g/dataverse-community) or email support@dataverse.org. + +## Installation + +If this is a new installation, please follow our [Installation Guide](https://guides.dataverse.org/en/latest/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/latest/installation/intro.html#getting-help) if you need it! + +Once you are in production, we would be delighted to update our [map of Dataverse installations](https://dataverse.org/installations) around the world to include yours! Please [create an issue](https://github.com/IQSS/dataverse-installations/issues) or email us at support@dataverse.org to join the club! + +You are also very welcome to join the [Global Dataverse Community Consortium](https://www.gdcc.io/) (GDCC). + +## Upgrade Instructions + +Upgrading requires a maintenance window and downtime. Please plan accordingly, create backups of your database, etc. + +These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.3. + +0\. These instructions assume that you are upgrading from the immediate previous version. If you are running an earlier version, the only supported way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to this version. + +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. + +In the following commands, we assume that Payara 6 is installed in `/usr/local/payara6`. If not, adjust as needed. + +```shell +export PAYARA=/usr/local/payara6` +``` + +(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) + +1\. Undeploy the previous version + +```shell +$PAYARA/bin/asadmin undeploy dataverse-6.3 +``` + +2\. Stop and start Payara + +```shell +service payara stop +sudo service payara start +``` + +3\. Deploy this version + +```shell +$PAYARA/bin/asadmin deploy dataverse-6.4.war +``` + +Note: if you have any trouble deploying, stop Payara, remove the following directories, start Payara, and try to deploy again. + +```shell +service payara stop +rm -rf $PAYARA/glassfish/domains/domain1/generated +rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache +rm -rf $PAYARA/glassfish/domains/domain1/lib/databases +``` + +4\. For installations with internationalization: + +Please remember to update translations via [Dataverse language packs](https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs). + +5\. Restart Payara + +```shell +service payara stop +service payara start +``` + +6\. Update metadata blocks + +These changes reflect incremental improvements made to the handling of core metadata fields. + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/scripts/api/data/metadatablocks/citation.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv +``` + +7\. Update Solr schema.xml file. Start with the standard v6.4 schema.xml, then, if your installation uses any custom or experimental metadata blocks, update it to include the extra fields (step 7a). + +Stop Solr (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.4/installation/prerequisites.html#solr-init-script)). + +```shell +service solr stop +``` + +Replace schema.xml + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/conf/solr/schema.xml +cp schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf +``` + +Start Solr (but if you use any custom metadata blocks, perform the next step, 7a first). + +```shell +service solr start +``` + +7a\. For installations with custom or experimental metadata blocks: + +Before starting Solr, update the schema to include all the extra metadata fields that your installation uses. We do this by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the names of the directories, if different): + +```shell + wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/conf/solr/update-fields.sh + chmod +x update-fields.sh + curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml +``` + +Now start Solr. + +8\. Reindex Solr + +Below is the simplest way to reindex Solr: + +```shell +curl http://localhost:8080/api/admin/index +``` + +The API above rebuilds the existing index "in place". If you want to be absolutely sure that your index is up-to-date and consistent, you may consider wiping it clean and reindexing everything from scratch (see [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html)). Just note that, depending on the size of your database, a full reindex may take a while and the users will be seeing incomplete search results during that window. + +9\. Run reExportAll to update dataset metadata exports + +This step is necessary because of changes described above for the `Datacite` and `oai_dc` export formats. + +Below is the simple way to reexport all dataset metadata. For more advanced usage, please see [the guides](http://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api). + +```shell +curl http://localhost:8080/api/admin/metadata/reExportAll +``` + +10\. Pushing updated metadata to DataCite + +(If you don't use DataCite, you can skip this.) + +Above you updated the citation metadata block and Solr with the new "relationType" field. With these two changes, the "Relation Type" fields will be available and creation/publication of datasets will result in the expanded XML being sent to DataCite. You've also already run "reExportAll" to update the `Datacite` metadata export format. + +Entries at DataCite for published datasets can be updated by a superuser using an API call (newly [documented](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider)): + +`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets/modifyRegistrationPIDMetadataAll` + +This will loop through all published datasets (and released files with PIDs). As long as the loop completes, the call will return a 200/OK response. Any PIDs for which the update fails can be found using the following command: + +`grep 'Failure for id' server.log` + +Failures may occur if PIDs were never registered, or if they were never made findable. Any such cases can be fixed manually in DataCite Fabrica or using the [Reserve a PID](https://guides.dataverse.org/en/6.4/api/native-api.html#reserve-a-pid) API call and the newly documented `/api/datasets//modifyRegistration` call respectively. See https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#send-dataset-metadata-to-pid-provider. Please reach out with any questions. + +PIDs can also be updated by a superuser on a per-dataset basis using + +`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets//modifyRegistrationMetadata` + +### Additional Upgrade Steps + +11\. If there are broken thumbnails + +To restore any broken thumbnails caused by the bug described above, you can call the `http://localhost:8080/api/admin/clearThumbnailFailureFlag` API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the `http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID` to clear the flag for individual files. Calling the former, batch API is recommended. + +12\. PermaLinks with custom base-url + +If you currently use PermaLinks with a custom `base-url`: You must manually append `/citation?persistentId=` to the base URL to maintain functionality. + +If you use a PermaLinks without a configured `base-url`, no changes are required. diff --git a/doc/release-notes/6.5-release-notes.md b/doc/release-notes/6.5-release-notes.md new file mode 100644 index 00000000000..2e27f4419bd --- /dev/null +++ b/doc/release-notes/6.5-release-notes.md @@ -0,0 +1,391 @@ +# Dataverse 6.5 + +Please note: To read these instructions in full, please go to https://github.com/IQSS/dataverse/releases/tag/v6.5 rather than the [list of releases](https://github.com/IQSS/dataverse/releases), which will cut them off. + +This release brings new features, enhancements, and bug fixes to Dataverse. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project! + +## Release Highlights + +Highlights for Dataverse 6.5 include: + +- new API endpoints, including editing of collections, Search API file counts, listing of exporters, comparing dataset versions, and auditing data files +- UX improvements, especially Preview URLs +- increased harvesting flexibility +- performance gains +- a [security vulnerability](https://github.com/IQSS/dataverse-security/issues/98) addressed +- many bug fixes +- and more! Please see below. + +## Features Added + +### Private URL Renamed to Preview URL and Improved + +The name of the URL that may be used by dataset administrators to share a draft version of a dataset has been changed from Private URL to Preview URL. + +Also, additional information about the creation of Preview URLs has been added to the popup accessed via edit menu of the Dataset Page. + +Users of the Anonymous Preview URL will no longer be able to see the name of the Dataverse that the dataset is in but will be able to see the name of the repository. + +Any Private URLs created in previous versions of Dataverse will continue to work. + +The old "privateUrl" API endpoints for the creation and deletion of Preview (formerly Private) URLs have been deprecated. They will continue to work but please switch to the "previewUrl" equivalents that have been [documented](https://guides.dataverse.org/en/6.5/api/native-api.html#create-a-preview-url-for-a-dataset) in the API Guide. + +See also #8184, #8185, #10950, #10961, and #11085. + +### Showing Differences Between Dataset Versions is More Scalable + +Showing differences between dataset versions, which is done during dataset edit operations and to populate the dataset page versions table, has been made significantly more scalable. See #10814 and #10818. + +### Version Differences Details Sorting Added + +In order to facilitate the comparison between the draft version and the published version of a dataset, a sort on subfields has been added. See #10969. + +### Reindexing After a Role Assignment is Less Memory Intensive + +Adding or removing a user from a role on a collection, particularly the root collection, could lead to a significant increase in memory use, resulting in Dataverse itself failing with an out-of-memory condition. Such changes now consume much less memory. A Solr reindexing step is included in the upgrade instructions below. See also #10697 and #10698. + +### Longer Custom Questions in Guestbooks + +Custom questions in Guestbooks can now be more than 255 characters and the bug causing a silent failure when questions were longer than this limit has been fixed. See also #9492, #10117, #10118. + +### PostgreSQL and Flyway Updates + +This release bumps the version of PostgreSQL and Flyway used in containers as well as the PostgreSQL JDBC driver used all installations, including classic (non-Docker) installations. PostgreSQL and its driver have been bumped to version 17. Flyway has been bumped to version 10. + +PostgreSQL 13 remains the version used with automated testing, leading us to continue to [recommend](https://guides.dataverse.org/en/6.5/installation/prerequisites.html#postgresql) that version for classic installations. + +As of Flyway 10, supporting older versions of PostgreSQL no longer requires a paid subscription. While we don't encourage the use of older PostgreSQL versions, this flexibility may benefit some of our long-standing installations in their upgrade paths. + +As part of this update, the containerized development environment now uses Postgres 17 instead of 16. Developers must delete their data (`rm -rf docker-dev-volumes`) and start with an empty database (rerun the [quickstart](https://guides.dataverse.org/en/6.5/developers/dev-environment.html#quickstart) in the dev guide), as [explained](https://groups.google.com/g/dataverse-dev/c/ffoNj5UXyzU/m/nE5oGY_sAQAJ) on the dev mailing list. + +The Docker compose file used for [evaluations or demos](https://guides.dataverse.org/en/6.4/container/running/demo.html) has been upgraded from Postgres 13 to 17. + +See also #10889 and #10912. + +### Harvesting "oai_dc" Metadata Prefix When Extended With Specific Namespaces + +Some data repositories extend the "oai_dc" metadata prefix with specific namespaces. In this case, harvesting of these datasets into Dataverse was not possible because an XML parsing error was raised. + +Harvesting of these datasets has been fixed by excluding tags with namespaces that are not "dc:". That is, only harvesting metadata with the "dc" namespace. See #10837. + +### Harvested Dataset PID from Record Header + +When harvesting, Dataverse can now use the identifier from the OAI-PMH record header as the persistent id for the harvested dataset. + +This will allow harvesting from sources that do not include a persistent id in their oai_dc metadata records, but use valid DOIs or handles as the OAI-PMH record header identifiers. + +It is also possible to optionally configure a harvesting client to use this OAI-PMH identifier as the **preferred** choice for the persistent id. See the [Harvesting Clients API](https://guides.dataverse.org/en/6.5/api/native-api.html#create-a-harvesting-client) section of the Guides, #11049 and #10982 for more information. + +### Harvested Datasets Can Have Multiple "otherId" Values + +When harvesting using the DDI format, datasets can now have multiple "otherId" values. See #10772. + +### Multiple Languages in Docker + +Documentation has been added to explain how to set up multiple languages (e.g. English and French) in the tutorial for setting up Dataverse in Docker. + +See [the tutorial](https://guides.dataverse.org/en/6.5/container/running/demo.html#multiple-languages), #10939, and #10940. + +### GlobusBatchLookupSize + +An optimization has been added for the Globus upload workflow, with a corresponding new database setting: `:GlobusBatchLookupSize` + +See the [Database Settings](https://guides.dataverse.org/en/6.5/installation/config.html#GlobusBatchLookupSize) section of the guides, #10977, and #11040 for more information. + +## Bugs Fixed + +### Relation Type (Related Publication) and DataCite + +The subfield "Relation Type" was added to the field "Related Publication" in Dataverse 6.4 (#10632) but couldn't be used without workarounds described in an [announcement](https://groups.google.com/g/dataverse-community/c/zlRGJtu3x4g/m/GtVZ26uaBQAJ) about the problem. The bug has been fixed and workarounds are no longer required. See #10926 and the announcement above. + +### Sort Order for Files + +"Newest" and "Oldest" were reversed when sorting files on the dataset landing page. This has been fixed. See #10742 and #11000. + +### Guestbook Email Validation + +In the Guestbook UI form, the email address is now checked for validity. See #10661 and #11022. + +### Updating Files Now Possible When Latest and Only Dataset Version is Deaccessioned + +When a dataset was deaccessioned, and was the only previous version, it would cause an error when trying to update the files. This has been fixed. See #9351 and #10901. + +### My Data Filter by Username Feature Restored + +The superuser-only feature of filtering by a username on the My Data page was not working. Entering a username in the "Results for Username" field now returns data for the desired user. See also #7239 and #10980. + +### Better Handling of Parallel Edit/Publish Errors + +Improvements have been made in handling the errors when a dataset has been edited in one browser window and an attempt is made to edit or publish it in another. (This practice is discouraged, by the way.) See #10793 and #10794. + +### Facets Filter Labels Now Translated Above Search Results + +On the main page, it's possible to filter results using search facets. If internationalization (i18n) has been enabled in the Dataverse installation, allowing pages to be displayed in several languages, the facets were correctly translated in the filter column at the left. However, they were not being translated above the search results, remaining in the default language, English. This has been fixed. See #9408 and #10158. + +### Unpublished File Bug Fix Related to Deaccessioning + +A bug fix was made related to retrieval of the major version of a Dataset when all major versions were deaccessioned. This fixes the incorrect showing of the files as "Unpublished" in the search list even when they are published. In the upgrade instructions below, there is a step to reindex Solr. See also #10947 and #10974. + +### Minor DataCiteXML Fix (Useless Null) + +A minor bug fix was made to avoid sending a useless ", null" in the DataCiteXML sent to DataCite and in the DataCite export when a dataset has a metadata entry for "Software Name" and no entry for "Software Version". The bug fix will update datasets upon publication. Anyone with existing published datasets with this problem can be fixed by [pushing updated metadata to DataCite for affected datasets](https://guides.dataverse.org/en/6.5/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) and [re-exporting the dataset metadata](https://guides.dataverse.org/en/6.5/admin/metadataexport.html#batch-exports-through-the-api). See "Pushing updated metadata to DataCite" in the upgrade instructions below. See also #10919. + +### PIDs and Make Data Count Citation Retrieval + +Make Data Count (MDC) citation retrieval with the PID settings has been fixed. PID parsing in Dataverse is now case insensitive, improving interaction with services that may change the case of PIDs. Warnings related to managed/excluded PID lists for PID providers have been reduced. See #10708. + +### Quirk in Overview Display When Using External Controlled Variables + +This bugfix corrects an issue when there are duplicated entries on the metadata page. It is fixed by correcting an IF-clause in metadataFragment.xhtml. See #11005 and #11034. + +### Globus "missing properties" Logging Fixed + +In previous releases, logging would show Globus-related strings were missing from properties files. This has been fixed. See #11030. + +## API Updates + +### Editing Collections + +A new endpoint (`PUT /api/dataverses/`) for updating an existing collection (dataverse) has been added. It uses the same JSON structure as the one used for collection creation. See also [the docs](https://guides.dataverse.org/en/6.5/api/native-api.html#update-a-dataverse-collection), #10904, and #10925. + +### fileCount Added to Search API + +A new search field called `fileCount` can be searched to discover the number of files per dataset. The upgrade instructions below explain how to update your Solr `schema.xml` file to add the new field and reindex Solr. See also #8941 and #10598. + +### List Dataset Metadata Exporters + +A list of available dataset metadata exporters can now be retrieved programmatically via API. See [the docs](https://guides.dataverse.org/en/6.5/api/native-api.html#get-export-formats) and #10739. + +### Comparing Dataset Versions + +An API has been added to compare dataset versions. See [the docs](https://guides.dataverse.org/en/6.5/api/native-api.html#compare-versions-of-a-dataset), #10888, and #10945. + +### Audit Data Files + +A superuser-only API endpoint has been added to audit datasets with data files where the physical files are missing or the file metadata is missing. See [the docs](https://guides.dataverse.org/en/6.5/api/native-api.html#datafile-audit), #11016, and [#220](https://github.com/IQSS/dataverse.harvard.edu/issues/220). + +### Update Collection API Inheritance + +The update collection (dataverse) API endpoint has been updated to support an "inherit from parent" configuration for metadata blocks, facets, and input levels. + +Previously, not setting these fields meant using a copy of the settings from the parent collection, which could get out of sync. See also [the docs](https://guides.dataverse.org/en/6.5/api/native-api.html#update-a-dataverse-collection), #11018, and #11026. + +### isMetadataBlockRoot and isFacetRoot + +The JSON payload of the "get collection" endpoint has been extended to include properties isMetadataBlockRoot and isFacetRoot. See also [the docs](https://guides.dataverse.org/en/6.5/api/native-api.html#view-a-dataverse-collection), #11012, and #11013. + +### Whitespace Trimming When Loading Metadata Block TSV Files + +When loading custom metadata blocks using the `api/admin/datasetfield/load` API endpoint, whitespace can be introduced into field names. Whitespace is now trimmed from the beginning and end of all values read into the API before persisting them. See #10688 and #10696. + +### Image URLs from the Search API + +As of 6.4 (#10855) `image_url` is being returned from the Search API. The logic has been updated to only show the image if each of the following are true: + +1. The data file is not harvested +2. A thumbnail is available for the data file +3. If the data file is restricted, then the caller must have DownloadFile permission for the data file +4. The data file is NOT actively embargoed +5. The data file's retention period has NOT expired + +See also #10875 and #10886. + +### Metrics API Bug Fixes + +Two bugs in the Metrics API have been fixed: + +- The /datasets and /datasets/byMonth endpoints could report incorrect values if or when they have been called using the "dataLocation" parameter (which allows getting metrics for local, remote (harvested), or all datasets) as the metrics cache was not storing different values for these cases. + +- Metrics endpoints whose calculation relied on finding the latest published dataset version were incorrect if/when the minor version number was > 9. + +The upgrade instructions below include a step for clearing the metrics cache. + +See also #10379 and #10865. + +### API Tokens + +An optional query parameter called "returnExpiration" has been added to the `/api/users/token/recreate` endpoint, which, if set to true, returns the expiration time in the response. See [the docs](https://guides.dataverse.org/en/6.5/api/native-api.html#recreate-a-token), #10857 and #10858. + +The `/api/users/token` endpoint has been extended to support any auth mechanism for retrieving the token information. Previously this endpoint only accepted an API token to retrieve its information. Now it accepts any authentication mechanism and returns the associated API token information. See #10914 and #10924. + +## Settings Added + +- `:GlobusBatchLookupSize` + +## Backward Incompatible Changes + +Generally speaking, see the [API Changelog](https://guides.dataverse.org/en/latest/api/changelog.html) for a list of backward-incompatible API changes. + +### List Collections Linked to a Dataset + +The API endpoint that returns a list of collections that a dataset has been linked to has been improved to provide a more structured JSON response. See [the docs](https://guides.dataverse.org/en/6.5/admin/dataverses-datasets.html#list-collections-that-are-linked-from-a-dataset), #9650, and #9665. + +## Complete List of Changes + +For the complete list of code changes in this release, see the [6.5 milestone](https://github.com/IQSS/dataverse/issues?q=milestone%3A6.5+is%3Aclosed) in GitHub. + +## Getting Help + +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/g/dataverse-community) or email support@dataverse.org. + +## Installation + +If this is a new installation, please follow our [Installation Guide](https://guides.dataverse.org/en/latest/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/latest/installation/intro.html#getting-help) if you need it! + +Once you are in production, we would be delighted to update our [map of Dataverse installations](https://dataverse.org/installations) around the world to include yours! Please [create an issue](https://github.com/IQSS/dataverse-installations/issues) or email us at support@dataverse.org to join the club! + +You are also very welcome to join the [Global Dataverse Community Consortium](https://www.gdcc.io/) (GDCC). + + +## Upgrade Instructions + +Upgrading requires a maintenance window and downtime. Please plan accordingly, create backups of your database, etc. + +These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.4. + +0\. These instructions assume that you are upgrading from the immediate previous version. If you are running an earlier version, the only supported way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to this version. + +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. By default, Payara runs as the `dataverse` user. In the commands below, we use sudo to run the commands as a non-root user. + +Also, we assume that Payara 6 is installed in `/usr/local/payara6`. If not, adjust as needed. + +```shell +export PAYARA=/usr/local/payara6 +``` + +(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) + +1\. List deployed applications + +```shell +$PAYARA/bin/asadmin list-applications +``` + +2\. Undeploy the previous version (should match "list-applications" above) + +```shell +$PAYARA/bin/asadmin undeploy dataverse-6.4 +``` + +3\. Stop and start Payara + +```shell +sudo service payara stop +sudo service payara start +``` + +4\. Download and deploy this version + +```shell +wget https://github.com/IQSS/dataverse/releases/download/v6.5/dataverse-6.5.war +$PAYARA/bin/asadmin deploy dataverse-6.5.war +``` + +Note: if you have any trouble deploying, stop Payara, remove the following directories, start Payara, and try to deploy again. + +```shell +sudo service payara stop +sudo rm -rf $PAYARA/glassfish/domains/domain1/generated +sudo rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache +sudo rm -rf $PAYARA/glassfish/domains/domain1/lib/databases +``` + +5\. For installations with internationalization: + +Please remember to update translations via [Dataverse language packs](https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs). + +6\. Restart Payara + +```shell +sudo service payara stop +sudo service payara start +``` +7\. Update Solr schema.xml file. Start with the standard v6.5 schema.xml, then, if your installation uses any custom or experimental metadata blocks, update it to include the extra fields (step 7a). + +Run the commands below as a non-root user. + +Stop Solr (usually `sudo service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.5/installation/prerequisites.html#solr-init-script)). + +```shell +sudo service solr stop +``` + +Replace schema.xml + +Please note that the path to Solr may differ from the example below. + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.5/conf/solr/schema.xml +sudo cp schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf +``` + +Start Solr (but if you use any custom metadata blocks, perform the next step, 7a first). + +```shell +sudo service solr start +``` + +7a\. For installations with custom or experimental metadata blocks: + +Before starting Solr, update the `schema.xml` file to include all the extra metadata fields that your installation uses. + +We do this by collecting the output of Dataverse's Solr schema API endpoint (`/api/admin/index/solr/schema`) and piping it to the `update-fields.sh` script which updates the `schema.xml` file supplied as an argument. + +The example below assumes the default installation location of Solr, but you can modify the commands as needed. + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.5/conf/solr/update-fields.sh +chmod +x update-fields.sh +curl "http://localhost:8080/api/admin/index/solr/schema" | sudo ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml +``` + +Now start Solr. + + +```shell +sudo service solr start +``` + +8\. Reindex Solr + +Below is the simplest way to reindex Solr: + +```shell +curl http://localhost:8080/api/admin/index +``` + +The API above rebuilds the existing index. If you want to be absolutely sure that your index is up-to-date and consistent, you may consider wiping it clean and reindexing everything from scratch (see [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html)). Just note that, depending on the size of your database, a full reindex may take a while and the users will be seeing incomplete search results during that window. + +9\. Run reExportAll to update dataset metadata exports + +Below is the simple way to reexport all dataset metadata. For more advanced usage, please see [the guides](http://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api). + +```shell +curl http://localhost:8080/api/admin/metadata/reExportAll +``` + +10\. Clear metrics cache + +Run the [clearMetricsCache](https://guides.dataverse.org/en/6.5/api/native-api.html#metrics) API endpoint to remove old cached values that may be incorrect. + +```shell +curl -X DELETE http://localhost:8080/api/admin/clearMetricsCache +``` + +11\. Pushing updated metadata to DataCite + +(If you don't use DataCite, you can skip this. Also, if you aren't affected by the "useless null" bug described above, you can skip this.) + +Entries at DataCite for published datasets can be updated by a superuser using an API call (newly [documented](https://guides.dataverse.org/en/6.5/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider)): + +`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets/modifyRegistrationPIDMetadataAll` + +This will loop through all published datasets (and released files with PIDs). As long as the loop completes, the call will return a 200/OK response. Any PIDs for which the update fails can be found using the following command: + +`grep 'Failure for id' server.log` + +Failures may occur if PIDs were never registered, or if they were never made findable. Any such cases can be fixed manually in DataCite Fabrica or using the [Reserve a PID](https://guides.dataverse.org/en/6.4/api/native-api.html#reserve-a-pid) API call and the newly documented `/api/datasets//modifyRegistration` call respectively. See https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#send-dataset-metadata-to-pid-provider. Please reach out with any questions. + +PIDs can also be updated by a superuser on a per-dataset basis using + +`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets//modifyRegistrationMetadata` diff --git a/doc/release-notes/master_json_fix.md b/doc/release-notes/master_json_fix.md new file mode 100644 index 00000000000..aa30b90c2cb --- /dev/null +++ b/doc/release-notes/master_json_fix.md @@ -0,0 +1 @@ +This pull request fixes an issue in the JsonPrinter class so that there are no duplicated entries in the JSON metadata or ommitted metadata properties. After the fix is applied the /api/metadatablocks/ endpoint should return correct JSON. \ No newline at end of file diff --git a/doc/sphinx-guides/source/_static/api/dataset-create-software.json b/doc/sphinx-guides/source/_static/api/dataset-create-software.json new file mode 100644 index 00000000000..4c649bff0aa --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/dataset-create-software.json @@ -0,0 +1,82 @@ +{ + "datasetType": "software", + "datasetVersion": { + "license": { + "name": "CC0 1.0", + "uri": "http://creativecommons.org/publicdomain/zero/1.0" + }, + "metadataBlocks": { + "citation": { + "fields": [ + { + "value": "pyDataverse", + "typeClass": "primitive", + "multiple": false, + "typeName": "title" + }, + { + "value": [ + { + "authorName": { + "value": "Range, Jan", + "typeClass": "primitive", + "multiple": false, + "typeName": "authorName" + }, + "authorAffiliation": { + "value": "University of Stuttgart", + "typeClass": "primitive", + "multiple": false, + "typeName": "authorAffiliation" + } + } + ], + "typeClass": "compound", + "multiple": true, + "typeName": "author" + }, + { + "value": [ + { "datasetContactEmail" : { + "typeClass": "primitive", + "multiple": false, + "typeName": "datasetContactEmail", + "value" : "jan@mailinator.com" + }, + "datasetContactName" : { + "typeClass": "primitive", + "multiple": false, + "typeName": "datasetContactName", + "value": "Range, Jan" + } + }], + "typeClass": "compound", + "multiple": true, + "typeName": "datasetContact" + }, + { + "value": [ { + "dsDescriptionValue":{ + "value": "A Python module for Dataverse.", + "multiple":false, + "typeClass": "primitive", + "typeName": "dsDescriptionValue" + }}], + "typeClass": "compound", + "multiple": true, + "typeName": "dsDescription" + }, + { + "value": [ + "Computer and Information Science" + ], + "typeClass": "controlledVocabulary", + "multiple": true, + "typeName": "subject" + } + ], + "displayName": "Citation Metadata" + } + } + } +} diff --git a/doc/sphinx-guides/source/_static/api/dataset-create-software.jsonld b/doc/sphinx-guides/source/_static/api/dataset-create-software.jsonld new file mode 100644 index 00000000000..6f072967dc8 --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/dataset-create-software.jsonld @@ -0,0 +1,16 @@ +{ + "http://purl.org/dc/terms/title": "Darwin's Finches", + "http://purl.org/dc/terms/subject": "Medicine, Health and Life Sciences", + "http://purl.org/dc/terms/creator": { + "https://dataverse.org/schema/citation/authorName": "Finch, Fiona", + "https://dataverse.org/schema/citation/authorAffiliation": "Birds Inc." + }, + "https://dataverse.org/schema/citation/datasetContact": { + "https://dataverse.org/schema/citation/datasetContactEmail": "finch@mailinator.com", + "https://dataverse.org/schema/citation/datasetContactName": "Finch, Fiona" + }, + "https://dataverse.org/schema/citation/dsDescription": { + "https://dataverse.org/schema/citation/dsDescriptionValue": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds." + }, + "https://dataverse.org/schema/core#datasetType": "software" +} diff --git a/doc/sphinx-guides/source/_static/api/dataverse-complete-optional-params.json b/doc/sphinx-guides/source/_static/api/dataverse-complete-optional-params.json new file mode 100644 index 00000000000..fef32aa1e2c --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/dataverse-complete-optional-params.json @@ -0,0 +1,65 @@ +{ + "name": "Scientific Research", + "alias": "science", + "dataverseContacts": [ + { + "contactEmail": "pi@example.edu" + }, + { + "contactEmail": "student@example.edu" + } + ], + "affiliation": "Scientific Research University", + "description": "We do all the science.", + "dataverseType": "LABORATORY", + "metadataBlocks": { + "metadataBlockNames": [ + "citation", "geospatial" + ], + "inputLevels": [ + { + "datasetFieldTypeName": "geographicCoverage", + "include": true, + "required": true + }, + { + "datasetFieldTypeName": "country", + "include": true, + "required": true + }, + { + "datasetFieldTypeName": "geographicUnit", + "include": false, + "required": false + }, + { + "datasetFieldTypeName": "geographicBoundingBox", + "include": false, + "required": false + }, + { + "datasetFieldTypeName": "westLongitude", + "include": false, + "required": false + }, + { + "datasetFieldTypeName": "eastLongitude", + "include": false, + "required": false + }, + { + "datasetFieldTypeName": "northLatitude", + "include": false, + "required": false + }, + { + "datasetFieldTypeName": "southLatitude", + "include": false, + "required": false + } + ], + "facetIds": [ + "authorName", "authorAffiliation" + ] + } +} diff --git a/doc/sphinx-guides/source/_static/util/counter_daily.sh b/doc/sphinx-guides/source/_static/util/counter_daily.sh index 674972b18f2..5095a83b7e2 100644 --- a/doc/sphinx-guides/source/_static/util/counter_daily.sh +++ b/doc/sphinx-guides/source/_static/util/counter_daily.sh @@ -1,6 +1,6 @@ #! /bin/bash -COUNTER_PROCESSOR_DIRECTORY="/usr/local/counter-processor-0.1.04" +COUNTER_PROCESSOR_DIRECTORY="/usr/local/counter-processor-1.05" MDC_LOG_DIRECTORY="/usr/local/payara6/glassfish/domains/domain1/logs/mdc" # counter_daily.sh diff --git a/doc/sphinx-guides/source/_templates/navbar.html b/doc/sphinx-guides/source/_templates/navbar.html index c7b81dcb937..d88306be8ae 100644 --- a/doc/sphinx-guides/source/_templates/navbar.html +++ b/doc/sphinx-guides/source/_templates/navbar.html @@ -25,7 +25,6 @@
  • About the Project
    • -
  • Add Data
  • Blog
  • Presentations
  • Publications
    • diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index 37494c57fa1..c6d325a9651 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -122,6 +122,8 @@ Creates a link between a dataset and a Dataverse collection (see the :ref:`datas curl -H "X-Dataverse-key: $API_TOKEN" -X PUT http://$SERVER/api/datasets/$linked-dataset-id/link/$linking-dataverse-alias +.. _list-collections-linked-from-dataset: + List Collections that are Linked from a Dataset ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -129,15 +131,21 @@ Lists the link(s) created between a dataset and a Dataverse collection (see the curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/datasets/$linked-dataset-id/links -It returns a list in the following format: +It returns a list in the following format (new format as of v6.4): .. code-block:: json { "status": "OK", "data": { - "dataverses that link to dataset id 56782": [ - "crc990 (id 18802)" + "id": 5, + "identifier": "FK2/OTCWMM", + "linked-dataverses": [ + { + "id": 2, + "alias": "dataverse1", + "displayName": "Lab experiments 2023 June" + } ] } } @@ -147,7 +155,7 @@ It returns a list in the following format: Unlink a Dataset ^^^^^^^^^^^^^^^^ -Removes a link between a dataset and a Dataverse collection. Only accessible to superusers. :: +Removes a link between a dataset and a Dataverse collection. Accessible to users with Publish Dataset permissions. :: curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/datasets/$linked-dataset-id/deleteLink/$linking-dataverse-alias @@ -195,12 +203,41 @@ Mints a new identifier for a dataset previously registered with a handle. Only a .. _send-metadata-to-pid-provider: -Send Dataset metadata to PID provider -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Update Target URL for a Published Dataset at the PID provider +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Forces update to the target URL provided to the PID provider of a published dataset and assures the PID is findable. +Only accessible to superusers. :: + + curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/datasets/$dataset-id/modifyRegistration + +Update Target URL for all Published Datasets at the PID provider +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Forces update to the target URL provided to the PID provider of all published datasets and assures the PID is findable. +Only accessible to superusers. :: -Forces update to metadata provided to the PID provider of a published dataset. Only accessible to superusers. :: + curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/datasets/modifyRegistrationAll + +Update Metadata for a Published Dataset at the PID provider +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Checks to see that the PID metadata for a published dataset (and any released files in it using file PIDs) +is up-to-date at the provider and updates the metadata if necessary. +Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/datasets/$dataset-id/modifyRegistrationMetadata + +Update Metadata for all Published Datasets at the PID provider +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Checks to see that the PID metadata is up-to-date at the provider for all published datasets +(and any released files in them using file PIDs) and updates the metadata if necessary. +Only accessible to superusers. :: + + curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/datasets/modifyRegistrationPIDMetadataAll + +The call returns 200/OK as long as the call completes. Any errors for individual datasets are reported in the log. Check for Unreserved PIDs and Reserve Them ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/sphinx-guides/source/admin/discoverability.rst b/doc/sphinx-guides/source/admin/discoverability.rst index 767bb55bce6..19ef7250a29 100644 --- a/doc/sphinx-guides/source/admin/discoverability.rst +++ b/doc/sphinx-guides/source/admin/discoverability.rst @@ -30,14 +30,21 @@ The HTML source of a dataset landing page includes "DC" (Dublin Core) ```` `` of the HTML source of a dataset landing page includes Schema.org JSON-LD metadata like this:: + + \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 427615f2f0b..732c634205f 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -182,6 +182,11 @@ webmanifest application/manifest+json + + xhtml + text/html + + diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index 6d5b0a5fe4f..03173faf989 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -12,7 +12,7 @@ or !empty termsOfUseAndAccess.originalArchive or !empty termsOfUseAndAccess.availabilityStatus or !empty termsOfUseAndAccess.contactForAccess or !empty termsOfUseAndAccess.sizeOfCollection or !empty termsOfUseAndAccess.studyCompletion - or termsOfUseAndAccess.fileAccessRequest}"/> + }"/>

    - +

    - - + +

    diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 936d354e9d7..9426884d349 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -20,6 +20,7 @@ + - + @@ -120,7 +121,7 @@ - + @@ -522,6 +523,16 @@
    + +
    + + #{bundle['dataset.unlinkBtn']} + +
    +
    @@ -585,13 +596,13 @@ - + - ) - +
    @@ -631,6 +642,7 @@ or !empty DatasetPage.datasetVersionUI.keywordDisplay or !empty DatasetPage.datasetVersionUI.subject.value or !empty DatasetPage.datasetVersionUI.relPublicationCitation + or !empty DatasetPage.datasetVersionUI.relPublicationUrl or !empty DatasetPage.datasetVersionUI.notes.value) and !empty DatasetPage.datasetSummaryFields}"> @@ -650,8 +662,10 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{DatasetPage.datasetVersionUI.datasetRelPublications.get(0).description}">
    - - #{DatasetPage.datasetVersionUI.relPublicationId} + + + #{empty DatasetPage.datasetVersionUI.relPublicationId ? DatasetPage.datasetVersionUI.relPublicationUrl:DatasetPage.datasetVersionUI.relPublicationId} + @@ -821,7 +835,6 @@ /> @@ -982,7 +995,6 @@

    +

    #{bundle['dataset.privateurl.introduction']}

    +

    +

    +

    #{bundle['dataset.privateurl.onlyone']}

    +

    + +

    +

    +

    +

    #{bundle['dataset.privateurl.general.description']}

    + + +
    +
    + +

    + #{privateUrlLink} +

    +
    +
    +
    + + +
    +     +

    + +

    +

    +

    +

    #{bundle['dataset.privateurl.anonymous.description']}

    +

    #{bundle['dataset.privateurl.anonymous.description.paragraph.two']} #{bundle['dataset.privateurl.createPrivateUrl.anonymized.unavailable']}.

    + + + + +

    + +
    +
    + +

    + #{privateUrlLink} +

    +
    +
    +
    + + +
    +     +

    #{bundle['dataset.privateurl.absent']}

    @@ -1187,17 +1273,11 @@

    -
    - - - - - -
    +

    #{bundle['dataset.privateurl.cannotCreate']}

    @@ -1211,7 +1291,10 @@

    #{bundle['dataset.privateurl.disableConfirmationText']}

    - + + + + @@ -1697,6 +1780,72 @@
    + + +
    +

    + + + + + + + + + + +

    +
    + +
    + + + + + + + + + + + + + +
    +
    +
    +
    + +   + + +
    +
    + + + + +
    +     +
    diff --git a/src/main/webapp/datasetLicenseInfoFragment.xhtml b/src/main/webapp/datasetLicenseInfoFragment.xhtml index 257f6b3b12f..a1bd604aa6a 100644 --- a/src/main/webapp/datasetLicenseInfoFragment.xhtml +++ b/src/main/webapp/datasetLicenseInfoFragment.xhtml @@ -22,7 +22,7 @@ xmlns:jsf="http://xmlns.jcp.org/jsf"> target="_blank">#{DatasetUtil:getLicenseName(DatasetPage.workingVersion)} -

    diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 7f70f28e194..d452f76c9cd 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -636,8 +636,9 @@
    - #{of:format1(bundle['alt.logo'], dv.name)} - + #{of:format1(bundle['alt.logo'], dv.name)} + #{of:format1(bundle['alt.logo'], dv.name)} +
    diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index 9ed8b5209b6..bcb71733716 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -367,6 +367,20 @@ + + + + + + + + + + + + + + diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 117710cfd53..154700f7cf4 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -301,24 +301,24 @@
  • - + +
  • - + -
  • diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml index 5948047d845..6eec4f100b1 100644 --- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml +++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml @@ -48,7 +48,7 @@ - diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 43d54f64c43..f8367ce01f8 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -126,11 +126,11 @@ - + - - + + @@ -251,7 +251,7 @@
    - + #{dsf.datasetFieldType.localeTitle}

    -
    diff --git a/src/main/webapp/template.xhtml b/src/main/webapp/template.xhtml index 280d5ed05b9..12dca612f1b 100644 --- a/src/main/webapp/template.xhtml +++ b/src/main/webapp/template.xhtml @@ -114,7 +114,7 @@
    - diff --git a/src/main/webapp/themeAndWidgetsFragment.xhtml b/src/main/webapp/themeAndWidgetsFragment.xhtml index 510b0d63981..ee6aaf53bd5 100644 --- a/src/main/webapp/themeAndWidgetsFragment.xhtml +++ b/src/main/webapp/themeAndWidgetsFragment.xhtml @@ -104,6 +104,31 @@
    +
    + +
    +

    #{bundle['dataverse.theme.logo.tip']}

    + + +

    + #{of:format1(bundle['alt.logo'], themeWidgetFragment.editDv.name)} + #{of:format1(bundle['alt.logo'], themeWidgetFragment.editDv.name)} +

    + +     + +     + + + +
    +
    • @@ -181,7 +206,7 @@
    - https://doi.org/" + doi + ", Root, V1"; + + getCitationAfter.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", is(expectedCitation)); + + Response exportDatasetAsDublinCore = UtilIT.exportDataset(datasetPid, "oai_dc", apiToken, true); + exportDatasetAsDublinCore.prettyPrint(); + exportDatasetAsDublinCore.then().assertThat() + .body("oai_dc.type", equalTo("Dataset")) + .body("oai_dc.date", equalTo("1999-12-31")) + .statusCode(OK.getStatusCode()); + + Response clearDateField = UtilIT.clearDatasetCitationDateField(datasetPid, apiToken); + clearDateField.prettyPrint(); + clearDateField.then().assertThat().statusCode(OK.getStatusCode()); + + // Clearing not enough. You have to reexport because the previous date is cached. + Response rexport = UtilIT.reexportDatasetAllFormats(datasetPid); + rexport.prettyPrint(); + rexport.then().assertThat().statusCode(OK.getStatusCode()); + + String todayDate = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + Response exportPostClear = UtilIT.exportDataset(datasetPid, "oai_dc", apiToken, true); + exportPostClear.prettyPrint(); + exportPostClear.then().assertThat() + .body("oai_dc.type", equalTo("Dataset")) + .body("oai_dc.date", equalTo(todayDate)) + .statusCode(OK.getStatusCode()); + } + @Test public void getVersionFiles() throws IOException, InterruptedException { Response createUser = UtilIT.createRandomUser(); @@ -4857,4 +5168,134 @@ public void testGetCanDownloadAtLeastOneFile() { Response getUserPermissionsOnDatasetInvalidIdResponse = UtilIT.getCanDownloadAtLeastOneFile("testInvalidId", DS_VERSION_LATEST, secondUserApiToken); getUserPermissionsOnDatasetInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); } + + @Test + public void testCompareDatasetVersionsAPI() throws InterruptedException { + + Response createUser = UtilIT.createRandomUser(); + assertEquals(200, createUser.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + Response makeSuperUser = UtilIT.makeSuperUser(username); + assertEquals(200, makeSuperUser.getStatusCode()); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + Response getDatasetJsonBeforePublishing = UtilIT.nativeGet(datasetId, apiToken); + String protocol = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.protocol"); + String authority = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.authority"); + String identifier = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.identifier"); + String datasetPersistentId = protocol + ":" + authority + "/" + identifier; + // used for all added files + JsonObjectBuilder json = Json.createObjectBuilder() + .add("description", "my description") + .add("directoryLabel", "/data/subdir1/") + .add("categories", Json.createArrayBuilder() + .add("Data") + ); + JsonObject jsonObj = json.build(); + String pathToFile = "src/main/webapp/resources/images/dataverse-icon-1200.png"; + Response uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, jsonObj, apiToken); + uploadResponse.prettyPrint(); + uploadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Integer modifyFileId = UtilIT.getDataFileIdFromResponse(uploadResponse); + pathToFile = "src/main/webapp/resources/images/dataverseproject_logo.jpg"; + uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, jsonObj, apiToken); + uploadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Integer deleteFileId = UtilIT.getDataFileIdFromResponse(uploadResponse); + + pathToFile = "src/main/webapp/resources/images/fav/favicon-16x16.png"; + uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, jsonObj, apiToken); + uploadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Integer replaceFileId = UtilIT.getDataFileIdFromResponse(uploadResponse); + + Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken); + assertEquals(200, publishDataverse.getStatusCode()); + + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken); + assertEquals(200, publishDataset.getStatusCode()); + + // post publish update to create DRAFT version + String pathToJsonFilePostPub = "doc/sphinx-guides/source/_static/api/dataset-add-metadata-after-pub.json"; + Response addDataToPublishedVersion = UtilIT.addDatasetMetadataViaNative(datasetPersistentId, pathToJsonFilePostPub, apiToken); + addDataToPublishedVersion.then().assertThat().statusCode(OK.getStatusCode()); + + // Test adding a file + pathToFile = "src/test/resources/tab/test.tab"; + Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToFile, jsonObj, apiToken); + uploadTabularFileResponse.prettyPrint(); + uploadTabularFileResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Integer addedFileId = UtilIT.getDataFileIdFromResponse(uploadTabularFileResponse); + + // Ensure tabular file is ingested + sleep(2000); + + String tabularTagName = "Survey"; + Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(String.valueOf(addedFileId), apiToken, List.of(tabularTagName)); + setFileTabularTagsResponse.prettyPrint(); + setFileTabularTagsResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Test removing a file + uploadResponse = UtilIT.deleteFile(deleteFileId, apiToken); + uploadResponse.prettyPrint(); + uploadResponse.then().assertThat() + .statusCode(NO_CONTENT.getStatusCode()); + + // Test Replacing a file + Response replaceResponse = UtilIT.replaceFile(String.valueOf(replaceFileId), "src/main/webapp/resources/images/fav/favicon-32x32.png", jsonObj, apiToken); + replaceResponse.prettyPrint(); + replaceResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Test modify by restricting the file + Response restrictResponse = UtilIT.restrictFile(modifyFileId.toString(), true, apiToken); + restrictResponse.prettyPrint(); + restrictResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Also test a terms of access change + String jsonLDTerms = "{\"https://dataverse.org/schema/core#fileTermsOfAccess\":{\"https://dataverse.org/schema/core#dataAccessPlace\":\"Somewhere\"}}"; + Response updateTerms = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, jsonLDTerms, true); + updateTerms.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response compareResponse = UtilIT.compareDatasetVersions(datasetPersistentId, ":latest-published", ":draft", apiToken); + compareResponse.prettyPrint(); + compareResponse.then().assertThat() + .body("data.oldVersion.versionNumber", CoreMatchers.equalTo("1.0")) + .body("data.newVersion.versionNumber", CoreMatchers.equalTo("DRAFT")) + .body("data.metadataChanges[0].blockName", CoreMatchers.equalTo("Citation Metadata")) + .body("data.metadataChanges[0].changed[0].fieldName", CoreMatchers.equalTo("Author")) + .body("data.metadataChanges[0].changed[0].oldValue", CoreMatchers.containsString("Finch, Fiona; (Birds Inc.)")) + .body("data.metadataChanges[1].blockName", CoreMatchers.equalTo("Life Sciences Metadata")) + .body("data.metadataChanges[1].changed[0].fieldName", CoreMatchers.equalTo("Design Type")) + .body("data.metadataChanges[1].changed[0].oldValue", CoreMatchers.containsString("")) + .body("data.metadataChanges[1].changed[0].newValue", CoreMatchers.containsString("Parallel Group Design; Nested Case Control Design")) + .body("data.filesAdded[0].fileName", CoreMatchers.equalTo("test.tab")) + .body("data.filesAdded[0].filePath", CoreMatchers.equalTo("data/subdir1")) + .body("data.filesAdded[0].description", CoreMatchers.equalTo("my description")) + .body("data.filesAdded[0].tags[0]", CoreMatchers.equalTo("Survey")) + .body("data.filesRemoved[0].fileName", CoreMatchers.equalTo("dataverseproject_logo.jpg")) + .body("data.fileChanges[0].fileName", CoreMatchers.equalTo("dataverse-icon-1200.png")) + .body("data.fileChanges[0].changed[0].newValue", CoreMatchers.equalTo("true")) + .body("data.filesReplaced[0].oldFile.fileName", CoreMatchers.equalTo("favicon-16x16.png")) + .body("data.filesReplaced[0].newFile.fileName", CoreMatchers.equalTo("favicon-32x32.png")) + .body("data.TermsOfAccess", CoreMatchers.notNullValue()) + .statusCode(OK.getStatusCode()); + + compareResponse = UtilIT.compareDatasetVersions(datasetPersistentId, ":draft", ":latest-published", apiToken); + compareResponse.prettyPrint(); + compareResponse.then().assertThat() + .body("message", CoreMatchers.equalTo(BundleUtil.getStringFromBundle("dataset.version.compare.incorrect.order"))) + .statusCode(BAD_REQUEST.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 79cc46cfa79..13c4c30190b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1,12 +1,15 @@ package edu.harvard.iq.dataverse.api; import io.restassured.RestAssured; + import static io.restassured.RestAssured.given; import static io.restassured.path.json.JsonPath.with; + import io.restassured.response.Response; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; + import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; @@ -14,6 +17,7 @@ import java.util.Arrays; import java.util.List; import java.util.logging.Logger; + import jakarta.json.Json; import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; @@ -25,12 +29,13 @@ import static jakarta.ws.rs.core.Response.Status.*; import static org.hamcrest.CoreMatchers.*; -import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.hasItemInArray; import static org.junit.jupiter.api.Assertions.*; import java.nio.file.Files; + import io.restassured.path.json.JsonPath; import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; @@ -43,7 +48,7 @@ public class DataversesIT { public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); } - + @AfterAll public static void afterClass() { Response removeExcludeEmail = UtilIT.deleteSetting(SettingsServiceBean.Key.ExcludeEmailFromExport); @@ -130,14 +135,16 @@ public void testDataverseCategory() { public void testMinimalDataverse() throws FileNotFoundException { Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); - String username = UtilIT.getUsernameFromResponse(createUser); String apiToken = UtilIT.getApiTokenFromResponse(createUser); JsonObject dvJson; FileReader reader = new FileReader("doc/sphinx-guides/source/_static/api/dataverse-minimal.json"); dvJson = Json.createReader(reader).readObject(); Response create = UtilIT.createDataverse(dvJson, apiToken); create.prettyPrint(); - create.then().assertThat().statusCode(CREATED.getStatusCode()); + create.then().assertThat() + .body("data.isMetadataBlockRoot", equalTo(false)) + .body("data.isFacetRoot", equalTo(false)) + .statusCode(CREATED.getStatusCode()); Response deleteDataverse = UtilIT.deleteDataverse("science", apiToken); deleteDataverse.prettyPrint(); deleteDataverse.then().assertThat().statusCode(OK.getStatusCode()); @@ -646,10 +653,182 @@ public void testImportDDI() throws IOException, InterruptedException { Response deleteUserResponse = UtilIT.deleteUser(username); assertEquals(200, deleteUserResponse.getStatusCode()); } - + @Test - public void testAttributesApi() throws Exception { + public void testImport() throws IOException, InterruptedException { + Response createUser = UtilIT.createRandomUser(); + String username = UtilIT.getUsernameFromResponse(createUser); + Response makeSuperUser = UtilIT.makeSuperUser(username); + assertEquals(200, makeSuperUser.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + assertEquals(200, publishDataverse.getStatusCode()); + + JsonObjectBuilder datasetJson = Json.createObjectBuilder() + .add("datasetVersion", Json.createObjectBuilder() + .add("license", Json.createObjectBuilder() + .add("name", "CC0 1.0") + ) + .add("metadataBlocks", Json.createObjectBuilder() + .add("citation", Json.createObjectBuilder() + .add("fields", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("typeName", "title") + .add("value", "Test Dataset") + .add("typeClass", "primitive") + .add("multiple", false) + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("authorName", + Json.createObjectBuilder() + .add("value", "Simpson, Homer") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "authorName")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "author") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetContactEmail", + Json.createObjectBuilder() + .add("value", "hsimpson@mailinator.com") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "datasetContactEmail")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "datasetContact") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("dsDescriptionValue", + Json.createObjectBuilder() + .add("value", "This a test dataset.") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "dsDescriptionValue")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "dsDescription") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add("Other") + ) + .add("typeClass", "controlledVocabulary") + .add("multiple", true) + .add("typeName", "subject") + ) + ) + ) + )); + + String json = datasetJson.build().toString(); + + Response importJSONNoPid = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, null, "no"); + logger.info(importJSONNoPid.prettyPrint()); + assertEquals(400, importJSONNoPid.getStatusCode()); + + String body = importJSONNoPid.getBody().asString(); + String status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + String message = JsonPath.from(body).getString("message"); + assertEquals( + "Please provide a persistent identifier, either by including it in the JSON, or by using the pid query parameter.", + message + ); + + Response importJSONNoPidRelease = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, null, "yes"); + logger.info( importJSONNoPidRelease.prettyPrint()); + assertEquals(400, importJSONNoPidRelease.getStatusCode()); + + body = importJSONNoPidRelease.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + message = JsonPath.from(body).getString("message"); + assertEquals( + "Please provide a persistent identifier, either by including it in the JSON, or by using the pid query parameter.", + message + ); + + Response importJSONUnmanagedPid = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, "doi:10.5073/FK2/ABCD11", "no"); + logger.info(importJSONUnmanagedPid.prettyPrint()); + assertEquals(400, importJSONUnmanagedPid.getStatusCode()); + + body = importJSONUnmanagedPid.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + message = JsonPath.from(body).getString("message"); + assertEquals( + "Cannot import a dataset that has a PID that doesn't match the server's settings", + message + ); + + // Under normal conditions, you shouldn't need to destroy these datasets. + // Uncomment if they're still around from a previous failed run. +// Response destroy1 = UtilIT.destroyDataset("doi:10.5072/FK2/ABCD11", apiToken); +// destroy1.prettyPrint(); +// Response destroy2 = UtilIT.destroyDataset("doi:10.5072/FK2/ABCD22", apiToken); +// destroy2.prettyPrint(); + + Response importJSONPid = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, "doi:10.5072/FK2/ABCD11", "no"); + logger.info(importJSONPid.prettyPrint()); + assertEquals(201, importJSONPid.getStatusCode()); + + Response importJSONPidRel = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, "doi:10.5072/FK2/ABCD22", "yes"); + logger.info(importJSONPidRel.prettyPrint()); + assertEquals(201, importJSONPidRel.getStatusCode()); + + Integer datasetIdInt = JsonPath.from(importJSONPid.body().asString()).getInt("data.id"); + + Response search1 = UtilIT.search("id:dataset_" + datasetIdInt + "_draft", apiToken); // santity check, can find it + search1.prettyPrint(); + search1.then().assertThat() + .body("data.total_count", CoreMatchers.is(1)) + .body("data.count_in_response", CoreMatchers.is(1)) + .body("data.items[0].name", CoreMatchers.is("Test Dataset")) + .statusCode(OK.getStatusCode()); + + //cleanup + + Response destroyDatasetResponse = UtilIT.destroyDataset(datasetIdInt, apiToken); + assertEquals(200, destroyDatasetResponse.getStatusCode()); + + Integer datasetIdIntPidRel = JsonPath.from(importJSONPidRel.body().asString()).getInt("data.id"); + Response destroyDatasetResponsePidRel = UtilIT.destroyDataset(datasetIdIntPidRel, apiToken); + assertEquals(200, destroyDatasetResponsePidRel.getStatusCode()); + + UtilIT.sleepForDeadlock(UtilIT.MAXIMUM_IMPORT_DURATION); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(username); + assertEquals(200, deleteUserResponse.getStatusCode()); + } + + @Test + public void testAttributesApi() { Response createUser = UtilIT.createRandomUser(); String apiToken = UtilIT.getApiTokenFromResponse(createUser); @@ -664,30 +843,70 @@ public void testAttributesApi() throws Exception { String collectionAlias = UtilIT.getAliasFromResponse(createDataverseResponse); String newCollectionAlias = collectionAlias + "RENAMED"; - - // Change the alias of the collection: - - Response changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "alias", newCollectionAlias, apiToken); - changeAttributeResp.prettyPrint(); - + + // Change the name of the collection: + + String newCollectionName = "Renamed Name"; + Response changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "name", newCollectionName, apiToken); changeAttributeResp.then().assertThat() .statusCode(OK.getStatusCode()) .body("message.message", equalTo("Update successful")); - - // Check on the collection, under the new alias: - + + // Change the description of the collection: + + String newDescription = "Renamed Description"; + changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "description", newDescription, apiToken); + changeAttributeResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("message.message", equalTo("Update successful")); + + // Change the affiliation of the collection: + + String newAffiliation = "Renamed Affiliation"; + changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "affiliation", newAffiliation, apiToken); + changeAttributeResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("message.message", equalTo("Update successful")); + + // Cannot update filePIDsEnabled from a regular user: + + changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "filePIDsEnabled", "true", apiToken); + changeAttributeResp.then().assertThat() + .statusCode(UNAUTHORIZED.getStatusCode()); + + // Change the alias of the collection: + + changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "alias", newCollectionAlias, apiToken); + changeAttributeResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("message.message", equalTo("Update successful")); + + // Check on the collection, under the new alias: + Response collectionInfoResponse = UtilIT.exportDataverse(newCollectionAlias, apiToken); - collectionInfoResponse.prettyPrint(); - collectionInfoResponse.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.alias", equalTo(newCollectionAlias)); - + .body("data.alias", equalTo(newCollectionAlias)) + .body("data.name", equalTo(newCollectionName)) + .body("data.description", equalTo(newDescription)) + .body("data.affiliation", equalTo(newAffiliation)); + // Delete the collection (again, using its new alias): - + Response deleteCollectionResponse = UtilIT.deleteDataverse(newCollectionAlias, apiToken); - deleteCollectionResponse.prettyPrint(); assertEquals(OK.getStatusCode(), deleteCollectionResponse.getStatusCode()); + + // Cannot update root collection from a regular user: + + changeAttributeResp = UtilIT.setCollectionAttribute("root", "name", newCollectionName, apiToken); + changeAttributeResp.then().assertThat() + .statusCode(UNAUTHORIZED.getStatusCode()); + + collectionInfoResponse = UtilIT.exportDataverse("root", apiToken); + + collectionInfoResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.name", equalTo("Root")); } @Test @@ -699,12 +918,24 @@ public void testListMetadataBlocks() { createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + // New Dataverse should return just the citation block and its displayOnCreate fields when onlyDisplayedOnCreate=true and returnDatasetFieldTypes=true + Response listMetadataBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken); + listMetadataBlocks.prettyPrint(); + listMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode()); + listMetadataBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(1)) + .body("data[0].name", is("citation")) + .body("data[0].fields.title.displayOnCreate", equalTo(true)) + .body("data[0].fields.size()", is(10)) + .body("data[0].fields.author.childFields.size()", is(4)); + Response setMetadataBlocksResponse = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("astrophysics"), apiToken); setMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); - String[] testInputLevelNames = {"geographicCoverage", "country", "city"}; - boolean[] testRequiredInputLevels = {false, true, false}; - boolean[] testIncludedInputLevels = {false, true, true}; + String[] testInputLevelNames = {"geographicCoverage", "country", "city", "notesText"}; + boolean[] testRequiredInputLevels = {false, true, false, false}; + boolean[] testIncludedInputLevels = {false, true, true, false}; Response updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); updateDataverseInputLevelsResponse.then().assertThat().statusCode(OK.getStatusCode()); @@ -774,17 +1005,28 @@ public void testListMetadataBlocks() { // Check dataset fields for the updated input levels are retrieved int geospatialMetadataBlockIndex = actualMetadataBlockDisplayName1.equals("Geospatial Metadata") ? 0 : actualMetadataBlockDisplayName2.equals("Geospatial Metadata") ? 1 : 2; + // Since the included property of notesText is set to false, we should retrieve the total number of fields minus one + int citationMetadataBlockIndex = geospatialMetadataBlockIndex == 0 ? 1 : 0; + listMetadataBlocksResponse.then().assertThat() + .body(String.format("data[%d].fields.size()", citationMetadataBlockIndex), equalTo(34)); + // Since the included property of geographicCoverage is set to false, we should retrieve the total number of fields minus one listMetadataBlocksResponse.then().assertThat() - .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(10)); + .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(2)); + + listMetadataBlocksResponse = UtilIT.getMetadataBlock("geospatial"); - String actualMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); - String actualMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.country.name", geospatialMetadataBlockIndex)); - String actualMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.city.name", geospatialMetadataBlockIndex)); + String actualGeospatialMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data.fields['geographicCoverage'].name")); + String actualGeospatialMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data.fields['geographicCoverage'].childFields['country'].name")); + String actualGeospatialMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data.fields['geographicCoverage'].childFields['city'].name")); + + listMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()) + .body("data.fields['geographicCoverage'].childFields.size()", equalTo(4)) + .body("data.fields['geographicBoundingBox'].childFields.size()", equalTo(4)); - assertNull(actualMetadataField1); - assertNotNull(actualMetadataField2); - assertNotNull(actualMetadataField3); + assertNotNull(actualGeospatialMetadataField1); + assertNotNull(actualGeospatialMetadataField2); + assertNotNull(actualGeospatialMetadataField3); // Existent dataverse and onlyDisplayedOnCreate=true and returnDatasetFieldTypes=true listMetadataBlocksResponse = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken); @@ -805,15 +1047,29 @@ public void testListMetadataBlocks() { geospatialMetadataBlockIndex = actualMetadataBlockDisplayName2.equals("Geospatial Metadata") ? 1 : 0; listMetadataBlocksResponse.then().assertThat() - .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(1)); + .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(0)); - actualMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); - actualMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.country.name", geospatialMetadataBlockIndex)); - actualMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.city.name", geospatialMetadataBlockIndex)); +// actualGeospatialMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); +// actualGeospatialMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.childFields['country'].name", geospatialMetadataBlockIndex)); +// actualGeospatialMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.childFields['city'].name", geospatialMetadataBlockIndex)); - assertNull(actualMetadataField1); - assertNotNull(actualMetadataField2); - assertNull(actualMetadataField3); +// assertNull(actualGeospatialMetadataField1); +// assertNotNull(actualGeospatialMetadataField2); +// assertNull(actualGeospatialMetadataField3); + + citationMetadataBlockIndex = geospatialMetadataBlockIndex == 0 ? 1 : 0; + + // notesText has displayOnCreate=true but has include=false, so should not be retrieved + String notesTextCitationMetadataField = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.notesText.name", citationMetadataBlockIndex)); + assertNotNull(notesTextCitationMetadataField); + + // producerName is a conditionally required field, so should not be retrieved + String producerNameCitationMetadataField = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.producerName.name", citationMetadataBlockIndex)); + assertNull(producerNameCitationMetadataField); + + // author is a required field, so should be retrieved + String authorCitationMetadataField = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.author.name", citationMetadataBlockIndex)); + assertNotNull(authorCitationMetadataField); // User has no permissions on the requested dataverse Response createSecondUserResponse = UtilIT.createRandomUser(); @@ -825,6 +1081,25 @@ public void testListMetadataBlocks() { listMetadataBlocksResponse = UtilIT.listMetadataBlocks(secondDataverseAlias, true, true, apiToken); listMetadataBlocksResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); + + // List metadata blocks from Root + listMetadataBlocksResponse = UtilIT.listMetadataBlocks("root", true, true, apiToken); + listMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); + listMetadataBlocksResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].displayName", equalTo("Citation Metadata")) + .body("data[0].fields", not(equalTo(null))) + .body("data.size()", equalTo(1)); + + // Checking child / parent logic + listMetadataBlocksResponse = UtilIT.getMetadataBlock("citation"); + listMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); + listMetadataBlocksResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.displayName", equalTo("Citation Metadata")) + .body("data.fields", not(equalTo(null))) + .body("data.fields.otherIdAgency", equalTo(null)) + .body("data.fields.otherId.childFields.size()", equalTo(2)); } @Test @@ -941,6 +1216,13 @@ public void testUpdateInputLevels() { .body("message", equalTo("Invalid dataset field type name: invalid1")) .statusCode(BAD_REQUEST.getStatusCode()); + // Update input levels with invalid configuration (field required but not included) + testIncludedInputLevels = new boolean[]{false, false}; + updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); + updateDataverseInputLevelsResponse.then().assertThat() + .body("message", equalTo(BundleUtil.getStringFromBundle("dataverse.inputlevels.error.cannotberequiredifnotincluded", List.of("geographicCoverage")))) + .statusCode(BAD_REQUEST.getStatusCode()); + // Update invalid empty input levels testInputLevelNames = new String[]{}; updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); @@ -948,4 +1230,350 @@ public void testUpdateInputLevels() { .body("message", equalTo("Error while updating dataverse input levels: Input level list cannot be null or empty")) .statusCode(INTERNAL_SERVER_ERROR.getStatusCode()); } + + @Test + public void testAddDataverse() { + Response createUser = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + String testAliasSuffix = "-add-dataverse"; + + // Without optional input levels and facet ids + String testDataverseAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; + Response createSubDataverseResponse = UtilIT.createSubDataverse(testDataverseAlias, null, apiToken, "root"); + createSubDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + Response listMetadataBlocksResponse = UtilIT.listMetadataBlocks(testDataverseAlias, false, false, apiToken); + listMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); + String actualMetadataBlockName = listMetadataBlocksResponse.then().extract().path("data[0].name"); + assertEquals(actualMetadataBlockName, "citation"); + + // Assert root facets are configured + String[] expectedRootFacetIds = {"authorName", "subject", "keywordValue", "dateOfDeposit"}; + Response listDataverseFacetsResponse = UtilIT.listDataverseFacets(testDataverseAlias, apiToken); + List actualFacetNames = listDataverseFacetsResponse.then().extract().path("data"); + assertThat("Facet names should match expected root facet ids", actualFacetNames, containsInAnyOrder(expectedRootFacetIds)); + + // With optional input levels and facet ids + String[] testInputLevelNames = {"geographicCoverage", "country"}; + String[] testFacetIds = {"language", "contributorName"}; + String[] testMetadataBlockNames = {"citation", "geospatial"}; + testDataverseAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; + createSubDataverseResponse = UtilIT.createSubDataverse(testDataverseAlias, null, apiToken, "root", testInputLevelNames, testFacetIds, testMetadataBlockNames); + createSubDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + + // Assert custom facets are configured + listDataverseFacetsResponse = UtilIT.listDataverseFacets(testDataverseAlias, apiToken); + String actualFacetName1 = listDataverseFacetsResponse.then().extract().path("data[0]"); + String actualFacetName2 = listDataverseFacetsResponse.then().extract().path("data[1]"); + assertNotEquals(actualFacetName1, actualFacetName2); + assertThat(testFacetIds, hasItemInArray(actualFacetName1)); + assertThat(testFacetIds, hasItemInArray(actualFacetName2)); + + // Assert input levels are configured + Response listDataverseInputLevelsResponse = UtilIT.listDataverseInputLevels(testDataverseAlias, apiToken); + String actualInputLevelName1 = listDataverseInputLevelsResponse.then().extract().path("data[0].datasetFieldTypeName"); + String actualInputLevelName2 = listDataverseInputLevelsResponse.then().extract().path("data[1].datasetFieldTypeName"); + assertNotEquals(actualFacetName1, actualFacetName2); + assertThat(testInputLevelNames, hasItemInArray(actualInputLevelName1)); + assertThat(testInputLevelNames, hasItemInArray(actualInputLevelName2)); + + // Assert metadata blocks are configured + listMetadataBlocksResponse = UtilIT.listMetadataBlocks(testDataverseAlias, false, false, apiToken); + listMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); + String actualMetadataBlockName1 = listMetadataBlocksResponse.then().extract().path("data[0].name"); + String actualMetadataBlockName2 = listMetadataBlocksResponse.then().extract().path("data[1].name"); + assertNotEquals(actualMetadataBlockName1, actualMetadataBlockName2); + assertThat(testMetadataBlockNames, hasItemInArray(actualMetadataBlockName1)); + assertThat(testMetadataBlockNames, hasItemInArray(actualMetadataBlockName2)); + + // Setting metadata blocks without citation + testDataverseAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; + String[] testMetadataBlockNamesWithoutCitation = {"geospatial"}; + createSubDataverseResponse = UtilIT.createSubDataverse(testDataverseAlias, null, apiToken, "root", null, null, testMetadataBlockNamesWithoutCitation); + createSubDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + + // Assert metadata blocks including citation are configured + String[] testExpectedBlockNames = {"citation", "geospatial"}; + actualMetadataBlockName1 = listMetadataBlocksResponse.then().extract().path("data[0].name"); + actualMetadataBlockName2 = listMetadataBlocksResponse.then().extract().path("data[1].name"); + assertNotEquals(actualMetadataBlockName1, actualMetadataBlockName2); + assertThat(testExpectedBlockNames, hasItemInArray(actualMetadataBlockName1)); + assertThat(testExpectedBlockNames, hasItemInArray(actualMetadataBlockName2)); + + // Should return error when an invalid facet id is sent + String invalidFacetId = "invalidFacetId"; + String[] testInvalidFacetIds = {"authorName", invalidFacetId}; + testDataverseAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; + createSubDataverseResponse = UtilIT.createSubDataverse(testDataverseAlias, null, apiToken, "root", testInputLevelNames, testInvalidFacetIds, testMetadataBlockNames); + createSubDataverseResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Cant find dataset field type \"" + invalidFacetId + "\"")); + + // Should return error when an invalid input level is sent + String invalidInputLevelName = "wrongInputLevel"; + String[] testInvalidInputLevelNames = {"geographicCoverage", invalidInputLevelName}; + testDataverseAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; + createSubDataverseResponse = UtilIT.createSubDataverse(testDataverseAlias, null, apiToken, "root", testInvalidInputLevelNames, testFacetIds, testMetadataBlockNames); + createSubDataverseResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Invalid dataset field type name: " + invalidInputLevelName)); + + // Should return error when an invalid metadata block name is sent + String invalidMetadataBlockName = "invalidMetadataBlockName"; + String[] testInvalidMetadataBlockNames = {"citation", invalidMetadataBlockName}; + testDataverseAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; + createSubDataverseResponse = UtilIT.createSubDataverse(testDataverseAlias, null, apiToken, "root", testInputLevelNames, testInvalidFacetIds, testInvalidMetadataBlockNames); + createSubDataverseResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Invalid metadata block name: \"" + invalidMetadataBlockName + "\"")); + } + + @Test + public void testUpdateDataverse() { + Response createUser = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + String testAliasSuffix = "-update-dataverse"; + + String testDataverseAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; + Response createSubDataverseResponse = UtilIT.createSubDataverse(testDataverseAlias, null, apiToken, "root"); + createSubDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + + String newAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; + String newName = "New Test Dataverse Name"; + String newAffiliation = "New Test Dataverse Affiliation"; + String newDataverseType = Dataverse.DataverseType.TEACHING_COURSES.toString(); + String[] newContactEmails = new String[]{"new_email@dataverse.com"}; + String[] newInputLevelNames = new String[]{"geographicCoverage"}; + String[] newFacetIds = new String[]{"contributorName"}; + String[] newMetadataBlockNames = new String[]{"citation", "geospatial", "biomedical"}; + + Response updateDataverseResponse = UtilIT.updateDataverse( + testDataverseAlias, + newAlias, + newName, + newAffiliation, + newDataverseType, + newContactEmails, + newInputLevelNames, + newFacetIds, + newMetadataBlockNames, + apiToken + ); + + // Assert dataverse properties are updated + updateDataverseResponse.then().assertThat().statusCode(OK.getStatusCode()); + String actualDataverseAlias = updateDataverseResponse.then().extract().path("data.alias"); + assertEquals(newAlias, actualDataverseAlias); + String actualDataverseName = updateDataverseResponse.then().extract().path("data.name"); + assertEquals(newName, actualDataverseName); + String actualDataverseAffiliation = updateDataverseResponse.then().extract().path("data.affiliation"); + assertEquals(newAffiliation, actualDataverseAffiliation); + String actualDataverseType = updateDataverseResponse.then().extract().path("data.dataverseType"); + assertEquals(newDataverseType, actualDataverseType); + String actualContactEmail = updateDataverseResponse.then().extract().path("data.dataverseContacts[0].contactEmail"); + assertEquals("new_email@dataverse.com", actualContactEmail); + + // Assert metadata blocks are updated + Response listMetadataBlocksResponse = UtilIT.listMetadataBlocks(newAlias, false, false, apiToken); + String actualDataverseMetadataBlock1 = listMetadataBlocksResponse.then().extract().path("data[0].name"); + String actualDataverseMetadataBlock2 = listMetadataBlocksResponse.then().extract().path("data[1].name"); + String actualDataverseMetadataBlock3 = listMetadataBlocksResponse.then().extract().path("data[2].name"); + assertThat(newMetadataBlockNames, hasItemInArray(actualDataverseMetadataBlock1)); + assertThat(newMetadataBlockNames, hasItemInArray(actualDataverseMetadataBlock2)); + assertThat(newMetadataBlockNames, hasItemInArray(actualDataverseMetadataBlock3)); + + // Assert custom facets are updated + Response listDataverseFacetsResponse = UtilIT.listDataverseFacets(newAlias, apiToken); + String actualFacetName = listDataverseFacetsResponse.then().extract().path("data[0]"); + assertThat(newFacetIds, hasItemInArray(actualFacetName)); + + // Assert input levels are updated + Response listDataverseInputLevelsResponse = UtilIT.listDataverseInputLevels(newAlias, apiToken); + String actualInputLevelName = listDataverseInputLevelsResponse.then().extract().path("data[0].datasetFieldTypeName"); + assertThat(newInputLevelNames, hasItemInArray(actualInputLevelName)); + + // The alias has been changed, so we should not be able to do any operation using the old one + String oldDataverseAlias = testDataverseAlias; + Response getDataverseResponse = UtilIT.listDataverseFacets(oldDataverseAlias, apiToken); + getDataverseResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); + + // Update the dataverse without setting metadata blocks, facets, or input levels + updateDataverseResponse = UtilIT.updateDataverse( + newAlias, + newAlias, + newName, + newAffiliation, + newDataverseType, + newContactEmails, + null, + null, + null, + apiToken + ); + updateDataverseResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Assert that the metadata blocks are inherited from the parent + listMetadataBlocksResponse = UtilIT.listMetadataBlocks(newAlias, false, false, apiToken); + listMetadataBlocksResponse + .then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(1)) + .body("data[0].name", equalTo("citation")); + + // Assert that the facets are inherited from the parent + String[] rootFacetIds = new String[]{"authorName", "subject", "keywordValue", "dateOfDeposit"}; + listDataverseFacetsResponse = UtilIT.listDataverseFacets(newAlias, apiToken); + String actualFacetName1 = listDataverseFacetsResponse.then().extract().path("data[0]"); + String actualFacetName2 = listDataverseFacetsResponse.then().extract().path("data[1]"); + String actualFacetName3 = listDataverseFacetsResponse.then().extract().path("data[2]"); + String actualFacetName4 = listDataverseFacetsResponse.then().extract().path("data[3]"); + assertThat(rootFacetIds, hasItemInArray(actualFacetName1)); + assertThat(rootFacetIds, hasItemInArray(actualFacetName2)); + assertThat(rootFacetIds, hasItemInArray(actualFacetName3)); + assertThat(rootFacetIds, hasItemInArray(actualFacetName4)); + + // Assert that the dataverse should not have any input level + listDataverseInputLevelsResponse = UtilIT.listDataverseInputLevels(newAlias, apiToken); + listDataverseInputLevelsResponse + .then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(0)); + + // Should return error when the dataverse to edit does not exist + updateDataverseResponse = UtilIT.updateDataverse( + "unexistingDataverseAlias", + newAlias, + newName, + newAffiliation, + newDataverseType, + newContactEmails, + newInputLevelNames, + newFacetIds, + newMetadataBlockNames, + apiToken + ); + updateDataverseResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); + + // User with unprivileged API token cannot update Root dataverse + updateDataverseResponse = UtilIT.updateDataverse( + "root", + newAlias, + newName, + newAffiliation, + newDataverseType, + newContactEmails, + newInputLevelNames, + newFacetIds, + newMetadataBlockNames, + apiToken + ); + updateDataverseResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); + + Response rootCollectionInfoResponse = UtilIT.exportDataverse("root", apiToken); + rootCollectionInfoResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.name", equalTo("Root")); + } + + @Test + public void testListFacets() { + Response createUserResponse = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + String[] expectedFacetNames = {"authorName", "subject", "keywordValue", "dateOfDeposit"}; + + // returnDetails is false + Response listFacetsResponse = UtilIT.listDataverseFacets(dataverseAlias, false, apiToken); + listFacetsResponse.then().assertThat().statusCode(OK.getStatusCode()); + String actualFacetName = listFacetsResponse.then().extract().path("data[0]"); + assertThat(expectedFacetNames, hasItemInArray(actualFacetName)); + + // returnDetails is true + String[] expectedDisplayNames = {"Author Name", "Subject", "Keyword Term", "Deposit Date"}; + listFacetsResponse = UtilIT.listDataverseFacets(dataverseAlias, true, apiToken); + listFacetsResponse.then().assertThat().statusCode(OK.getStatusCode()); + actualFacetName = listFacetsResponse.then().extract().path("data[0].name"); + assertThat(expectedFacetNames, hasItemInArray(actualFacetName)); + String actualDisplayName = listFacetsResponse.then().extract().path("data[0].displayName"); + assertThat(expectedDisplayNames, hasItemInArray(actualDisplayName)); + String actualId = listFacetsResponse.then().extract().path("data[0].id"); + assertNotNull(actualId); + + // Dataverse with custom facets + String dataverseWithCustomFacetsAlias = UtilIT.getRandomDvAlias() + "customFacets"; + + String[] testFacetNames = {"authorName", "authorAffiliation"}; + String[] testMetadataBlockNames = {"citation", "geospatial"}; + + Response createSubDataverseResponse = UtilIT.createSubDataverse(dataverseWithCustomFacetsAlias, null, apiToken, "root", null, testFacetNames, testMetadataBlockNames); + createSubDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + + listFacetsResponse = UtilIT.listDataverseFacets(dataverseWithCustomFacetsAlias, true, apiToken); + listFacetsResponse.then().assertThat().statusCode(OK.getStatusCode()); + + String actualFacetName1 = listFacetsResponse.then().extract().path("data[0].name"); + String actualFacetName2 = listFacetsResponse.then().extract().path("data[1].name"); + assertNotEquals(actualFacetName1, actualFacetName2); + assertThat(testFacetNames, hasItemInArray(actualFacetName1)); + assertThat(testFacetNames, hasItemInArray(actualFacetName2)); + + String[] testFacetExpectedDisplayNames = {"Author Name", "Author Affiliation"}; + String actualFacetDisplayName1 = listFacetsResponse.then().extract().path("data[0].displayName"); + String actualFacetDisplayName2 = listFacetsResponse.then().extract().path("data[1].displayName"); + assertNotEquals(actualFacetDisplayName1, actualFacetDisplayName2); + assertThat(testFacetExpectedDisplayNames, hasItemInArray(actualFacetDisplayName1)); + assertThat(testFacetExpectedDisplayNames, hasItemInArray(actualFacetDisplayName2)); + } + + @Test + public void testGetUserPermissionsOnDataverse() { + Response createUserResponse = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // Call for dataverse created by the user + Response getUserPermissionsOnDataverseResponse = UtilIT.getUserPermissionsOnDataverse(dataverseAlias, apiToken); + getUserPermissionsOnDataverseResponse.then().assertThat().statusCode(OK.getStatusCode()); + boolean canAddDataverse = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canAddDataverse"); + assertTrue(canAddDataverse); + boolean canAddDataset = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canAddDataset"); + assertTrue(canAddDataset); + boolean canViewUnpublishedDataverse = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canViewUnpublishedDataverse"); + assertTrue(canViewUnpublishedDataverse); + boolean canEditDataverse = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canEditDataverse"); + assertTrue(canEditDataverse); + boolean canManageDataversePermissions = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canManageDataversePermissions"); + assertTrue(canManageDataversePermissions); + boolean canPublishDataverse = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canPublishDataverse"); + assertTrue(canPublishDataverse); + boolean canDeleteDataverse = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canDeleteDataverse"); + assertTrue(canDeleteDataverse); + + // Call for root dataverse + getUserPermissionsOnDataverseResponse = UtilIT.getUserPermissionsOnDataverse("root", apiToken); + getUserPermissionsOnDataverseResponse.then().assertThat().statusCode(OK.getStatusCode()); + canAddDataverse = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canAddDataverse"); + assertTrue(canAddDataverse); + canAddDataset = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canAddDataset"); + assertTrue(canAddDataset); + canViewUnpublishedDataverse = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canViewUnpublishedDataverse"); + assertFalse(canViewUnpublishedDataverse); + canEditDataverse = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canEditDataverse"); + assertFalse(canEditDataverse); + canManageDataversePermissions = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canManageDataversePermissions"); + assertFalse(canManageDataversePermissions); + canPublishDataverse = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canPublishDataverse"); + assertFalse(canPublishDataverse); + canDeleteDataverse = JsonPath.from(getUserPermissionsOnDataverseResponse.body().asString()).getBoolean("data.canDeleteDataverse"); + assertFalse(canDeleteDataverse); + + // Call with invalid dataverse alias + Response getUserPermissionsOnDataverseInvalidIdResponse = UtilIT.getUserPermissionsOnDataverse("testInvalidAlias", apiToken); + getUserPermissionsOnDataverseInvalidIdResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index e3c26284d55..98107eca33a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -2275,7 +2275,6 @@ public void testDeleteFile() { Response v1 = UtilIT.getDatasetVersion(datasetPid, "1.0", apiToken); v1.prettyPrint(); v1.then().assertThat() - .body("data.files[0].dataFile.filename", equalTo("cc0.png")) .statusCode(OK.getStatusCode()); Map v1files1 = with(v1.body().asString()).param("fileToFind", "cc0.png") @@ -2290,7 +2289,6 @@ public void testDeleteFile() { Response postv1draft2 = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken); postv1draft2.prettyPrint(); postv1draft2.then().assertThat() - .body("data.files[0].dataFile.filename", equalTo("orcid_16x16.png")) .statusCode(OK.getStatusCode()); Map v1files2 = with(postv1draft2.body().asString()).param("fileToFind", "orcid_16x16.png") diff --git a/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java b/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java index 5e436dd0e98..b198d2769a0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java @@ -6,11 +6,14 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static jakarta.ws.rs.core.Response.Status.OK; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; +import org.skyscreamer.jsonassert.JSONAssert; public class InfoIT { @@ -81,6 +84,22 @@ public void testGetZipDownloadLimit() { .body("data", notNullValue()); } + @Test + public void testGetExportFormats() throws IOException { + Response response = given().urlEncodingEnabled(false) + .get("/api/info/exportFormats"); + response.prettyPrint(); + response.then().assertThat().statusCode(OK.getStatusCode()); + + String actual = response.getBody().asString(); + String expected = + java.nio.file.Files.readString( + Paths.get("src/test/resources/json/export-formats.json"), + StandardCharsets.UTF_8); + JSONAssert.assertEquals(expected, actual, true); + + } + private void testSettingEndpoint(SettingsServiceBean.Key settingKey, String testSettingValue) { String endpoint = "/api/info/settings/" + settingKey; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java index 5f5a7fbc0f8..3b0b56740eb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.api; import io.restassured.RestAssured; + import io.restassured.response.Response; import org.hamcrest.CoreMatchers; import org.junit.jupiter.api.BeforeAll; @@ -9,6 +10,7 @@ import static jakarta.ws.rs.core.Response.Status.CREATED; import static jakarta.ws.rs.core.Response.Status.OK; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeFalse; @@ -42,18 +44,27 @@ void testListMetadataBlocks() { // returnDatasetFieldTypes=true listMetadataBlocksResponse = UtilIT.listMetadataBlocks(false, true); + int expectedNumberOfMetadataFields = 35; + listMetadataBlocksResponse.prettyPrint(); listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].fields", not(equalTo(null))) - .body("data.size()", equalTo(expectedDefaultNumberOfMetadataBlocks)); + .body("data[0].fields.size()", equalTo(expectedNumberOfMetadataFields)) + .body("data.size()", equalTo(expectedDefaultNumberOfMetadataBlocks)) + .body("data[1].fields.geographicCoverage.childFields.size()", is(4)) + .body("data[0].fields.publication.childFields.size()", is(5)); // onlyDisplayedOnCreate=true and returnDatasetFieldTypes=true listMetadataBlocksResponse = UtilIT.listMetadataBlocks(true, true); + listMetadataBlocksResponse.prettyPrint(); + expectedNumberOfMetadataFields = 10; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].fields", not(equalTo(null))) + .body("data[0].fields.size()", equalTo(expectedNumberOfMetadataFields)) .body("data[0].displayName", equalTo("Citation Metadata")) - .body("data.size()", equalTo(expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks)); + .body("data.size()", equalTo(expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks)) + .body("data[0].fields.author.childFields.size()", is(4)); } @Test @@ -67,7 +78,7 @@ void testGetMetadataBlock() { .body("data.fields.title.typeClass", CoreMatchers.is("primitive")) .body("data.fields.title.isRequired", CoreMatchers.is(true)); } - + @Test void testDatasetWithAllDefaultMetadata() { // given diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java index f7135ce7f3b..8951b0bd42e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java @@ -278,8 +278,8 @@ public void testMoveLinkedDataset() { .body("message", equalTo("Use the query parameter forceMove=true to complete the move. This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. ")); JsonObject linksBeforeData = Json.createReader(new StringReader(getLinksBefore.asString())).readObject(); - assertEquals("OK", linksBeforeData.getString("status")); - assertEquals(dataverse2Alias + " (id " + dataverse2Id + ")", linksBeforeData.getJsonObject("data").getJsonArray("dataverses that link to dataset id " + datasetId).getString(0)); + assertEquals(datasetId, linksBeforeData.getJsonObject("data").getInt("id")); + assertEquals(dataverse2Id, linksBeforeData.getJsonObject("data").getJsonArray("linked-dataverses").get(0).asJsonObject().getInt("id")); boolean forceMove = true; Response forceMoveLinkedDataset = UtilIT.moveDataset(datasetId.toString(), dataverse2Alias, forceMove, superuserApiToken); @@ -308,8 +308,7 @@ public void testMoveLinkedDataset() { JsonObject linksAfterData = Json.createReader(new StringReader(getLinksAfter.asString())).readObject(); assertEquals("OK", linksAfterData.getString("status")); - assertEquals(0, linksAfterData.getJsonObject("data").getJsonArray("dataverses that link to dataset id " + datasetId).size()); - + assertEquals(0, linksAfterData.getJsonObject("data").getJsonArray("linked-dataverses").size()); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SavedSearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SavedSearchIT.java new file mode 100644 index 00000000000..08ebec31cd6 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/SavedSearchIT.java @@ -0,0 +1,170 @@ +package edu.harvard.iq.dataverse.api; + +import io.restassured.RestAssured; +import io.restassured.path.json.JsonPath; +import io.restassured.response.Response; +import jakarta.json.Json; +import jakarta.json.JsonArrayBuilder; +import jakarta.json.JsonObjectBuilder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; +import static jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; +import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SavedSearchIT { + + @BeforeAll + public static void setUpClass() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + } + + @AfterAll + public static void afterClass() { + + } + + @Test + public void testSavedSearches() { + + Response createAdminUser = UtilIT.createRandomUser(); + String adminUsername = UtilIT.getUsernameFromResponse(createAdminUser); + String adminApiToken = UtilIT.getApiTokenFromResponse(createAdminUser); + UtilIT.makeSuperUser(adminUsername); + + Response createDataverseResponse = UtilIT.createRandomDataverse(adminApiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + Integer dataverseId = UtilIT.getDataverseIdFromResponse(createDataverseResponse); + + //dataset-finch1-nolicense.json + Response createDatasetResponse1 = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminApiToken); + createDatasetResponse1.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse1); + + Response createDatasetResponse2 = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminApiToken); + createDatasetResponse2.prettyPrint(); + Integer datasetId2 = UtilIT.getDatasetIdFromResponse(createDatasetResponse2); + + // missing body + Response resp = UtilIT.setSavedSearch(); + resp.prettyPrint(); + resp.then().assertThat() + .statusCode(INTERNAL_SERVER_ERROR.getStatusCode()); + + // creatorId null + resp = UtilIT.setSavedSearch(createSavedSearchJson("*", null, dataverseId, "subject_ss:Medicine, Health and Life Sciences")); + resp.prettyPrint(); + resp.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()); + + // creatorId string + resp = UtilIT.setSavedSearch(createSavedSearchJson("*", "1", dataverseId.toString(), "subject_ss:Medicine, Health and Life Sciences")); + resp.prettyPrint(); + resp.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()); + + // creatorId not found + resp = UtilIT.setSavedSearch(createSavedSearchJson("*", 9999, dataverseId, "subject_ss:Medicine, Health and Life Sciences")); + resp.prettyPrint(); + resp.then().assertThat() + .statusCode(NOT_FOUND.getStatusCode()); + + // definitionPointId null + resp = UtilIT.setSavedSearch(createSavedSearchJson("*", 1, null, "subject_ss:Medicine, Health and Life Sciences")); + resp.prettyPrint(); + resp.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()); + + // definitionPointId string + resp = UtilIT.setSavedSearch(createSavedSearchJson("*", "1", "9999", "subject_ss:Medicine, Health and Life Sciences")); + resp.prettyPrint(); + resp.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()); + + // definitionPointId not found + resp = UtilIT.setSavedSearch(createSavedSearchJson("*", 1, 9999, "subject_ss:Medicine, Health and Life Sciences")); + resp.prettyPrint(); + resp.then().assertThat() + .statusCode(NOT_FOUND.getStatusCode()); + + // missing filter + resp = UtilIT.setSavedSearch(createSavedSearchJson("*", 1, dataverseId)); + resp.prettyPrint(); + resp.then().assertThat() + .statusCode(OK.getStatusCode()); + + // create a saved search as superuser : OK + resp = UtilIT.setSavedSearch(createSavedSearchJson("*", 1, dataverseId, "subject_ss:Medicine, Health and Life Sciences")); + resp.prettyPrint(); + resp.then().assertThat() + .statusCode(OK.getStatusCode()); + + JsonPath path = JsonPath.from(resp.body().asString()); + Integer createdSavedSearchId = path.getInt("data.id"); + + // get list as non superuser : OK + Response getListReponse = UtilIT.getSavedSearchList(); + getListReponse.prettyPrint(); + getListReponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + JsonPath path2 = JsonPath.from(getListReponse.body().asString()); + List listBeforeDelete = path2.getList("data.savedSearches"); + + // makelinks/all as non superuser : OK + Response makelinksAll = UtilIT.setSavedSearchMakelinksAll(); + makelinksAll.prettyPrint(); + makelinksAll.then().assertThat() + .statusCode(OK.getStatusCode()); + + //delete a saved search as non superuser : OK + Response deleteReponse = UtilIT.deleteSavedSearchById(createdSavedSearchId); + deleteReponse.prettyPrint(); + deleteReponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // check list count minus 1 + getListReponse = UtilIT.getSavedSearchList(); + getListReponse.prettyPrint(); + JsonPath path3 = JsonPath.from(getListReponse.body().asString()); + List listAfterDelete = path3.getList("data.savedSearches"); + assertEquals(listBeforeDelete.size() - 1, listAfterDelete.size()); + } + + public String createSavedSearchJson(String query, Integer creatorId, Integer definitionPointId, String... filterQueries) { + + JsonArrayBuilder arr = Json.createArrayBuilder(); + for (String filterQuery : filterQueries) { + arr.add(filterQuery); + } + + JsonObjectBuilder json = Json.createObjectBuilder(); + if(query != null) json.add("query", query); + if(creatorId != null) json.add("creatorId", creatorId); + if(definitionPointId != null) json.add("definitionPointId", definitionPointId); + if(filterQueries.length > 0) json.add("filterQueries", arr); + return json.build().toString(); + } + + public String createSavedSearchJson(String query, String creatorId, String definitionPointId, String... filterQueries) { + + JsonArrayBuilder arr = Json.createArrayBuilder(); + for (String filterQuery : filterQueries) { + arr.add(filterQuery); + } + + JsonObjectBuilder json = Json.createObjectBuilder(); + if(query != null) json.add("query", query); + if(creatorId != null) json.add("creatorId", creatorId); + if(definitionPointId != null) json.add("definitionPointId", definitionPointId); + if(filterQueries.length > 0) json.add("filterQueries", arr); + return json.build().toString(); + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index 6e4fd5b0bb3..c97762526b0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -4,6 +4,9 @@ import io.restassured.path.json.JsonPath; import io.restassured.response.Response; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; + +import java.util.List; +import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; import jakarta.json.Json; @@ -18,22 +21,18 @@ import java.io.UnsupportedEncodingException; import java.util.Base64; import jakarta.json.JsonArray; -import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; -import static jakarta.ws.rs.core.Response.Status.OK; -import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; import org.hamcrest.CoreMatchers; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import java.awt.image.BufferedImage; import java.io.IOException; -import static java.lang.Thread.sleep; import javax.imageio.ImageIO; -import static jakarta.ws.rs.core.Response.Status.CREATED; -import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; -import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED; + import org.hamcrest.Matchers; import jakarta.json.JsonObjectBuilder; +import static jakarta.ws.rs.core.Response.Status.*; +import static java.lang.Thread.sleep; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -55,12 +54,12 @@ public static void setUpClass() { } @Test - public void testSearchPermisions() throws InterruptedException { + public void testSearchPermisions() { Response createUser1 = UtilIT.createRandomUser(); - String username1 = UtilIT.getUsernameFromResponse(createUser1); String apiToken1 = UtilIT.getApiTokenFromResponse(createUser1); + String affiliation = "testAffiliation"; - Response createDataverse1Response = UtilIT.createRandomDataverse(apiToken1); + Response createDataverse1Response = UtilIT.createRandomDataverse(apiToken1, affiliation); createDataverse1Response.prettyPrint(); assertEquals(201, createDataverse1Response.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse1Response); @@ -70,9 +69,9 @@ public void testSearchPermisions() throws InterruptedException { createDataset1Response.then().assertThat() .statusCode(CREATED.getStatusCode()); - Integer datasetId1 = UtilIT.getDatasetIdFromResponse(createDataset1Response); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset1Response); - Response shouldBeVisibleToUser1 = UtilIT.search("id:dataset_" + datasetId1 + "_draft", apiToken1); + Response shouldBeVisibleToUser1 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken1); shouldBeVisibleToUser1.prettyPrint(); shouldBeVisibleToUser1.then().assertThat() .body("data.total_count", CoreMatchers.is(1)) @@ -84,7 +83,7 @@ public void testSearchPermisions() throws InterruptedException { String username2 = UtilIT.getUsernameFromResponse(createUser2); String apiToken2 = UtilIT.getApiTokenFromResponse(createUser2); - Response shouldNotBeVisibleToUser2 = UtilIT.search("id:dataset_" + datasetId1 + "_draft", apiToken2); + Response shouldNotBeVisibleToUser2 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken2); shouldNotBeVisibleToUser2.prettyPrint(); shouldNotBeVisibleToUser2.then().assertThat() .body("data.total_count", CoreMatchers.is(0)) @@ -92,7 +91,7 @@ public void testSearchPermisions() throws InterruptedException { String nullToken = null; - Response shouldNotBeVisibleToTokenLess = UtilIT.search("id:dataset_" + datasetId1 + "_draft", nullToken); + Response shouldNotBeVisibleToTokenLess = UtilIT.search("id:dataset_" + datasetId + "_draft", nullToken); shouldNotBeVisibleToTokenLess.prettyPrint(); shouldNotBeVisibleToTokenLess.then().assertThat() .body("data.total_count", CoreMatchers.is(0)) @@ -104,15 +103,16 @@ public void testSearchPermisions() throws InterruptedException { grantUser2AccessOnDataset.prettyPrint(); assertEquals(200, grantUser2AccessOnDataset.getStatusCode()); - String searchPart = "id:dataset_" + datasetId1 + "_draft"; + String searchPart = "id:dataset_" + datasetId + "_draft"; assertTrue(UtilIT.sleepForSearch(searchPart, apiToken2, "", 1, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if search exceeds max duration " + searchPart); - Response shouldBeVisibleToUser2 = UtilIT.search("id:dataset_" + datasetId1 + "_draft", apiToken2); + Response shouldBeVisibleToUser2 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken2); shouldBeVisibleToUser2.prettyPrint(); shouldBeVisibleToUser2.then().assertThat() .body("data.total_count", CoreMatchers.is(1)) .body("data.count_in_response", CoreMatchers.is(1)) .body("data.items[0].name", CoreMatchers.is("Darwin's Finches")) + .body("data.items[0].publicationStatuses", CoreMatchers.hasItems("Unpublished", "Draft")) .statusCode(OK.getStatusCode()); Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken1); @@ -120,17 +120,16 @@ public void testSearchPermisions() throws InterruptedException { publishDataverse.then().assertThat() .statusCode(OK.getStatusCode()); - Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId1, "major", apiToken1); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken1); publishDataset.prettyPrint(); publishDataset.then().assertThat() .statusCode(OK.getStatusCode()); - Response makeSureTokenlessSearchIsEnabled = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiRequiresToken); makeSureTokenlessSearchIsEnabled.then().assertThat() .statusCode(OK.getStatusCode()); - Response publishedPublicDataShouldBeVisibleToTokenless = UtilIT.search("id:dataset_" + datasetId1, nullToken); + Response publishedPublicDataShouldBeVisibleToTokenless = UtilIT.search("id:dataset_" + datasetId, nullToken); publishedPublicDataShouldBeVisibleToTokenless.prettyPrint(); publishedPublicDataShouldBeVisibleToTokenless.then().assertThat() .body("data.total_count", CoreMatchers.is(1)) @@ -146,13 +145,15 @@ public void testSearchPermisions() throws InterruptedException { .body("data.count_in_response", CoreMatchers.is(1)) .body("data.items[0].name", CoreMatchers.is(dataverseAlias)) .body("data.items[0].type", CoreMatchers.is("dataverse")) - .body("data.items[0].identifier", CoreMatchers.is(dataverseAlias)); + .body("data.items[0].affiliation", CoreMatchers.is(affiliation)) + .body("data.items[0].parentDataverseName", CoreMatchers.is("Root")) + .body("data.items[0].parentDataverseIdentifier", CoreMatchers.is("root")); Response disableTokenlessSearch = UtilIT.setSetting(SettingsServiceBean.Key.SearchApiRequiresToken, "true"); disableTokenlessSearch.then().assertThat() .statusCode(OK.getStatusCode()); - Response dataverse47behaviorOfTokensBeingRequired = UtilIT.search("id:dataset_" + datasetId1, nullToken); + Response dataverse47behaviorOfTokensBeingRequired = UtilIT.search("id:dataset_" + datasetId, nullToken); dataverse47behaviorOfTokensBeingRequired.prettyPrint(); dataverse47behaviorOfTokensBeingRequired.then().assertThat() .body("message", CoreMatchers.equalTo(AbstractApiBean.RESPONSE_MESSAGE_AUTHENTICATED_USER_REQUIRED)) @@ -178,6 +179,7 @@ public void testSearchCitation() { Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); createDatasetResponse.prettyPrint(); Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); Response searchResponse = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken); searchResponse.prettyPrint(); @@ -188,20 +190,49 @@ public void testSearchCitation() { .body("data.items[0].citationHtml", Matchers.containsString("href")) .statusCode(200); - Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); - deleteDatasetResponse.prettyPrint(); - deleteDatasetResponse.then().assertThat() + String pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; + Response uploadImage = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadImage.prettyPrint(); + uploadImage.then().assertThat() + .statusCode(200); + + Response publishResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + publishResponse.prettyPrint(); + publishResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + publishResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + publishResponse.prettyPrint(); + publishResponse.then().assertThat() .statusCode(OK.getStatusCode()); - Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); - deleteDataverseResponse.prettyPrint(); - deleteDataverseResponse.then().assertThat() + Response updateTitleResponseAuthor = UtilIT.updateDatasetTitleViaSword(datasetPersistentId, "New Title", apiToken); + updateTitleResponseAuthor.prettyPrint(); + updateTitleResponseAuthor.then().assertThat() .statusCode(OK.getStatusCode()); - Response deleteUserResponse = UtilIT.deleteUser(username); - deleteUserResponse.prettyPrint(); - assertEquals(200, deleteUserResponse.getStatusCode()); + // search descending will get the latest 100. + // This could fail if more than 100 get created between our update and the search. Highly unlikely + searchResponse = UtilIT.search("*&type=file&sort=date&order=desc&per_page=100&start=0&subtree=root" , apiToken); + searchResponse.prettyPrint(); + int i=0; + String parentCitation = ""; + String datasetName = ""; + // most likely ours is in index 0, but it's not a guaranty. + while (i < 100) { + String dataset_persistent_id = searchResponse.body().jsonPath().getString("data.items[" + i + "].dataset_persistent_id"); + if (datasetPersistentId.equalsIgnoreCase(dataset_persistent_id)) { + parentCitation = searchResponse.body().jsonPath().getString("data.items[" + i + "].dataset_citation"); + datasetName = searchResponse.body().jsonPath().getString("data.items[" + i + "].dataset_name"); + break; + } + i++; + } + // see https://github.com/IQSS/dataverse/issues/10735 + // was showing the citation of the draft version and not the released parent + assertFalse(parentCitation.contains("New Title")); + assertTrue(parentCitation.contains(datasetName)); + assertFalse(parentCitation.contains("DRAFT")); } @Test @@ -599,7 +630,7 @@ public void testDatasetThumbnail() { String datasetLogo = "src/main/webapp/resources/images/cc0.png"; File datasetLogoFile = new File(datasetLogo); - String datasetLogoAsBase64 = datasetLogoAsBase64 = ImageThumbConverter.generateImageThumbnailFromFileAsBase64(datasetLogoFile, ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); + String datasetLogoAsBase64 = ImageThumbConverter.generateImageThumbnailFromFileAsBase64(datasetLogoFile, ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); if (datasetLogoAsBase64 == null) { Logger.getLogger(SearchIT.class.getName()).log(Level.SEVERE, "Failed to generate a base64 thumbnail from the file dataverseproject.png"); @@ -1286,4 +1317,169 @@ public void tearDownDataverse() { public static void cleanup() { } + @Test + public void testSearchFilesAndUrlImages() throws InterruptedException { + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + System.out.println("id: " + datasetId); + String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId"); + System.out.println("datasetPid: " + datasetPid); + String pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; + Response logoResponse = UtilIT.uploadDatasetLogo(datasetPid, pathToFile, apiToken); + logoResponse.prettyPrint(); + logoResponse.then().assertThat() + .statusCode(200); + + Response uploadImage = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadImage.prettyPrint(); + uploadImage.then().assertThat() + .statusCode(200); + pathToFile = "src/main/webapp/resources/js/mydata.js"; + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(200); + pathToFile = "src/test/resources/tab/test.tab"; + String searchableUniqueId = "testtab"+ UUID.randomUUID().toString().substring(0, 8); // so the search only returns 1 file + JsonObjectBuilder json = Json.createObjectBuilder() + .add("description", searchableUniqueId) + .add("restrict", "true") + .add("categories", Json.createArrayBuilder().add("Data")); + Response uploadTabFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, json.build(), apiToken); + uploadTabFile.prettyPrint(); + uploadTabFile.then().assertThat() + .statusCode(200); + // Ensure tabular file is ingested + sleep(2000); + // Set tabular tags + String tabularFileId = uploadTabFile.getBody().jsonPath().getString("data.files[0].dataFile.id"); + List testTabularTags = List.of("Survey", "Genomics"); + Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, testTabularTags); + setFileTabularTagsResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken); + publishDataverse.prettyPrint(); + publishDataverse.then().assertThat() + .statusCode(OK.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + publishDataset.prettyPrint(); + publishDataset.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response searchResp = UtilIT.search("dataverseproject", apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("file")) + .body("data.items[0].file_content_type", CoreMatchers.is("image/png")) + .body("data.items[0].url", CoreMatchers.containsString("/api/access/datafile/")) + .body("data.items[0].image_url", CoreMatchers.containsString("/api/access/datafile/")) + .body("data.items[0].image_url", CoreMatchers.containsString("imageThumb=true")); + + searchResp = UtilIT.search(dataverseAlias, apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("dataverse")) + .body("data.items[0].url", CoreMatchers.containsString("/dataverse/")) + .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("image_url"))); + + searchResp = UtilIT.search(datasetPid, apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("dataset")) + .body("data.items[0].image_url", CoreMatchers.containsString("/logo")); + + searchResp = UtilIT.search("mydata", apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("file")) + .body("data.items[0].url", CoreMatchers.containsString("/datafile/")) + .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("image_url"))); + searchResp = UtilIT.search(searchableUniqueId, apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("file")) + .body("data.items[0].url", CoreMatchers.containsString("/datafile/")) + .body("data.items[0].variables", CoreMatchers.is(3)) + .body("data.items[0].observations", CoreMatchers.is(10)) + .body("data.items[0].restricted", CoreMatchers.is(true)) + .body("data.items[0].canDownloadFile", CoreMatchers.is(true)) + .body("data.items[0].tabularTags", CoreMatchers.hasItem("Genomics")) + .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("image_url"))); + } + + @Test + public void testShowTypeCounts() { + //Create 1 user and 1 Dataverse/Collection + Response createUser = UtilIT.createRandomUser(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + String affiliation = "testAffiliation"; + + // test total_count_per_object_type is not included because the results are empty + Response searchResp = UtilIT.search(username, apiToken, "&show_type_counts=true"); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count_per_object_type", CoreMatchers.equalTo(null)); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken, affiliation); + assertEquals(201, createDataverseResponse.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // create 3 Datasets, each with 2 Datafiles + for (int i = 0; i < 3; i++) { + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse).toString(); + + // putting the dataverseAlias in the description of each file so the search q={dataverseAlias} will return dataverse, dataset, and files for this test only + String jsonAsString = "{\"description\":\"" + dataverseAlias + "\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\" }"; + + String pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; + Response uploadImage = UtilIT.uploadFileViaNative(datasetId, pathToFile, jsonAsString, apiToken); + uploadImage.then().assertThat() + .statusCode(200); + pathToFile = "src/main/webapp/resources/js/mydata.js"; + Response uploadFile = UtilIT.uploadFileViaNative(datasetId, pathToFile, jsonAsString, apiToken); + uploadFile.then().assertThat() + .statusCode(200); + + // This call forces a wait for dataset indexing to finish and gives time for file uploads to complete + UtilIT.search("id:dataset_" + datasetId, apiToken); + } + + // Test Search without show_type_counts + searchResp = UtilIT.search(dataverseAlias, apiToken); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count_per_object_type", CoreMatchers.equalTo(null)); + // Test Search with show_type_counts = FALSE + searchResp = UtilIT.search(dataverseAlias, apiToken, "&show_type_counts=false"); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count_per_object_type", CoreMatchers.equalTo(null)); + // Test Search with show_type_counts = TRUE + searchResp = UtilIT.search(dataverseAlias, apiToken, "&show_type_counts=true"); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count_per_object_type.Dataverses", CoreMatchers.is(1)) + .body("data.total_count_per_object_type.Datasets", CoreMatchers.is(3)) + .body("data.total_count_per_object_type.Files", CoreMatchers.is(6)); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java index 4df6c89411d..709908ac6eb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java @@ -462,7 +462,7 @@ public void testCreateAndDeleteDatasetInRoot() { assertNull(attemptToGetFileId); } catch (Exception ex) { System.out.println("We expect an exception here because we can no longer find the file because deleted it: " + ex); - assertTrue(ex.getClass().getName().equals(ArrayIndexOutOfBoundsException.class.getName())); + assertTrue(ex instanceof ArrayIndexOutOfBoundsException); } String newTitle = "A New Hope"; @@ -850,7 +850,7 @@ public void testDeleteFiles() { String citation = atomEntryDraftV2.body().xmlPath().getString("entry.bibliographicCitation"); logger.info("citation (should contain 'DRAFT'): " + citation); boolean draftStringFoundInCitation = citation.matches(".*DRAFT.*"); - assertEquals(true, draftStringFoundInCitation); + assertTrue(draftStringFoundInCitation); List oneFileLeftInV2Draft = statement3.getBody().xmlPath().getList("feed.entry.id"); logger.info("Number of files remaining in this post version 1 draft:" + oneFileLeftInV2Draft.size()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java index 0189ffd6e58..eb78a216626 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java @@ -1,29 +1,33 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.util.BundleUtil; import io.restassured.RestAssured; + import static io.restassured.RestAssured.given; + import io.restassured.http.ContentType; import io.restassured.path.json.JsonPath; import io.restassured.response.Response; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import java.util.ArrayList; + import java.util.Arrays; import java.util.List; import java.util.UUID; + import jakarta.json.Json; import jakarta.json.JsonObjectBuilder; -import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; -import static jakarta.ws.rs.core.Response.Status.CREATED; -import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; -import static jakarta.ws.rs.core.Response.Status.OK; -import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED; + +import static jakarta.ws.rs.core.Response.Status.*; +import static org.hamcrest.Matchers.notNullValue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.contains; import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hamcrest.CoreMatchers; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class UsersIT { @@ -371,29 +375,38 @@ public void testAPITokenEndpoints() { .body("data.message", containsString(userApiToken)) .body("data.message", containsString("expires on")); + // Recreate given a bad API token Response recreateToken = UtilIT.recreateToken("BAD-Token-blah-89234"); recreateToken.prettyPrint(); recreateToken.then().assertThat() .statusCode(UNAUTHORIZED.getStatusCode()); + // Recreate given a valid API token recreateToken = UtilIT.recreateToken(userApiToken); recreateToken.prettyPrint(); recreateToken.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.message", containsString("New token for")); + .body("data.message", containsString("New token for")) + .body("data.message", CoreMatchers.not(containsString("and expires on"))); + // Recreate given a valid API token and returning expiration createUser = UtilIT.createRandomUser(); - createUser.prettyPrint(); - assertEquals(200, createUser.getStatusCode()); + assertEquals(OK.getStatusCode(), createUser.getStatusCode()); + + userApiToken = UtilIT.getApiTokenFromResponse(createUser); + + recreateToken = UtilIT.recreateToken(userApiToken, true); + recreateToken.prettyPrint(); + recreateToken.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", containsString("New token for")) + .body("data.message", containsString("and expires on")); - String userApiTokenForDelete = UtilIT.getApiTokenFromResponse(createUser); - /* Add tests for Private URL */ createUser = UtilIT.createRandomUser(); - String username = UtilIT.getUsernameFromResponse(createUser); String apiToken = UtilIT.getApiTokenFromResponse(createUser); Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); createDataverseResponse.prettyPrint(); @@ -416,8 +429,12 @@ public void testAPITokenEndpoints() { getExpiration = UtilIT.getTokenExpiration(tokenForPrivateUrlUser); getExpiration.prettyPrint(); getExpiration.then().assertThat() - .statusCode(NOT_FOUND.getStatusCode()); + .statusCode(UNAUTHORIZED.getStatusCode()); + + createUser = UtilIT.createRandomUser(); + assertEquals(OK.getStatusCode(), createUser.getStatusCode()); + String userApiTokenForDelete = UtilIT.getApiTokenFromResponse(createUser); Response deleteToken = UtilIT.deleteToken(userApiTokenForDelete); deleteToken.prettyPrint(); @@ -500,6 +517,177 @@ public void testDeleteAuthenticatedUser() { } + @Test + // This test is disabled because it is only compatible with the containerized development environment and would cause the Jenkins job to fail. + @Disabled + public void testRegisterOIDCUser() { + // Set Up - Get the admin access token from the OIDC provider + Response adminOidcLoginResponse = UtilIT.performKeycloakROPCLogin("admin", "admin"); + adminOidcLoginResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("access_token", notNullValue()); + String adminOidcAccessToken = adminOidcLoginResponse.jsonPath().getString("access_token"); + + // Set Up - Create random user in the OIDC provider without some necessary claims (email, firstName and lastName) + String randomUsername = UUID.randomUUID().toString().substring(0, 8); + + String newKeycloakUserWithoutClaimsJson = "{" + + "\"username\":\"" + randomUsername + "\"," + + "\"enabled\":true," + + "\"credentials\":[" + + " {" + + " \"type\":\"password\"," + + " \"value\":\"password\"," + + " \"temporary\":false" + + " }" + + "]" + + "}"; + + Response createKeycloakOidcUserResponse = UtilIT.createKeycloakUser(adminOidcAccessToken, newKeycloakUserWithoutClaimsJson); + createKeycloakOidcUserResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + + Response newUserOidcLoginResponse = UtilIT.performKeycloakROPCLogin(randomUsername, "password"); + String userWithoutClaimsAccessToken = newUserOidcLoginResponse.jsonPath().getString("access_token"); + + // Set Up - Create a second random user in the OIDC provider with all necessary claims (including email, firstName and lastName) + randomUsername = UUID.randomUUID().toString().substring(0, 8); + String email = randomUsername + "@dataverse.org"; + String firstName = "John"; + String lastName = "Doe"; + + String newKeycloakUserWithClaimsJson = "{" + + "\"username\":\"" + randomUsername + "\"," + + "\"enabled\":true," + + "\"email\":\"" + email + "\"," + + "\"firstName\":\"" + firstName + "\"," + + "\"lastName\":\"" + lastName + "\"," + + "\"credentials\":[" + + " {" + + " \"type\":\"password\"," + + " \"value\":\"password\"," + + " \"temporary\":false" + + " }" + + "]" + + "}"; + + Response createKeycloakOidcUserWithClaimsResponse = UtilIT.createKeycloakUser(adminOidcAccessToken, newKeycloakUserWithClaimsJson); + createKeycloakOidcUserWithClaimsResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + + Response newUserWithClaimsOidcLoginResponse = UtilIT.performKeycloakROPCLogin(randomUsername, "password"); + String userWithClaimsAccessToken = newUserWithClaimsOidcLoginResponse.jsonPath().getString("access_token"); + + // Should return error when empty token is passed + Response registerOidcUserResponse = UtilIT.registerOidcUser( + "{}", + "" + ); + registerOidcUserResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo(BundleUtil.getStringFromBundle("users.api.errors.bearerTokenRequired"))); + + // Should return error when a malformed User JSON is sent + registerOidcUserResponse = UtilIT.registerOidcUser( + "{{{user:abcde}", + "Bearer testBearerToken" + ); + registerOidcUserResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Error parsing the POSTed User json: Invalid token=CURLYOPEN at (line no=1, column no=2, offset=1). Expected tokens are: [STRING]")); + + // Should return error when the provided User JSON is valid but the provided Bearer token is invalid + registerOidcUserResponse = UtilIT.registerOidcUser( + "{" + + "\"termsAccepted\":true" + + "}", + "Bearer testBearerToken" + ); + registerOidcUserResponse.then().assertThat() + .statusCode(UNAUTHORIZED.getStatusCode()) + .body("message", equalTo("Unauthorized bearer token.")); + + // Should return an error when the termsAccepted field is missing in the User JSON + registerOidcUserResponse = UtilIT.registerOidcUser( + "{" + + "\"affiliation\":\"YourAffiliation\"," + + "\"position\":\"YourPosition\"" + + "}", + "Bearer testBearerToken" + ); + registerOidcUserResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Error parsing the POSTed User json: Field 'termsAccepted' is mandatory")); + + // Should return an error when the Bearer token is valid but required claims are missing in the IdP, needing completion from the request JSON + registerOidcUserResponse = UtilIT.registerOidcUser( + "{" + + "\"termsAccepted\":true" + + "}", + "Bearer " + userWithoutClaimsAccessToken + ); + registerOidcUserResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.invalidFields"))) + .body("fieldErrors.firstName", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldRequired", List.of("firstName")))) + .body("fieldErrors.lastName", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldRequired", List.of("lastName")))) + .body("fieldErrors.emailAddress", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldRequired", List.of("emailAddress")))); + + // Should register user when the Bearer token is valid and the provided User JSON contains the missing claims in the IdP + registerOidcUserResponse = UtilIT.registerOidcUser( + "{" + + "\"firstName\":\"testFirstName\"," + + "\"lastName\":\"testLastName\"," + + "\"emailAddress\":\"" + UUID.randomUUID().toString().substring(0, 8) + "@dataverse.org\"," + + "\"termsAccepted\":true" + + "}", + "Bearer " + userWithoutClaimsAccessToken + ); + registerOidcUserResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("User registered.")); + + // Should return error when attempting to re-register with the same Bearer token but different User data + String newUserJson = "{" + + "\"firstName\":\"newFirstName\"," + + "\"lastName\":\"newLastName\"," + + "\"emailAddress\":\"newEmail@dataverse.com\"," + + "\"termsAccepted\":true" + + "}"; + registerOidcUserResponse = UtilIT.registerOidcUser( + newUserJson, + "Bearer " + userWithoutClaimsAccessToken + ); + registerOidcUserResponse.then().assertThat() + .statusCode(FORBIDDEN.getStatusCode()) + .body("message", equalTo("User is already registered with this token.")); + + // Should return an error when the Bearer token is valid and attempting to set JSON properties that conflict with existing claims in the IdP + registerOidcUserResponse = UtilIT.registerOidcUser( + "{" + + "\"firstName\":\"testFirstName\"," + + "\"lastName\":\"testLastName\"," + + "\"emailAddress\":\"" + UUID.randomUUID().toString().substring(0, 8) + "@dataverse.org\"," + + "\"termsAccepted\":true" + + "}", + "Bearer " + userWithClaimsAccessToken + ); + registerOidcUserResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("fieldErrors.firstName", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldAlreadyPresentInProvider", List.of("firstName")))) + .body("fieldErrors.lastName", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldAlreadyPresentInProvider", List.of("lastName")))) + .body("fieldErrors.emailAddress", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldAlreadyPresentInProvider", List.of("emailAddress")))); + + // Should register user when the Bearer token is valid and all required claims are present in the IdP, requiring only minimal data in the User JSON + registerOidcUserResponse = UtilIT.registerOidcUser( + "{" + + "\"termsAccepted\":true" + + "}", + "Bearer " + userWithClaimsAccessToken + ); + registerOidcUserResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("User registered.")); + } + private Response convertUserFromBcryptToSha1(long idOfBcryptUserToConvert, String password) { JsonObjectBuilder data = Json.createObjectBuilder(); data.add("builtinUserId", idOfBcryptUserToConvert); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 0216859b869..2b8b2ce45e3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import io.restassured.http.ContentType; import io.restassured.path.json.JsonPath; import io.restassured.response.Response; @@ -12,6 +13,7 @@ import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; +import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import static jakarta.ws.rs.core.Response.Status.CREATED; import java.nio.charset.StandardCharsets; @@ -22,6 +24,7 @@ import edu.harvard.iq.dataverse.api.datadeposit.SwordConfigurationImpl; import io.restassured.path.xml.XmlPath; import edu.harvard.iq.dataverse.mydata.MyDataFilterParams; +import jakarta.ws.rs.core.HttpHeaders; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.Test; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -46,6 +49,7 @@ import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldValue; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.util.StringUtil; import java.util.Collections; @@ -238,6 +242,22 @@ public static Response clearThumbnailFailureFlag(long fileId) { return response; } + public static Response auditFiles(String apiToken, Long firstId, Long lastId, String csvList) { + String params = ""; + if (firstId != null) { + params = "?firstId="+ firstId; + } + if (lastId != null) { + params = params + (params.isEmpty() ? "?" : "&") + "lastId="+ lastId; + } + if (csvList != null) { + params = params + (params.isEmpty() ? "?" : "&") + "datasetIdentifierList="+ csvList; + } + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/admin/datafiles/auditFiles" + params); + } + private static String getAuthenticatedUserAsJsonString(String persistentUserId, String firstName, String lastName, String authenticationProviderId, String identifier) { JsonObjectBuilder builder = Json.createObjectBuilder(); builder.add("authenticationProviderId", authenticationProviderId); @@ -322,7 +342,14 @@ static Integer getDatasetIdFromResponse(Response createDatasetResponse) { logger.info("Id found in create dataset response: " + datasetId); return datasetId; } - + + static Integer getDataFileIdFromResponse(Response uploadDataFileResponse) { + JsonPath dataFile = JsonPath.from(uploadDataFileResponse.body().asString()); + int dataFileId = dataFile.getInt("data.files[0].dataFile.id"); + logger.info("Id found in upload DataFile response: " + dataFileId); + return dataFileId; + } + static Integer getSearchCountFromResponse(Response searchResponse) { JsonPath createdDataset = JsonPath.from(searchResponse.body().asString()); int searchCount = createdDataset.getInt("data.total_count"); @@ -358,24 +385,109 @@ public static Response getServiceDocument(String apiToken) { static Response createDataverse(String alias, String category, String apiToken) { return createSubDataverse(alias, category, apiToken, ":root"); } - + + static Response createDataverse(String alias, String category, String affiliation, String apiToken) { + return createSubDataverse(alias, category, apiToken, ":root", null, null, null, affiliation); + } + static Response createSubDataverse(String alias, String category, String apiToken, String parentDV) { + return createSubDataverse(alias, category, apiToken, parentDV, null, null, null, null); + } + + static Response createSubDataverse(String alias, String category, String apiToken, String parentDV, String[] inputLevelNames, String[] facetIds, String[] metadataBlockNames) { + return createSubDataverse(alias, category, apiToken, parentDV, inputLevelNames, facetIds, metadataBlockNames, null); + } + + static Response createSubDataverse(String alias, String category, String apiToken, String parentDV, String[] inputLevelNames, String[] facetIds, String[] metadataBlockNames, String affiliation) { JsonArrayBuilder contactArrayBuilder = Json.createArrayBuilder(); contactArrayBuilder.add(Json.createObjectBuilder().add("contactEmail", getEmailFromUserName(getRandomIdentifier()))); JsonArrayBuilder subjectArrayBuilder = Json.createArrayBuilder(); subjectArrayBuilder.add("Other"); - JsonObject dvData = Json.createObjectBuilder() + JsonObjectBuilder objectBuilder = Json.createObjectBuilder() .add("alias", alias) .add("name", alias) .add("dataverseContacts", contactArrayBuilder) .add("dataverseSubjects", subjectArrayBuilder) // don't send "dataverseType" if category is null, must be a better way - .add(category != null ? "dataverseType" : "notTheKeyDataverseType", category != null ? category : "whatever") - .build(); - Response createDataverseResponse = given() + .add(category != null ? "dataverseType" : "notTheKeyDataverseType", category != null ? category : "whatever"); + + if (affiliation != null) { + objectBuilder.add("affiliation", affiliation); + } + + updateDataverseRequestJsonWithMetadataBlocksConfiguration(inputLevelNames, facetIds, metadataBlockNames, objectBuilder); + + JsonObject dvData = objectBuilder.build(); + return given() .body(dvData.toString()).contentType(ContentType.JSON) .when().post("/api/dataverses/" + parentDV + "?key=" + apiToken); - return createDataverseResponse; + } + + static Response updateDataverse(String alias, + String newAlias, + String newName, + String newAffiliation, + String newDataverseType, + String[] newContactEmails, + String[] newInputLevelNames, + String[] newFacetIds, + String[] newMetadataBlockNames, + String apiToken) { + JsonArrayBuilder contactArrayBuilder = Json.createArrayBuilder(); + for(String contactEmail : newContactEmails) { + contactArrayBuilder.add(Json.createObjectBuilder().add("contactEmail", contactEmail)); + } + NullSafeJsonBuilder jsonBuilder = jsonObjectBuilder() + .add("alias", newAlias) + .add("name", newName) + .add("affiliation", newAffiliation) + .add("dataverseContacts", contactArrayBuilder) + .add("dataverseType", newDataverseType) + .add("affiliation", newAffiliation); + + updateDataverseRequestJsonWithMetadataBlocksConfiguration(newInputLevelNames, newFacetIds, newMetadataBlockNames, jsonBuilder); + + JsonObject dvData = jsonBuilder.build(); + return given() + .body(dvData.toString()).contentType(ContentType.JSON) + .when().put("/api/dataverses/" + alias + "?key=" + apiToken); + } + + private static void updateDataverseRequestJsonWithMetadataBlocksConfiguration(String[] inputLevelNames, + String[] facetIds, + String[] metadataBlockNames, + JsonObjectBuilder objectBuilder) { + JsonObjectBuilder metadataBlocksObjectBuilder = Json.createObjectBuilder(); + + if (inputLevelNames != null) { + JsonArrayBuilder inputLevelsArrayBuilder = Json.createArrayBuilder(); + for(String inputLevelName : inputLevelNames) { + inputLevelsArrayBuilder.add(Json.createObjectBuilder() + .add("datasetFieldTypeName", inputLevelName) + .add("required", true) + .add("include", true) + ); + } + metadataBlocksObjectBuilder.add("inputLevels", inputLevelsArrayBuilder); + } + + if (metadataBlockNames != null) { + JsonArrayBuilder metadataBlockNamesArrayBuilder = Json.createArrayBuilder(); + for(String metadataBlockName : metadataBlockNames) { + metadataBlockNamesArrayBuilder.add(metadataBlockName); + } + metadataBlocksObjectBuilder.add("metadataBlockNames", metadataBlockNamesArrayBuilder); + } + + if (facetIds != null) { + JsonArrayBuilder facetIdsArrayBuilder = Json.createArrayBuilder(); + for(String facetId : facetIds) { + facetIdsArrayBuilder.add(facetId); + } + metadataBlocksObjectBuilder.add("facetIds", facetIdsArrayBuilder); + } + + objectBuilder.add("metadataBlocks", metadataBlocksObjectBuilder); } static Response createDataverse(JsonObject dvData, String apiToken) { @@ -392,6 +504,12 @@ static Response createRandomDataverse(String apiToken) { return createDataverse(alias, category, apiToken); } + static Response createRandomDataverse(String apiToken, String affiliation) { + String alias = "dv" + getRandomIdentifier(); + String category = null; + return createDataverse(alias, category, affiliation, apiToken); + } + /** * A convenience method for creating a random collection and getting its * alias in one step. @@ -497,6 +615,15 @@ static Response createDataset(String dataverseAlias, String datasetJson, String return createDatasetResponse; } + static Response createDatasetSemantic(String dataverseAlias, String datasetJson, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(datasetJson) + .contentType("application/ld+json") + .post("/api/dataverses/" + dataverseAlias + "/datasets"); + return response; + } + static String getDatasetJson(String pathToJsonFile) { File datasetVersionJson = new File(pathToJsonFile); try { @@ -1505,7 +1632,16 @@ static Response getDatasetVersion(String persistentId, String versionNumber, Str + persistentId + (excludeFiles ? "&excludeFiles=true" : "")); } - + static Response compareDatasetVersions(String persistentId, String versionNumber1, String versionNumber2, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/datasets/:persistentId/versions/" + + versionNumber1 + + "/compare/" + + versionNumber2 + + "?persistentId=" + + persistentId); + } static Response getDatasetWithOwners(String persistentId, String apiToken, boolean returnOwners) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) @@ -2058,19 +2194,22 @@ static Response uploadProvFreeForm(String idOrPersistentId, JsonObject jsonObjec // return requestSpecification.delete("/api/files/" + idInPath + "/prov-freeform" + optionalQueryParam); // } static Response exportDataset(String datasetPersistentId, String exporter) { - return exportDataset(datasetPersistentId, exporter, null); + return exportDataset(datasetPersistentId, exporter, null, false); } - static Response exportDataset(String datasetPersistentId, String exporter, String apiToken) { -// http://localhost:8080/api/datasets/export?exporter=dataverse_json&persistentId=doi%3A10.5072/FK2/W6WIMQ + return exportDataset(datasetPersistentId, exporter, apiToken, false); + } + static Response exportDataset(String datasetPersistentId, String exporter, String apiToken, boolean wait) { + // Wait for the Async call to finish to get the updated data + if (wait) { + sleepForReexport(datasetPersistentId, apiToken, 10); + } RequestSpecification requestSpecification = given(); if (apiToken != null) { requestSpecification = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); } return requestSpecification - // .header(API_TOKEN_HTTP_HEADER, apiToken) - // .get("/api/datasets/:persistentId/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); .get("/api/datasets/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); } @@ -2173,6 +2312,16 @@ public static Response setSetting(String settingKey, String value) { return response; } + static Response getFeatureFlags() { + Response response = given().when().get("/api/admin/featureFlags"); + return response; + } + + static Response getFeatureFlag(FeatureFlags featureFlag) { + Response response = given().when().get("/api/admin/featureFlags/" + featureFlag); + return response; + } + static Response getRoleAssignmentsOnDataverse(String dataverseAliasOrId, String apiToken) { String url = "/api/dataverses/" + dataverseAliasOrId + "/assignments"; return given() @@ -2738,10 +2887,15 @@ static Response getTokenExpiration( String apiToken) { return response; } - static Response recreateToken( String apiToken) { + static Response recreateToken(String apiToken) { + return recreateToken(apiToken, false); + } + + static Response recreateToken(String apiToken, boolean returnExpiration) { Response response = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .post("api/users/token/recreate"); + .header(API_TOKEN_HTTP_HEADER, apiToken) + .queryParam("returnExpiration", returnExpiration) + .post("api/users/token/recreate"); return response; } @@ -3542,6 +3696,31 @@ private static DatasetField constructPrimitive(String fieldName, String value) { return field; } + static Response importDatasetNativeJson(String apiToken, String dataverseAlias, String jsonString, String pid, String release) { + + String postString = "/api/dataverses/" + dataverseAlias + "/datasets/:import"; + if (pid != null || release != null) { + //postString = postString + "?"; + if (pid != null) { + postString = postString + "?pid=" + pid; + if (release != null && release.compareTo("yes") == 0) { + postString = postString + "&release=" + release.toString(); + } + } else { + if (release != null && release.compareTo("yes") == 0) { + postString = postString + "?release=" + release.toString(); + } + } + } + + RequestSpecification importJson = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .urlEncodingEnabled(false) + .body(jsonString) + .contentType("application/json"); + + return importJson.post(postString); + } static Response importDatasetDDIViaNativeApi(String apiToken, String dataverseAlias, String xml, String pid, String release) { @@ -3572,6 +3751,35 @@ static Response importDatasetDDIViaNativeApi(String apiToken, String dataverseAl return importDDI.post(postString); } + + static Response importDatasetViaNativeApi(String apiToken, String dataverseAlias, String json, String pid, String release) { + String postString = "/api/dataverses/" + dataverseAlias + "/datasets/:import"; + if (pid != null || release != null ) { + //postString = postString + "?"; + if (pid != null) { + postString = postString + "?pid=" + pid; + if (release != null && release.compareTo("yes") == 0) { + postString = postString + "&release=" + release; + } + } else { + if (release != null && release.compareTo("yes") == 0) { + postString = postString + "?release=" + release; + } + } + } + logger.info("Here importDatasetViaNativeApi"); + logger.info(postString); + + RequestSpecification importJSON = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .urlEncodingEnabled(false) + .body(json) + .contentType("application/json"); + + return importJSON.post(postString); + } + + static Response retrieveMyDataAsJsonString(String apiToken, String userIdentifier, ArrayList roleIds) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) @@ -3635,6 +3843,33 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, boo return response; } + static Response setDatasetCitationDateField(String datasetIdOrPersistentId, String dateField, String apiToken) { + String idInPath = datasetIdOrPersistentId; // Assume it's a number. + String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. + if (!NumberUtils.isCreatable(datasetIdOrPersistentId)) { + idInPath = ":persistentId"; + optionalQueryParam = "?persistentId=" + datasetIdOrPersistentId; + } + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(dateField) + .put("/api/datasets/" + idInPath + "/citationdate" + optionalQueryParam); + return response; + } + + static Response clearDatasetCitationDateField(String datasetIdOrPersistentId, String apiToken) { + String idInPath = datasetIdOrPersistentId; // Assume it's a number. + String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. + if (!NumberUtils.isCreatable(datasetIdOrPersistentId)) { + idInPath = ":persistentId"; + optionalQueryParam = "?persistentId=" + datasetIdOrPersistentId; + } + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .delete("/api/datasets/" + idInPath + "/citationdate" + optionalQueryParam); + return response; + } + static Response getFileCitation(Integer fileId, String datasetVersion, String apiToken) { Boolean includeDeaccessioned = null; return getFileCitation(fileId, datasetVersion, includeDeaccessioned, apiToken); @@ -3739,6 +3974,12 @@ static Response getUserPermissionsOnDataset(String datasetId, String apiToken) { .get("/api/datasets/" + datasetId + "/userPermissions"); } + static Response getUserPermissionsOnDataverse(String dataverseAlias, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/dataverses/" + dataverseAlias + "/userPermissions"); + } + static Response getCanDownloadAtLeastOneFile(String datasetId, String versionId, String apiToken) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) @@ -3926,8 +4167,37 @@ static Response setDatasetStorageDriver(Integer datasetId, String driverLabel, S .body(driverLabel) .put("/api/datasets/" + datasetId + "/storageDriver"); } - - + + /** GET on /api/admin/savedsearches/list */ + static Response getSavedSearchList() { + return given().get("/api/admin/savedsearches/list"); + } + + /** POST on /api/admin/savedsearches without body */ + static Response setSavedSearch() { + return given() + .contentType("application/json") + .post("/api/admin/savedsearches"); + } + + /** POST on /api/admin/savedsearches with body */ + static Response setSavedSearch(String body) { + return given() + .body(body) + .contentType("application/json") + .post("/api/admin/savedsearches"); + } + + /** PUT on /api/admin/savedsearches/makelinks/all */ + static Response setSavedSearchMakelinksAll() { + return given().put("/api/admin/savedsearches/makelinks/all"); + } + + /** DELETE on /api/admin/savedsearches/{id} with identifier */ + static Response deleteSavedSearchById(Integer id) { + return given().delete("/api/admin/savedsearches/" + id); + } + //Globus Store related - not currently used static Response getDatasetGlobusUploadParameters(Integer datasetId, String locale, String apiToken) { @@ -3986,4 +4256,107 @@ public static Response getOpenAPI(String accept, String format) { .get("/openapi"); return response; } + + static Response listDataverseFacets(String dataverseAlias, String apiToken) { + return listDataverseFacets(dataverseAlias, false, apiToken); + } + + static Response listDataverseFacets(String dataverseAlias, boolean returnDetails, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .queryParam("returnDetails", returnDetails) + .get("/api/dataverses/" + dataverseAlias + "/facets"); + } + + static Response listAllFacetableDatasetFields() { + return given() + .get("/api/datasetfields/facetables"); + } + + static Response listDataverseInputLevels(String dataverseAlias, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/json") + .get("/api/dataverses/" + dataverseAlias + "/inputLevels"); + } + + public static Response getDatasetTypes() { + Response response = given() + .get("/api/datasets/datasetTypes"); + return response; + } + + static Response getDatasetType(String idOrName) { + return given() + .get("/api/datasets/datasetTypes/" + idOrName); + } + + static Response addDatasetType(String jsonIn, String apiToken) { + System.out.println("called addDatasetType..."); + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(jsonIn) + .contentType(ContentType.JSON) + .post("/api/datasets/datasetTypes"); + } + + static Response deleteDatasetTypes(long doomed, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .delete("/api/datasets/datasetTypes/" + doomed); + } + + static Response registerOidcUser(String jsonIn, String bearerToken) { + return given() + .header(HttpHeaders.AUTHORIZATION, bearerToken) + .body(jsonIn) + .contentType(ContentType.JSON) + .post("/api/users/register"); + } + + /** + * Creates a new user in the development Keycloak instance. + *

    This method is specifically designed for use in the containerized Keycloak development + * environment. The configured Keycloak instance must be accessible at the specified URL. + * The method sends a request to the Keycloak Admin API to create a new user in the given realm. + * + *

    Refer to the {@code testRegisterOidc()} method in the {@code UsersIT} class for an example + * of this method in action. + * + * @param bearerToken The Bearer token used for authenticating the request to the Keycloak Admin API. + * @param userJson The JSON representation of the user to be created. + * @return A {@link Response} containing the result of the user creation request. + */ + static Response createKeycloakUser(String bearerToken, String userJson) { + return given() + .contentType(ContentType.JSON) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + bearerToken) + .body(userJson) + .post("http://keycloak.mydomain.com:8090/admin/realms/test/users"); + } + + /** + * Performs an OIDC login in the development Keycloak instance using the Resource Owner Password Credentials (ROPC) + * grant type to retrieve authentication tokens from a Keycloak instance. + * + *

    This method is specifically designed for use in the containerized Keycloak development + * environment. The configured Keycloak instance must be accessible at the specified URL. + * + *

    Refer to the {@code testRegisterOidc()} method in the {@code UsersIT} class for an example + * of this method in action. + * + * @return A {@link Response} containing authentication tokens, including access and refresh tokens, + * if the login is successful. + */ + static Response performKeycloakROPCLogin(String username, String password) { + return given() + .contentType(ContentType.URLENC) + .formParam("client_id", "test") + .formParam("client_secret", "94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8") + .formParam("username", username) + .formParam("password", password) + .formParam("grant_type", "password") + .formParam("scope", "openid") + .post("http://keycloak.mydomain.com:8090/realms/test/protocol/openid-connect/token"); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/ApiKeyAuthMechanismTest.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/ApiKeyAuthMechanismTest.java index 486697664e6..12216819cf8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/auth/ApiKeyAuthMechanismTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/ApiKeyAuthMechanismTest.java @@ -84,9 +84,9 @@ public void testFindUserFromRequest_ApiKeyProvided_AnonymizedPrivateUrlUserAuthe sut.userSvc = Mockito.mock(UserServiceBean.class); ContainerRequestContext testContainerRequest = new ApiKeyContainerRequestTestFake(TEST_API_KEY, TEST_PATH); - WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); + WrappedUnauthorizedAuthErrorResponse wrappedUnauthorizedAuthErrorResponse = assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); - assertEquals(RESPONSE_MESSAGE_BAD_API_KEY, wrappedAuthErrorResponse.getMessage()); + assertEquals(RESPONSE_MESSAGE_BAD_API_KEY, wrappedUnauthorizedAuthErrorResponse.getMessage()); } @Test @@ -123,8 +123,8 @@ public void testFindUserFromRequest_ApiKeyProvided_CanNotAuthenticateUserWithAny sut.userSvc = Mockito.mock(UserServiceBean.class); ContainerRequestContext testContainerRequest = new ApiKeyContainerRequestTestFake(TEST_API_KEY, TEST_PATH); - WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); + WrappedUnauthorizedAuthErrorResponse wrappedUnauthorizedAuthErrorResponse = assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); - assertEquals(RESPONSE_MESSAGE_BAD_API_KEY, wrappedAuthErrorResponse.getMessage()); + assertEquals(RESPONSE_MESSAGE_BAD_API_KEY, wrappedUnauthorizedAuthErrorResponse.getMessage()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java index 7e1c23d26f4..ab4090eb0a0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java @@ -1,15 +1,13 @@ package edu.harvard.iq.dataverse.api.auth; -import com.nimbusds.oauth2.sdk.ParseException; -import com.nimbusds.oauth2.sdk.token.BearerAccessToken; import edu.harvard.iq.dataverse.UserServiceBean; import edu.harvard.iq.dataverse.api.auth.doubles.BearerTokenKeyContainerRequestTestFake; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; -import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier; -import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthProvider; +import edu.harvard.iq.dataverse.authorization.exceptions.AuthorizationException; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import org.junit.jupiter.api.BeforeEach; @@ -18,18 +16,13 @@ import jakarta.ws.rs.container.ContainerRequestContext; -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; - -import static edu.harvard.iq.dataverse.api.auth.BearerTokenAuthMechanism.*; import static org.junit.jupiter.api.Assertions.*; @LocalJvmSettings @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth") class BearerTokenAuthMechanismTest { - private static final String TEST_API_KEY = "test-api-key"; + private static final String TEST_BEARER_TOKEN = "Bearer test"; private BearerTokenAuthMechanism sut; @@ -49,119 +42,42 @@ void testFindUserFromRequest_no_token() throws WrappedAuthErrorResponse { } @Test - void testFindUserFromRequest_invalid_token() { - Mockito.when(sut.authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Collections.emptySet()); - - ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer "); - WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); - - //then - assertEquals(INVALID_BEARER_TOKEN, wrappedAuthErrorResponse.getMessage()); - } - @Test - void testFindUserFromRequest_no_OidcProvider() { - Mockito.when(sut.authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Collections.emptySet()); - - ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer " +TEST_API_KEY); - WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); - - //then - assertEquals(BEARER_TOKEN_DETECTED_NO_OIDC_PROVIDER_CONFIGURED, wrappedAuthErrorResponse.getMessage()); - } - - @Test - void testFindUserFromRequest_oneProvider_invalidToken_1() throws ParseException, IOException { - OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class); - String providerID = "OIEDC"; - Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID); - // ensure that a valid OIDCAuthProvider is available within the AuthenticationServiceBean - Mockito.when(sut.authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Collections.singleton(providerID)); - Mockito.when(sut.authSvc.getAuthenticationProvider(providerID)).thenReturn(oidcAuthProvider); - - // ensure that the OIDCAuthProvider returns a valid UserRecordIdentifier for a given Token - BearerAccessToken token = BearerAccessToken.parse("Bearer " + TEST_API_KEY); - Mockito.when(oidcAuthProvider.getUserIdentifier(token)).thenReturn(Optional.empty()); + void testFindUserFromRequest_invalid_token() throws AuthorizationException { + String testErrorMessage = "test error"; + Mockito.when(sut.authSvc.lookupUserByOIDCBearerToken(TEST_BEARER_TOKEN)).thenThrow(new AuthorizationException(testErrorMessage)); // when - ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer " + TEST_API_KEY); - WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); + ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake(TEST_BEARER_TOKEN); + WrappedUnauthorizedAuthErrorResponse wrappedUnauthorizedAuthErrorResponse = assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); //then - assertEquals(UNAUTHORIZED_BEARER_TOKEN, wrappedAuthErrorResponse.getMessage()); + assertEquals(testErrorMessage, wrappedUnauthorizedAuthErrorResponse.getMessage()); } @Test - void testFindUserFromRequest_oneProvider_invalidToken_2() throws ParseException, IOException { - OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class); - String providerID = "OIEDC"; - Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID); - // ensure that a valid OIDCAuthProvider is available within the AuthenticationServiceBean - Mockito.when(sut.authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Collections.singleton(providerID)); - Mockito.when(sut.authSvc.getAuthenticationProvider(providerID)).thenReturn(oidcAuthProvider); - - // ensure that the OIDCAuthProvider returns a valid UserRecordIdentifier for a given Token - BearerAccessToken token = BearerAccessToken.parse("Bearer " + TEST_API_KEY); - Mockito.when(oidcAuthProvider.getUserIdentifier(token)).thenThrow(IOException.class); - - // when - ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer " + TEST_API_KEY); - WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); - - //then - assertEquals(UNAUTHORIZED_BEARER_TOKEN, wrappedAuthErrorResponse.getMessage()); - } - @Test - void testFindUserFromRequest_oneProvider_validToken() throws WrappedAuthErrorResponse, ParseException, IOException { - OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class); - String providerID = "OIEDC"; - Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID); - // ensure that a valid OIDCAuthProvider is available within the AuthenticationServiceBean - Mockito.when(sut.authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Collections.singleton(providerID)); - Mockito.when(sut.authSvc.getAuthenticationProvider(providerID)).thenReturn(oidcAuthProvider); - - // ensure that the OIDCAuthProvider returns a valid UserRecordIdentifier for a given Token - UserRecordIdentifier userinfo = new UserRecordIdentifier(providerID, "KEY"); - BearerAccessToken token = BearerAccessToken.parse("Bearer " + TEST_API_KEY); - Mockito.when(oidcAuthProvider.getUserIdentifier(token)).thenReturn(Optional.of(userinfo)); - - // ensures that the AuthenticationServiceBean can retrieve an Authenticated user based on the UserRecordIdentifier + void testFindUserFromRequest_validToken_accountExists() throws WrappedAuthErrorResponse, AuthorizationException { AuthenticatedUser testAuthenticatedUser = new AuthenticatedUser(); - Mockito.when(sut.authSvc.lookupUser(userinfo)).thenReturn(testAuthenticatedUser); + Mockito.when(sut.authSvc.lookupUserByOIDCBearerToken(TEST_BEARER_TOKEN)).thenReturn(testAuthenticatedUser); Mockito.when(sut.userSvc.updateLastApiUseTime(testAuthenticatedUser)).thenReturn(testAuthenticatedUser); // when - ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer " + TEST_API_KEY); + ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake(TEST_BEARER_TOKEN); User actual = sut.findUserFromRequest(testContainerRequest); //then assertEquals(testAuthenticatedUser, actual); Mockito.verify(sut.userSvc, Mockito.atLeastOnce()).updateLastApiUseTime(testAuthenticatedUser); - } - @Test - void testFindUserFromRequest_oneProvider_validToken_noAccount() throws WrappedAuthErrorResponse, ParseException, IOException { - OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class); - String providerID = "OIEDC"; - Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID); - // ensure that a valid OIDCAuthProvider is available within the AuthenticationServiceBean - Mockito.when(sut.authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Collections.singleton(providerID)); - Mockito.when(sut.authSvc.getAuthenticationProvider(providerID)).thenReturn(oidcAuthProvider); - - // ensure that the OIDCAuthProvider returns a valid UserRecordIdentifier for a given Token - UserRecordIdentifier userinfo = new UserRecordIdentifier(providerID, "KEY"); - BearerAccessToken token = BearerAccessToken.parse("Bearer " + TEST_API_KEY); - Mockito.when(oidcAuthProvider.getUserIdentifier(token)).thenReturn(Optional.of(userinfo)); - - // ensures that the AuthenticationServiceBean can retrieve an Authenticated user based on the UserRecordIdentifier - Mockito.when(sut.authSvc.lookupUser(userinfo)).thenReturn(null); + @Test + void testFindUserFromRequest_validToken_noAccount() throws AuthorizationException { + Mockito.when(sut.authSvc.lookupUserByOIDCBearerToken(TEST_BEARER_TOKEN)).thenReturn(null); // when - ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer " + TEST_API_KEY); - User actual = sut.findUserFromRequest(testContainerRequest); + ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake(TEST_BEARER_TOKEN); + WrappedForbiddenAuthErrorResponse wrappedForbiddenAuthErrorResponse = assertThrows(WrappedForbiddenAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); //then - assertNull(actual); - + assertEquals(BundleUtil.getStringFromBundle("bearerTokenAuthMechanism.errors.tokenValidatedButNoRegisteredUser"), wrappedForbiddenAuthErrorResponse.getMessage()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java index 74db6e544da..6fd7d2e1d8e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java @@ -65,9 +65,9 @@ public void testFindUserFromRequest_SignedUrlTokenProvided_UserExists_InvalidSig sut.authSvc = authenticationServiceBeanStub; ContainerRequestContext testContainerRequest = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID); - WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); + WrappedUnauthorizedAuthErrorResponse wrappedUnauthorizedAuthErrorResponse = assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); - assertEquals(RESPONSE_MESSAGE_BAD_SIGNED_URL, wrappedAuthErrorResponse.getMessage()); + assertEquals(RESPONSE_MESSAGE_BAD_SIGNED_URL, wrappedUnauthorizedAuthErrorResponse.getMessage()); } @Test @@ -79,9 +79,9 @@ public void testFindUserFromRequest_SignedUrlTokenProvided_UserExists_UserApiTok sut.authSvc = authenticationServiceBeanStub; ContainerRequestContext testContainerRequest = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID); - WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); + WrappedUnauthorizedAuthErrorResponse wrappedUnauthorizedAuthErrorResponse = assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); - assertEquals(RESPONSE_MESSAGE_BAD_SIGNED_URL, wrappedAuthErrorResponse.getMessage()); + assertEquals(RESPONSE_MESSAGE_BAD_SIGNED_URL, wrappedUnauthorizedAuthErrorResponse.getMessage()); } @Test @@ -92,8 +92,8 @@ public void testFindUserFromRequest_SignedUrlTokenProvided_UserDoesNotExistForTh sut.authSvc = authenticationServiceBeanStub; ContainerRequestContext testContainerRequest = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID); - WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); + WrappedUnauthorizedAuthErrorResponse wrappedUnauthorizedAuthErrorResponse = assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); - assertEquals(RESPONSE_MESSAGE_BAD_SIGNED_URL, wrappedAuthErrorResponse.getMessage()); + assertEquals(RESPONSE_MESSAGE_BAD_SIGNED_URL, wrappedUnauthorizedAuthErrorResponse.getMessage()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/WorkflowKeyAuthMechanismTest.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/WorkflowKeyAuthMechanismTest.java index 3f90fa73fa9..22c3abffe2b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/auth/WorkflowKeyAuthMechanismTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/WorkflowKeyAuthMechanismTest.java @@ -54,8 +54,8 @@ public void testFindUserFromRequest_WorkflowKeyProvided_UserNotAuthenticated() { sut.authSvc = authenticationServiceBeanStub; ContainerRequestContext testContainerRequest = new WorkflowKeyContainerRequestTestFake(TEST_WORKFLOW_KEY); - WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); + WrappedUnauthorizedAuthErrorResponse wrappedUnauthorizedAuthErrorResponse = assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest)); - assertEquals(RESPONSE_MESSAGE_BAD_WORKFLOW_KEY, wrappedAuthErrorResponse.getMessage()); + assertEquals(RESPONSE_MESSAGE_BAD_WORKFLOW_KEY, wrappedUnauthorizedAuthErrorResponse.getMessage()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBeanTest.java index 44739f3f62a..acf5d970358 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBeanTest.java @@ -1,6 +1,13 @@ package edu.harvard.iq.dataverse.api.imports; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; +import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; + +import org.apache.commons.io.FileUtils; +import com.google.gson.Gson; +import java.io.File; +import java.io.IOException; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -8,6 +15,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import java.nio.charset.StandardCharsets; + @ExtendWith(MockitoExtension.class) public class ImportGenericServiceBeanTest { @@ -15,7 +24,47 @@ public class ImportGenericServiceBeanTest { private ImportGenericServiceBean importGenericService; @Test - public void testReassignIdentifierAsGlobalId() { + void testIdentifierHarvestableWithOtherID() throws IOException { + // "otherIdValue" containing the value : doi:10.7910/DVN/TJCLKP + File file = new File("src/test/resources/json/importGenericWithOtherId.json"); + String text = FileUtils.readFileToString(file, StandardCharsets.UTF_8); + DatasetVersionDTO dto = new Gson().fromJson(text, DatasetVersionDTO.class); + + assertEquals("doi:10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "https://doi.org/10.7910/DVN/TJCLKP")); + // junk or null + assertEquals("doi:10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "junk")); + assertEquals("doi:10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, null)); + assertEquals("doi:10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "http://www.example.com")); + assertEquals("doi:10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "https://dataverse.org")); + } + + @Test + void testIdentifierHarvestableWithoutOtherID() throws IOException { + // Does not contain data of type "otherIdValue" + File file = new File("src/test/resources/json/importGenericWithoutOtherId.json"); + String text = FileUtils.readFileToString(file, StandardCharsets.UTF_8); + DatasetVersionDTO dto = new Gson().fromJson(text, DatasetVersionDTO.class); + + // non-URL + assertEquals("doi:10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "doi:10.7910/DVN/TJCLKP")); + assertEquals("hdl:10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "hdl:10.7910/DVN/TJCLKP")); + // HTTPS + assertEquals("https://doi.org/10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "https://doi.org/10.7910/DVN/TJCLKP")); + assertEquals("https://dx.doi.org/10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "https://dx.doi.org/10.7910/DVN/TJCLKP")); + assertEquals("https://hdl.handle.net/10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "https://hdl.handle.net/10.7910/DVN/TJCLKP")); + // HTTP (no S) + assertEquals("http://doi.org/10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "http://doi.org/10.7910/DVN/TJCLKP")); + assertEquals("http://dx.doi.org/10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "http://dx.doi.org/10.7910/DVN/TJCLKP")); + assertEquals("http://hdl.handle.net/10.7910/DVN/TJCLKP", importGenericService.selectIdentifier(dto, "http://hdl.handle.net/10.7910/DVN/TJCLKP")); + // junk or null + assertNull(importGenericService.selectIdentifier(dto, "junk")); + assertNull(importGenericService.selectIdentifier(dto, null)); + assertNull(importGenericService.selectIdentifier(dto, "http://www.example.com")); + assertNull(importGenericService.selectIdentifier(dto, "https://dataverse.org")); + } + + @Test + void testReassignIdentifierAsGlobalId() { // non-URL assertEquals("doi:10.7910/DVN/TJCLKP", importGenericService.reassignIdentifierAsGlobalId("doi:10.7910/DVN/TJCLKP", new DatasetDTO())); assertEquals("hdl:10.7910/DVN/TJCLKP", importGenericService.reassignIdentifierAsGlobalId("hdl:10.7910/DVN/TJCLKP", new DatasetDTO())); @@ -29,6 +78,8 @@ public void testReassignIdentifierAsGlobalId() { assertEquals("hdl:10.7910/DVN/TJCLKP", importGenericService.reassignIdentifierAsGlobalId("http://hdl.handle.net/10.7910/DVN/TJCLKP", new DatasetDTO())); // junk assertNull(importGenericService.reassignIdentifierAsGlobalId("junk", new DatasetDTO())); + assertNull(importGenericService.reassignIdentifierAsGlobalId("http://www.example.com", new DatasetDTO())); + assertNull(importGenericService.reassignIdentifierAsGlobalId("https://dataverse.org", new DatasetDTO())); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationProviderTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationProviderTest.java index eac9a605c9e..d4d7b6fa69d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationProviderTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationProviderTest.java @@ -15,7 +15,7 @@ public class AuthenticationProviderTest { - private final static String[] authProviders = {"null", "builtin", "github", "google", "orcid", "orcid-sandbox", "shib"}; + private static final String[] authProviders = {"null", "builtin", "github", "google", "orcid", "orcid-sandbox", "shib"}; private static Map bundleTestMap; @BeforeAll diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBeanTest.java new file mode 100644 index 00000000000..56ac4eefb3d --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBeanTest.java @@ -0,0 +1,152 @@ +package edu.harvard.iq.dataverse.authorization; + +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import com.nimbusds.openid.connect.sdk.claims.UserInfo; +import edu.harvard.iq.dataverse.authorization.exceptions.AuthorizationException; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2Exception; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthProvider; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.util.BundleUtil; +import jakarta.persistence.EntityManager; +import jakarta.persistence.NoResultException; +import jakarta.persistence.TypedQuery; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; + +public class AuthenticationServiceBeanTest { + + private AuthenticationServiceBean sut; + private static final String TEST_BEARER_TOKEN = "Bearer test"; + + @BeforeEach + public void setUp() { + sut = new AuthenticationServiceBean(); + sut.authProvidersRegistrationService = Mockito.mock(AuthenticationProvidersRegistrationServiceBean.class); + sut.em = Mockito.mock(EntityManager.class); + } + + @Test + void testLookupUserByOIDCBearerToken_no_OIDCProvider() { + // Given no OIDC providers are configured + Mockito.when(sut.authProvidersRegistrationService.getAuthenticationProvidersMap()).thenReturn(Map.of()); + + // When invoking lookupUserByOIDCBearerToken + AuthorizationException exception = assertThrows(AuthorizationException.class, + () -> sut.lookupUserByOIDCBearerToken(TEST_BEARER_TOKEN)); + + // Then the exception message should indicate no OIDC provider is configured + assertEquals(BundleUtil.getStringFromBundle("authenticationServiceBean.errors.bearerTokenDetectedNoOIDCProviderConfigured"), exception.getMessage()); + } + + @Test + void testLookupUserByOIDCBearerToken_oneProvider_invalidToken_1() throws ParseException, OAuth2Exception, IOException { + // Given a single OIDC provider that cannot find a user + OIDCAuthProvider oidcAuthProviderStub = stubOIDCAuthProvider("OIEDC"); + BearerAccessToken token = BearerAccessToken.parse(TEST_BEARER_TOKEN); + Mockito.when(oidcAuthProviderStub.getUserInfo(token)).thenReturn(Optional.empty()); + + // When invoking lookupUserByOIDCBearerToken + AuthorizationException exception = assertThrows(AuthorizationException.class, + () -> sut.lookupUserByOIDCBearerToken(TEST_BEARER_TOKEN)); + + // Then the exception message should indicate an unauthorized token + assertEquals(BundleUtil.getStringFromBundle("authenticationServiceBean.errors.unauthorizedBearerToken"), exception.getMessage()); + } + + @Test + void testLookupUserByOIDCBearerToken_oneProvider_invalidToken_2() throws ParseException, IOException, OAuth2Exception { + // Given a single OIDC provider that throws an IOException + OIDCAuthProvider oidcAuthProviderStub = stubOIDCAuthProvider("OIEDC"); + BearerAccessToken token = BearerAccessToken.parse(TEST_BEARER_TOKEN); + Mockito.when(oidcAuthProviderStub.getUserInfo(token)).thenThrow(IOException.class); + + // When invoking lookupUserByOIDCBearerToken + AuthorizationException exception = assertThrows(AuthorizationException.class, + () -> sut.lookupUserByOIDCBearerToken(TEST_BEARER_TOKEN)); + + // Then the exception message should indicate an unauthorized token + assertEquals(BundleUtil.getStringFromBundle("authenticationServiceBean.errors.unauthorizedBearerToken"), exception.getMessage()); + } + + @Test + void testLookupUserByOIDCBearerToken_oneProvider_validToken() throws ParseException, IOException, AuthorizationException, OAuth2Exception { + // Given a single OIDC provider that returns a valid user identifier + setUpOIDCProviderWhichValidatesToken(); + + // Setting up an authenticated user is found + AuthenticatedUser authenticatedUser = setupAuthenticatedUserQueryWithResult(new AuthenticatedUser()); + + // When invoking lookupUserByOIDCBearerToken + User actualUser = sut.lookupUserByOIDCBearerToken(TEST_BEARER_TOKEN); + + // Then the actual user should match the expected authenticated user + assertEquals(authenticatedUser, actualUser); + } + + @Test + void testLookupUserByOIDCBearerToken_oneProvider_validToken_noAccount() throws ParseException, IOException, AuthorizationException, OAuth2Exception { + // Given a single OIDC provider that returns a valid user identifier + setUpOIDCProviderWhichValidatesToken(); + + // Setting up an authenticated user is not found + setupAuthenticatedUserQueryWithNoResult(); + + // When invoking lookupUserByOIDCBearerToken + User actualUser = sut.lookupUserByOIDCBearerToken(TEST_BEARER_TOKEN); + + // Then no user should be found, and result should be null + assertNull(actualUser); + } + + private AuthenticatedUser setupAuthenticatedUserQueryWithResult(AuthenticatedUser authenticatedUser) { + TypedQuery queryStub = Mockito.mock(TypedQuery.class); + AuthenticatedUserLookup lookupStub = Mockito.mock(AuthenticatedUserLookup.class); + Mockito.when(lookupStub.getAuthenticatedUser()).thenReturn(authenticatedUser); + Mockito.when(queryStub.getSingleResult()).thenReturn(lookupStub); + Mockito.when(sut.em.createNamedQuery("AuthenticatedUserLookup.findByAuthPrvID_PersUserId", AuthenticatedUserLookup.class)).thenReturn(queryStub); + return authenticatedUser; + } + + private void setupAuthenticatedUserQueryWithNoResult() { + TypedQuery queryStub = Mockito.mock(TypedQuery.class); + Mockito.when(queryStub.getSingleResult()).thenThrow(new NoResultException()); + Mockito.when(sut.em.createNamedQuery("AuthenticatedUserLookup.findByAuthPrvID_PersUserId", AuthenticatedUserLookup.class)).thenReturn(queryStub); + } + + private void setUpOIDCProviderWhichValidatesToken() throws ParseException, IOException, OAuth2Exception { + OIDCAuthProvider oidcAuthProviderStub = stubOIDCAuthProvider("OIDC"); + + BearerAccessToken token = BearerAccessToken.parse(TEST_BEARER_TOKEN); + + // Stub the UserInfo returned by the provider + UserInfo userInfoStub = Mockito.mock(UserInfo.class); + Mockito.when(oidcAuthProviderStub.getUserInfo(token)).thenReturn(Optional.of(userInfoStub)); + + // Stub OAuth2UserRecord and its associated UserRecordIdentifier + OAuth2UserRecord oAuth2UserRecordStub = Mockito.mock(OAuth2UserRecord.class); + UserRecordIdentifier userRecordIdentifierStub = Mockito.mock(UserRecordIdentifier.class); + Mockito.when(userRecordIdentifierStub.getUserIdInRepo()).thenReturn("testUserId"); + Mockito.when(userRecordIdentifierStub.getUserRepoId()).thenReturn("testRepoId"); + Mockito.when(oAuth2UserRecordStub.getUserRecordIdentifier()).thenReturn(userRecordIdentifierStub); + + // Stub the OIDCAuthProvider to return OAuth2UserRecord + Mockito.when(oidcAuthProviderStub.getUserRecord(userInfoStub)).thenReturn(oAuth2UserRecordStub); + } + + private OIDCAuthProvider stubOIDCAuthProvider(String providerID) { + OIDCAuthProvider oidcAuthProviderStub = Mockito.mock(OIDCAuthProvider.class); + Mockito.when(oidcAuthProviderStub.getId()).thenReturn(providerID); + Mockito.when(sut.authProvidersRegistrationService.getAuthenticationProvidersMap()).thenReturn(Map.of(providerID, oidcAuthProviderStub)); + return oidcAuthProviderStub; + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java index ee6823ef98a..58b792691b9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java @@ -7,7 +7,6 @@ import edu.harvard.iq.dataverse.api.auth.BearerTokenAuthMechanism; import edu.harvard.iq.dataverse.api.auth.doubles.BearerTokenKeyContainerRequestTestFake; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; -import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier; import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2Exception; import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -38,16 +37,12 @@ import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import static edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactoryIT.clientId; import static edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactoryIT.clientSecret; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.mockito.Mockito.when; @@ -143,7 +138,7 @@ void testCreateProvider() throws Exception { /** * This test covers using an OIDC provider as authorization party when accessing the Dataverse API with a - * Bearer Token. See {@link BearerTokenAuthMechanism}. It needs to mock the auth services to avoid adding + * Bearer Token. See {@link BearerTokenAuthMechanism}. It needs to mock the auth service to avoid adding * more dependencies. */ @Test @@ -158,19 +153,15 @@ void testApiBearerAuth() throws Exception { String accessToken = getBearerTokenViaKeycloakAdminClient(); assumeFalse(accessToken == null); - OIDCAuthProvider oidcAuthProvider = getProvider(); // This will also receive the details from the remote Keycloak in the container - UserRecordIdentifier identifier = oidcAuthProvider.getUserIdentifier(new BearerAccessToken(accessToken)).get(); String token = "Bearer " + accessToken; BearerTokenKeyContainerRequestTestFake request = new BearerTokenKeyContainerRequestTestFake(token); AuthenticatedUser user = new MockAuthenticatedUser(); // setup mocks (we don't want or need a database here) - when(authService.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Set.of(oidcAuthProvider.getId())); - when(authService.getAuthenticationProvider(oidcAuthProvider.getId())).thenReturn(oidcAuthProvider); - when(authService.lookupUser(identifier)).thenReturn(user); + when(authService.lookupUserByOIDCBearerToken(token)).thenReturn(user); when(userService.updateLastApiUseTime(user)).thenReturn(user); - + // when (let's do this again, but now with the actual subject under test!) User lookedUpUser = bearerTokenAuthMechanism.findUserFromRequest(request); diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUserTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUserTest.java index 7bd802b3b02..bd3bfcc1a60 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUserTest.java @@ -378,20 +378,20 @@ public void testMutingInJson() { public void testHasEmailMuted() { testUser.setMutedEmails(mutedTypes); System.out.println("hasEmailMuted"); - assertEquals(true, testUser.hasEmailMuted(Type.ASSIGNROLE)); - assertEquals(true, testUser.hasEmailMuted(Type.REVOKEROLE)); - assertEquals(false, testUser.hasEmailMuted(Type.CREATEDV)); - assertEquals(false, testUser.hasEmailMuted(null)); + assertTrue(testUser.hasEmailMuted(Type.ASSIGNROLE)); + assertTrue(testUser.hasEmailMuted(Type.REVOKEROLE)); + assertFalse(testUser.hasEmailMuted(Type.CREATEDV)); + assertFalse(testUser.hasEmailMuted(null)); } @Test public void testHasNotificationsMutedMuted() { testUser.setMutedNotifications(mutedTypes); System.out.println("hasNotificationMuted"); - assertEquals(true, testUser.hasNotificationMuted(Type.ASSIGNROLE)); - assertEquals(true, testUser.hasNotificationMuted(Type.REVOKEROLE)); - assertEquals(false, testUser.hasNotificationMuted(Type.CREATEDV)); - assertEquals(false, testUser.hasNotificationMuted(null)); + assertTrue(testUser.hasNotificationMuted(Type.ASSIGNROLE)); + assertTrue(testUser.hasNotificationMuted(Type.REVOKEROLE)); + assertFalse(testUser.hasNotificationMuted(Type.CREATEDV)); + assertFalse(testUser.hasNotificationMuted(null)); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUserTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUserTest.java index a8dda2f6a7e..d3c5cdca470 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUserTest.java @@ -38,7 +38,7 @@ void getIdentifier() { @Test void testGetDisplayInfo() { RoleAssigneeDisplayInfo displayInfo = privateUrlUser.getDisplayInfo(); - assertEquals("Private URL Enabled", displayInfo.getTitle()); + assertEquals("Preview URL Enabled", displayInfo.getTitle()); assertNull(displayInfo.getEmailAddress()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIOTest.java index 552d76b74e8..ea5cc4b66a8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIOTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIOTest.java @@ -89,20 +89,20 @@ public void tearDownClass() throws IOException { */ @Test public void testOpen() throws IOException { - assertEquals(false, datasetAccess.canRead()); - assertEquals(false, datasetAccess.canWrite()); + assertFalse(datasetAccess.canRead()); + assertFalse(datasetAccess.canWrite()); datasetAccess.open(DataAccessOption.READ_ACCESS); - assertEquals(true, datasetAccess.canRead()); - assertEquals(false, datasetAccess.canWrite()); + assertTrue(datasetAccess.canRead()); + assertFalse(datasetAccess.canWrite()); datasetAccess.open(DataAccessOption.WRITE_ACCESS); - assertEquals(false, datasetAccess.canRead()); - assertEquals(true, datasetAccess.canWrite()); + assertFalse(datasetAccess.canRead()); + assertTrue(datasetAccess.canWrite()); dataFileAccess.open(DataAccessOption.READ_ACCESS); - assertEquals(true, dataFileAccess.canRead()); - assertEquals(false, dataFileAccess.canWrite()); + assertTrue(dataFileAccess.canRead()); + assertFalse(dataFileAccess.canWrite()); } /** @@ -133,7 +133,7 @@ public void testOpenAuxChannel() throws IOException { */ @Test public void testIsAuxObjectCached() throws IOException { - assertEquals(true, datasetAccess.isAuxObjectCached("Dataset")); + assertTrue(datasetAccess.isAuxObjectCached("Dataset")); } /** diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java index 2c0e0a5c6b7..c57fa71a340 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java @@ -41,7 +41,7 @@ public class RemoteOverlayAccessIOTest { public void setUp() { System.setProperty("dataverse.files.test.type", "remote"); System.setProperty("dataverse.files.test.label", "testOverlay"); - System.setProperty("dataverse.files.test.base-url", "https://demo.dataverse.org/resources"); + System.setProperty("dataverse.files.test.base-url", "https://data.qdr.syr.edu/resources"); System.setProperty("dataverse.files.test.base-store", "file"); System.setProperty("dataverse.files.test.download-redirect", "true"); System.setProperty("dataverse.files.test.remote-store-name", "DemoDataCorp"); diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java index 84a241b90f6..3aab66dc63b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java @@ -189,48 +189,48 @@ public void testResponseHeaders() { @Test public void testFileLocation() { - assertEquals(true, instance.isLocalFile()); + assertTrue(instance.isLocalFile()); instance.setIsLocalFile(false); - assertEquals(false, instance.isLocalFile()); + assertFalse(instance.isLocalFile()); - assertEquals(false, instance.isRemoteAccess()); + assertFalse(instance.isRemoteAccess()); instance.setIsRemoteAccess(true); - assertEquals(true, instance.isRemoteAccess()); + assertTrue(instance.isRemoteAccess()); } @Test public void testHttpAccess() { - assertEquals(false, instance.isHttpAccess()); + assertFalse(instance.isHttpAccess()); instance.setIsHttpAccess(true); - assertEquals(true, instance.isHttpAccess()); + assertTrue(instance.isHttpAccess()); }*/ @Test public void testDownloadSupported() { - assertEquals(true, instance.isDownloadSupported()); + assertTrue(instance.isDownloadSupported()); instance.setIsDownloadSupported(false); - assertEquals(false, instance.isDownloadSupported()); + assertFalse(instance.isDownloadSupported()); } @Test public void testSubsetSupported() { - assertEquals(false, instance.isSubsetSupported()); + assertFalse(instance.isSubsetSupported()); instance.setIsSubsetSupported(true); - assertEquals(true, instance.isSubsetSupported()); + assertTrue(instance.isSubsetSupported()); } @Test public void testZippedStream() { - assertEquals(false, instance.isZippedStream()); + assertFalse(instance.isZippedStream()); instance.setIsZippedStream(true); - assertEquals(true, instance.isZippedStream()); + assertTrue(instance.isZippedStream()); } @Test public void testNoVarHeader() { - assertEquals(false, instance.noVarHeader()); + assertFalse(instance.noVarHeader()); instance.setNoVarHeader(true); - assertEquals(true, instance.noVarHeader()); + assertTrue(instance.noVarHeader()); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIOTest.java index 942e4329384..27e0ac758e0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIOTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIOTest.java @@ -51,14 +51,14 @@ public void setUpClass() throws IOException { */ @Test public void testPerms() throws IOException { - assertEquals(false, datasetAccess.canRead()); - assertEquals(false, datasetAccess.canWrite()); + assertFalse(datasetAccess.canRead()); + assertFalse(datasetAccess.canWrite()); } @Test public void testIsExpiryExpired() { long currentTime = 1502221467; - assertEquals(false, swiftAccess.isExpiryExpired(60, 1502281, currentTime)); + assertFalse(swiftAccess.isExpiryExpired(60, 1502281, currentTime)); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java index eb19f22df63..148d34dc5f7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java @@ -18,6 +18,7 @@ import org.apache.http.message.BasicStatusLine; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; import org.junit.jupiter.api.Test; public class DataCaptureModuleUtilTest { @@ -25,13 +26,13 @@ public class DataCaptureModuleUtilTest { @Test public void testRsyncSupportEnabled() { System.out.println("rsyncSupportEnabled"); - assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled(null)); - assertEquals(true, DataCaptureModuleUtil.rsyncSupportEnabled("dcm/rsync+ssh")); + assertFalse(DataCaptureModuleUtil.rsyncSupportEnabled(null)); + assertTrue(DataCaptureModuleUtil.rsyncSupportEnabled("dcm/rsync+ssh")); // Comma sepratated lists of upload methods are supported. - assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("native/http:dcm/rsync+ssh")); - assertEquals(true, DataCaptureModuleUtil.rsyncSupportEnabled("native/http,dcm/rsync+ssh")); - assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("native/http")); - assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("junk")); + assertFalse(DataCaptureModuleUtil.rsyncSupportEnabled("native/http:dcm/rsync+ssh")); + assertTrue(DataCaptureModuleUtil.rsyncSupportEnabled("native/http,dcm/rsync+ssh")); + assertFalse(DataCaptureModuleUtil.rsyncSupportEnabled("native/http")); + assertFalse(DataCaptureModuleUtil.rsyncSupportEnabled("junk")); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/dataset/DatasetUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/dataset/DatasetUtilTest.java index 8eed2a33c5a..2db8851c48a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataset/DatasetUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataset/DatasetUtilTest.java @@ -70,8 +70,8 @@ public void testGetThumbnailRestricted() { */ @Test public void testDeleteDatasetLogo() { - assertEquals(false, DatasetUtil.deleteDatasetLogo(null)); - assertEquals(false, DatasetUtil.deleteDatasetLogo(new Dataset())); + assertFalse(DatasetUtil.deleteDatasetLogo(null)); + assertFalse(DatasetUtil.deleteDatasetLogo(new Dataset())); } /** @@ -106,7 +106,7 @@ public void testGetThumbnailAsInputStream() { @Test public void testIsDatasetLogoPresent() { Dataset dataset = MocksFactory.makeDataset(); - assertEquals(false, DatasetUtil.isDatasetLogoPresent(dataset, ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE)); + assertFalse(DatasetUtil.isDatasetLogoPresent(dataset, ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE)); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/datavariable/VariableMetadataDDIParserTest.java b/src/test/java/edu/harvard/iq/dataverse/datavariable/VariableMetadataDDIParserTest.java index bfb9134cfca..475b4c1cff5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datavariable/VariableMetadataDDIParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/datavariable/VariableMetadataDDIParserTest.java @@ -66,16 +66,16 @@ void variableTest(Map vmMap) { assertEquals(vm.getLiteralquestion(), "This is a literal question."); assertEquals(vm.getNotes(), "These are notes.\nA lot of them."); assertEquals(vm.getUniverse(),"Our universe"); - assertEquals(false, vm.isIsweightvar()); - assertEquals(false, vm.isWeighted()); + assertFalse(vm.isIsweightvar()); + assertFalse(vm.isWeighted()); testCategoriesVar1(vm); vm = vmMap.get(1169L); assertNotNull(vm); - assertEquals(false, vm.isIsweightvar()); - assertEquals(true, vm.isWeighted()); + assertFalse(vm.isIsweightvar()); + assertTrue(vm.isWeighted()); assertEquals(vm.getLabel(), "age_rollup" ); assertEquals(vm.getInterviewinstruction(), null); @@ -90,8 +90,8 @@ void variableTest(Map vmMap) { vm = vmMap.get(1168L); assertNotNull(vm); - assertEquals(true, vm.isIsweightvar()); - assertEquals(false, vm.isWeighted()); + assertTrue(vm.isIsweightvar()); + assertFalse(vm.isWeighted()); assertEquals(vm.getLabel(), "weight" ); assertEquals(vm.getInterviewinstruction(), null); assertEquals(vm.getLiteralquestion(), "Literal question for weight"); diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java index f2c03adea20..b4b9c0d33f2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java @@ -8,6 +8,7 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailServiceBean; import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; +import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; @@ -234,7 +235,12 @@ public ActionLogServiceBean actionLog() { public MetadataBlockServiceBean metadataBlocks() { return null; } - + + @Override + public DatasetTypeServiceBean datasetTypes() { + return null; + } + @Override public StorageUseServiceBean storageUse() { return null; diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java new file mode 100644 index 00000000000..f49ebcea39c --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java @@ -0,0 +1,264 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit; +import edu.harvard.iq.dataverse.util.JhoveFileType; +import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; +import org.jetbrains.annotations.NotNull; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.text.MessageFormat; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static edu.harvard.iq.dataverse.DataFile.ChecksumType.MD5; +import static org.apache.commons.io.file.FilesUncheck.createDirectories; +import static org.apache.commons.io.file.PathUtils.deleteDirectory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; + + +@LocalJvmSettings +public class CreateNewDataFilesTest { + // TODO keep constants for annotations in sync with class name + Path testDir = Path.of("target/test/").resolve(getClass().getSimpleName()); + PrintStream original_stderr; + + @BeforeEach + public void cleanTmpDir() throws IOException { + original_stderr = System.err; + if(testDir.toFile().exists()) + deleteDirectory(testDir); + } + + @AfterEach void restoreStderr() { + System.setErr(original_stderr); + } + + @Test + @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp") + public void execute_fails_to_upload_when_tmp_does_not_exist() throws FileNotFoundException { + + mockTmpLookup(); + var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion(), 1000L, 500L); + var ctxt = mockCommandContext(mockSysConfig(true, 0L, MD5, 10)); + + assertThatThrownBy(() -> cmd.execute(ctxt)) + .isInstanceOf(CommandException.class) + .hasMessageContaining("Failed to save the upload as a temp file (temp disk space?)") + .hasRootCauseInstanceOf(NoSuchFileException.class) + .getRootCause() + .hasMessageStartingWith("target/test/CreateNewDataFilesTest/tmp/temp/tmp"); + } + + @Test + @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp") + public void execute_fails_on_size_limit() throws Exception { + createDirectories(Path.of("target/test/CreateNewDataFilesTest/tmp/temp")); + + mockTmpLookup(); + var cmd = createCmd("scripts/search/data/binary/3files.zip", mockDatasetVersion(), 1000L, 500L); + var ctxt = mockCommandContext(mockSysConfig(true, 50L, MD5, 0)); + try (var mockedStatic = Mockito.mockStatic(JhoveFileType.class)) { + mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf"); + + assertThatThrownBy(() -> cmd.execute(ctxt)) + .isInstanceOf(CommandException.class) + .hasMessage("This file size (462 B) exceeds the size limit of 50 B."); + } + } + + @Test + @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp") + public void execute_loads_individual_files_from_uploaded_zip() throws Exception { + var tempDir = testDir.resolve("tmp/temp"); + createDirectories(tempDir); + + mockTmpLookup(); + var cmd = createCmd("src/test/resources/own-cloud-downloads/greetings.zip", mockDatasetVersion(), 1000L, 500L); + var ctxt = mockCommandContext(mockSysConfig(false, 1000000L, MD5, 10)); + try (MockedStatic mockedStatic = Mockito.mockStatic(JhoveFileType.class)) { + mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf"); + + // the test + var result = cmd.execute(ctxt); + + assertThat(result.getErrors()).hasSize(0); + assertThat(result.getDataFiles().stream().map(dataFile -> + dataFile.getFileMetadata().getDirectoryLabel() + "/" + dataFile.getDisplayName() + )).containsExactlyInAnyOrder( + "DD-1576/goodbye.txt", "DD-1576/hello.txt" + ); + var storageIds = result.getDataFiles().stream().map(DataFile::getStorageIdentifier).toList(); + assertThat(tempDir.toFile().list()) + .containsExactlyInAnyOrderElementsOf(storageIds); + } + } + + @Test + @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp") + public void execute_rezips_sets_of_shape_files_from_uploaded_zip() throws Exception { + var tempDir = testDir.resolve("tmp/temp"); + createDirectories(tempDir); + + mockTmpLookup(); + var cmd = createCmd("src/test/resources/own-cloud-downloads/shapes.zip", mockDatasetVersion(), 1000L, 500L); + var ctxt = mockCommandContext(mockSysConfig(false, 100000000L, MD5, 10)); + try (var mockedJHoveFileType = Mockito.mockStatic(JhoveFileType.class)) { + mockedJHoveFileType.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf"); + + // the test + var result = cmd.execute(ctxt); + + assertThat(result.getErrors()).hasSize(0); + assertThat(result.getDataFiles().stream().map(dataFile -> + (dataFile.getFileMetadata().getDirectoryLabel() + "/" + dataFile.getDisplayName()) + .replaceAll(".*/dataDir/", "") + )).containsExactlyInAnyOrder( + "shape1.zip", + "shape2/shape2", + "shape2/shape2.pdf", + "shape2/shape2.txt", + "shape2/shape2.zip", + "extra/shp_dictionary.xls", + "extra/notes", + "extra/README.MD" + ); + var storageIds = result.getDataFiles().stream().map(DataFile::getStorageIdentifier).toList(); + assertThat(tempDir.toFile().list()) + .containsExactlyInAnyOrderElementsOf(storageIds); + } + } + + @Disabled("Too slow. Intended for manual execution.") + @Test + @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "/tmp/test/CreateNewDataFilesTest/tmp") + public void extract_zip_performance() throws Exception { + /* + Developed to test performance difference between the old implementation with ZipInputStream and the new ZipFile implementation. + Play with numbers depending on: + - the time you want to spend on this test + - how much system stress you want to examine + */ + var nrOfZipFiles = 20; + var avgNrOfFilesPerZip = 300; + var avgFileLength = 5000; + + var tmpUploadStorage = Path.of("/tmp/test/CreateNewDataFilesTest/tmp/temp"); + if(tmpUploadStorage.toFile().exists()) { + deleteDirectory(tmpUploadStorage); + } + createDirectories(tmpUploadStorage); // temp in target would choke intellij + + var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + var random = new SecureRandom(); + var totalNrOfFiles = 0; + var totalFileSize = 0; + var totalTime = 0L; + var tmp = Path.of(Files.createTempDirectory(null).toString()); + var ctxt = mockCommandContext(mockSysConfig(false, 100000000L, MD5, 10000)); + try (var mockedJHoveFileType = Mockito.mockStatic(JhoveFileType.class)) { + mockedJHoveFileType.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf"); + for (var zipNr = 1; zipNr <= nrOfZipFiles; zipNr++) { + // build the zip + var zip = tmp.resolve(zipNr + "-data.zip"); + var nrOfFilesInZip = random.nextInt(avgNrOfFilesPerZip * 2); + try (var zipStream = new ZipOutputStream(new FileOutputStream(zip.toFile()))) { + for (var fileInZipNr = 1; fileInZipNr <= nrOfFilesInZip; fileInZipNr++) { + // build content for a file + var stringLength = random.nextInt(avgFileLength * 2 -5); + StringBuilder sb = new StringBuilder(stringLength); + for (int i = 1; i <= stringLength; i++) {// zero length causes buffer underflow + sb.append(chars.charAt(random.nextInt(chars.length()))); + } + // add the file to the zip + zipStream.putNextEntry(new ZipEntry(fileInZipNr + ".txt")); + zipStream.write((sb.toString()).getBytes()); + zipStream.closeEntry(); + totalFileSize += stringLength; + } + } + + // upload the zip + var before = DateTime.now(); + var result = createCmd(zip.toString(), mockDatasetVersion(), 1000L, 500L) + .execute(ctxt); + totalTime += DateTime.now().getMillis() - before.getMillis(); + + assertThat(result.getErrors()).hasSize(0); + assertThat(result.getDataFiles()).hasSize(nrOfFilesInZip); + totalNrOfFiles += nrOfFilesInZip; + + // report after each zip to have some data even when aborting a test that takes too long + System.out.println(MessageFormat.format( + "Total time: {0}ms; nr of zips {1} total nr of files {2}; total file size {3}", + totalTime, zipNr, totalNrOfFiles, totalFileSize + )); + } + assertThat(tmpUploadStorage.toFile().list()).hasSize(totalNrOfFiles); + } + } + + private static @NotNull CreateNewDataFilesCommand createCmd(String name, DatasetVersion dsVersion, long allocatedQuotaLimit, long usedQuotaLimit) throws FileNotFoundException { + return new CreateNewDataFilesCommand( + Mockito.mock(DataverseRequest.class), + dsVersion, + new FileInputStream(name), + "example.zip", + "application/zip", + null, + new UploadSessionQuotaLimit(allocatedQuotaLimit, usedQuotaLimit), + "sha"); + } + + private static @NotNull CommandContext mockCommandContext(SystemConfig sysCfg) { + var ctxt = Mockito.mock(CommandContext.class); + Mockito.when(ctxt.systemConfig()).thenReturn(sysCfg); + return ctxt; + } + + private static @NotNull SystemConfig mockSysConfig(boolean isStorageQuataEnforced, long maxFileUploadSizeForStore, DataFile.ChecksumType checksumType, int zipUploadFilesLimit) { + var sysCfg = Mockito.mock(SystemConfig.class); + Mockito.when(sysCfg.isStorageQuotasEnforced()).thenReturn(isStorageQuataEnforced); + Mockito.when(sysCfg.getMaxFileUploadSizeForStore(any())).thenReturn(maxFileUploadSizeForStore); + Mockito.when(sysCfg.getFileFixityChecksumAlgorithm()).thenReturn(checksumType); + Mockito.when(sysCfg.getZipUploadFilesLimit()).thenReturn(zipUploadFilesLimit); + return sysCfg; + } + + private static void mockTmpLookup() { + JvmSettings mockFilesDirectory = Mockito.mock(JvmSettings.class); + Mockito.when(mockFilesDirectory.lookup()).thenReturn("/mocked/path"); + } + + private static @NotNull DatasetVersion mockDatasetVersion() { + var dsVersion = Mockito.mock(DatasetVersion.class); + Mockito.when(dsVersion.getDataset()).thenReturn(Mockito.mock(Dataset.class)); + return dsVersion; + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java index 508eac46cb4..0ba29f74774 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java @@ -18,7 +18,9 @@ import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.search.SolrIndexServiceBean; import edu.harvard.iq.dataverse.util.SystemConfig; +import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.concurrent.Future; @@ -171,9 +173,9 @@ public void testCreatePrivateUrlSuccessfully() throws CommandException { assertEquals(expectedUser.getIdentifier(), privateUrl.getRoleAssignment().getAssigneeIdentifier()); assertEquals(expectedUser.isSuperuser(), false); assertEquals(expectedUser.isAuthenticated(), false); - assertEquals(expectedUser.getDisplayInfo().getTitle(), "Private URL Enabled"); + assertEquals(expectedUser.getDisplayInfo().getTitle(), "Preview URL Enabled"); assertNotNull(privateUrl.getToken()); - assertEquals("https://dataverse.example.edu/privateurl.xhtml?token=" + privateUrl.getToken(), privateUrl.getLink()); + assertEquals("https://dataverse.example.edu/previewurl.xhtml?token=" + privateUrl.getToken(), privateUrl.getLink()); } @Test @@ -188,22 +190,24 @@ public void testCreateAnonymizedAccessPrivateUrlSuccessfully() throws CommandExc assertEquals(expectedUser.getIdentifier(), privateUrl.getRoleAssignment().getAssigneeIdentifier()); assertEquals(expectedUser.isSuperuser(), false); assertEquals(expectedUser.isAuthenticated(), false); - assertEquals(expectedUser.getDisplayInfo().getTitle(), "Private URL Enabled"); + assertEquals(expectedUser.getDisplayInfo().getTitle(), "Preview URL Enabled"); assertNotNull(privateUrl.getToken()); assertTrue(privateUrl.isAnonymizedAccess()); - assertEquals("https://dataverse.example.edu/privateurl.xhtml?token=" + privateUrl.getToken(), privateUrl.getLink()); + assertEquals("https://dataverse.example.edu/previewurl.xhtml?token=" + privateUrl.getToken(), privateUrl.getLink()); } @Test - public void testAttemptCreateAnonymizedAccessPrivateUrlOnReleased() { + public void testAttemptCreateAnonymizedAccessPrivateUrlOnReleased() throws CommandException { dataset = new Dataset(); List versions = new ArrayList<>(); + dataset.setPublicationDate(new Timestamp(new Date().getTime())); DatasetVersion datasetVersion = new DatasetVersion(); datasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED); DatasetVersion datasetVersion2 = new DatasetVersion(); - - versions.add(datasetVersion); + datasetVersion2.setVersionState(DatasetVersion.VersionState.DRAFT); + versions.add(datasetVersion2); + versions.add(datasetVersion); dataset.setVersions(versions); dataset.setId(versionIsReleased); PrivateUrl privateUrl = null; @@ -211,6 +215,7 @@ public void testAttemptCreateAnonymizedAccessPrivateUrlOnReleased() { privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset, true)); assertTrue(false); } catch (CommandException ex) { + } assertNull(privateUrl); } diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommandTest.java index ed6112539ed..380a4bbcf18 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommandTest.java @@ -6,7 +6,10 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetLock; +import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseRoleServiceBean; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.Guestbook; @@ -14,6 +17,8 @@ import edu.harvard.iq.dataverse.GuestbookServiceBean; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.PermissionServiceBean; +import edu.harvard.iq.dataverse.RoleAssignment; +import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.RoleAssignee; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.GuestUser; @@ -25,6 +30,8 @@ import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import static edu.harvard.iq.dataverse.mocks.MocksFactory.makeAuthenticatedUser; +import static edu.harvard.iq.dataverse.mocks.MocksFactory.makeRole; +import static edu.harvard.iq.dataverse.mocks.MocksFactory.nextId; import edu.harvard.iq.dataverse.search.IndexServiceBean; import java.sql.Timestamp; import java.util.ArrayList; @@ -48,6 +55,9 @@ import jakarta.persistence.metamodel.Metamodel; import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.core.Context; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.*; @@ -61,16 +71,17 @@ * @author skraffmi */ public class MoveDatasetCommandTest { - Dataset moved, movedResponses; - Dataverse root, childA, childB, grandchildAA, childDraft, grandchildBB; - DataverseEngine testEngine; - MetadataBlock blockA, blockB, blockC, blockD; - AuthenticatedUser auth, nobody; - Guestbook gbA, gbB, gbC; - GuestbookResponse gbResp; - @Context - protected HttpServletRequest httpRequest; - + + Dataset moved, movedResponses, movedPerms, movedSamePerms; + Dataverse root, childA, childB, grandchildAA, childDraft, grandchildBB, childEditor, sibEditor; + DataverseEngine testEngine; + MetadataBlock blockA, blockB, blockC, blockD; + AuthenticatedUser auth, nobody; + Guestbook gbA, gbB, gbC; + GuestbookResponse gbResp; + @Context + protected HttpServletRequest httpRequest; + @BeforeEach public void setUp() { @@ -79,43 +90,60 @@ public void setUp() { nobody = makeAuthenticatedUser("Nick", "Nobody"); nobody.setSuperuser(false); - - root = new Dataverse(); root.setName("root"); root.setId(1l); root.setPublicationDate(new Timestamp(new Date().getTime())); - + root.setDefaultContributorRole(roles.findBuiltinRoleByAlias(DataverseRole.CURATOR)); + childA = new Dataverse(); childA.setName("childA"); childA.setId(2l); childA.setPublicationDate(new Timestamp(new Date().getTime())); - + childB = new Dataverse(); childB.setName("childB"); childB.setId(3l); - childB.setPublicationDate(new Timestamp(new Date().getTime())); - + childB.setPublicationDate(new Timestamp(new Date().getTime())); + grandchildAA = new Dataverse(); grandchildAA.setName("grandchildAA"); grandchildAA.setId(4l); grandchildAA.setPublicationDate(new Timestamp(new Date().getTime())); - + childDraft = new Dataverse(); childDraft.setName("childDraft"); childDraft.setId(5l); - + grandchildBB = new Dataverse(); grandchildBB.setName("grandchildBB"); grandchildBB.setId(6l); grandchildBB.setPublicationDate(new Timestamp(new Date().getTime())); - + childEditor = new Dataverse(); + childEditor.setName("childEditor"); + childEditor.setId(7l); + childEditor.setDefaultContributorRole(roles.findBuiltinRoleByAlias(DataverseRole.EDITOR)); + + sibEditor = new Dataverse(); + sibEditor.setName("sibEditor"); + sibEditor.setId(8l); + sibEditor.setDefaultContributorRole(roles.findBuiltinRoleByAlias(DataverseRole.EDITOR)); + + movedPerms = new Dataset(); + movedPerms.setOwner(childEditor); + DatasetLock lock = new DatasetLock(DatasetLock.Reason.InReview, nobody, null); + movedPerms.addLock(lock); + + movedSamePerms = new Dataset(); + movedSamePerms.setOwner(childEditor); + movedSamePerms.addLock(lock); + moved = new Dataset(); moved.setOwner(root); moved.setPublicationDate(new Timestamp(new Date().getTime())); moved.setId(1l); - + movedResponses = new Dataset(); movedResponses.setOwner(root); movedResponses.setPublicationDate(new Timestamp(new Date().getTime())); @@ -126,39 +154,39 @@ public void setUp() { grandchildAA.setOwner(childA); grandchildBB.setOwner(childA); childDraft.setOwner(childA); - - gbA= new Guestbook(); + + gbA = new Guestbook(); gbA.setId(1l); - gbB= new Guestbook(); + gbB = new Guestbook(); gbB.setId(2l); - gbC= new Guestbook(); + gbC = new Guestbook(); gbC.setId(3l); - + moved.setGuestbook(gbA); movedResponses.setGuestbook(gbA); - - GuestbookResponse gbResp = new GuestbookResponse(); + + GuestbookResponse gbResp = new GuestbookResponse(); gbResp.setGuestbook(gbA); gbResp.setDataset(movedResponses); - + List includeA = new ArrayList(); includeA.add(gbA); includeA.add(gbB); - + grandchildAA.setGuestbooks(includeA); - + List notIncludeA = new ArrayList(); notIncludeA.add(gbC); notIncludeA.add(gbB); - + childB.setGuestbooks(notIncludeA); - - List none = new ArrayList(); + + List none = new ArrayList(); root.setGuestbooks(none); grandchildBB.setGuestbooks(none); grandchildBB.setGuestbookRoot(false); childA.setGuestbooks(includeA); - + testEngine = new TestDataverseEngine(new TestCommandContext() { @Override public DataverseServiceBean dataverses() { @@ -170,31 +198,46 @@ public Dataverse save(Dataverse dataverse) { } }; } - + + @Override + public DatasetServiceBean datasets() { + return new DatasetServiceBean() { + @Override + public void removeDatasetLocks(Dataset dataset, DatasetLock.Reason aReason) { + new HashSet<>(dataset.getLocks()).stream() + .filter(l -> l.getReason() == aReason) + .forEach(lock -> { + dataset.removeLock(lock); + }); + + } + }; + } + @Override public GuestbookServiceBean guestbooks() { return new GuestbookServiceBean() { @Override public Long findCountResponsesForGivenDataset(Long guestbookId, Long datasetId) { //We're going to fake a response for a dataset with responses - if(datasetId == 1){ + if (datasetId == 1) { return new Long(0); - } else{ + } else { return new Long(1); } } }; } - + @Override - public IndexServiceBean index(){ - return new IndexServiceBean(){ + public IndexServiceBean index() { + return new IndexServiceBean() { @Override - public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp){ + public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) { } }; } - + @Override public EntityManager em() { return new MockEntityManager() { @@ -217,14 +260,55 @@ public boolean isUserAllowedOn(RoleAssignee roleAssignee, Command command, Dv } }; } - + }); } - - /** - * Moving ChildB to ChildA - * @throws Exception - should not throw an exception - */ + + DataverseRoleServiceBean roles = new DataverseRoleServiceBean() { + + List assignments = new LinkedList<>(); + + Map builtInRoles; + + { + builtInRoles = new HashMap<>(); + builtInRoles.put(DataverseRole.EDITOR, makeRole("default-editor", false)); + builtInRoles.put(DataverseRole.ADMIN, makeRole("default-admin")); + builtInRoles.put(DataverseRole.MANAGER, makeRole("default-manager")); + builtInRoles.put(DataverseRole.CURATOR, makeRole("curator")); + } + + @Override + public DataverseRole findBuiltinRoleByAlias(String alias) { + return builtInRoles.get(alias); + } + + @Override + public RoleAssignment save(RoleAssignment assignment) { + assignment.setId(nextId()); + assignments.add(assignment); + return assignment; + } + + @Override + public RoleAssignment save(RoleAssignment assignment, boolean index) { + return save(assignment); + } + + @Override + public List directRoleAssignments(DvObject dvo) { + // works since there's only one dataverse involved in the context + // of this unit test. + return assignments; + } + + }; + + /** + * Moving ChildB to ChildA + * + * @throws Exception - should not throw an exception + */ @Test public void testValidMove() throws Exception { @@ -234,11 +318,10 @@ public void testValidMove() throws Exception { assertEquals(childA, moved.getOwner()); } - + /** - * Moving grandchildAA - * Guestbook is not null because target includes it. - */ + * Moving grandchildAA Guestbook is not null because target includes it. + */ @Test public void testKeepGuestbook() throws Exception { @@ -248,12 +331,10 @@ public void testKeepGuestbook() throws Exception { assertNotNull(moved.getGuestbook()); } - - /** - * Moving to grandchildBB - * Guestbook is not null because target inherits it. - */ - + + /** + * Moving to grandchildBB Guestbook is not null because target inherits it. + */ @Test public void testKeepGuestbookInherit() throws Exception { @@ -263,39 +344,53 @@ public void testKeepGuestbookInherit() throws Exception { assertNotNull(moved.getGuestbook()); } - - + /** - * Moving to ChildB - * Guestbook is null because target does not include it - */ + * Moving to ChildB Guestbook is null because target does not include it + */ @Test public void testRemoveGuestbook() throws Exception { DataverseRequest aRequest = new DataverseRequest(auth, httpRequest); testEngine.submit(new MoveDatasetCommand(aRequest, moved, childB, true)); - assertNull( moved.getGuestbook()); + assertNull(moved.getGuestbook()); } - - - /** - * Moving DS to its owning DV - * @throws IllegalCommandException - */ + + @Test + public void testMoveToDifferentPerms() throws Exception { + DataverseRequest aRequest = new DataverseRequest(auth, httpRequest); + testEngine.submit(new MoveDatasetCommand(aRequest, movedPerms, root, true)); + assertTrue(movedPerms.getLocks().isEmpty()); + assertTrue(movedPerms.getOwner().equals(root)); + } + + @Test + public void testMoveToSamePerms() throws Exception { + DataverseRequest aRequest = new DataverseRequest(auth, httpRequest); + testEngine.submit(new MoveDatasetCommand(aRequest, movedSamePerms, sibEditor, true)); + assertTrue(movedSamePerms.getLocks().size() == 1); + assertTrue(movedSamePerms.getOwner().equals(sibEditor)); + } + + /** + * Moving DS to its owning DV + * + * @throws IllegalCommandException + */ @Test void testInvalidMove() { DataverseRequest aRequest = new DataverseRequest(auth, httpRequest); assertThrows(IllegalCommandException.class, - () -> testEngine.submit(new MoveDatasetCommand(aRequest, moved, root, false))); + () -> testEngine.submit(new MoveDatasetCommand(aRequest, moved, root, false))); } - + /** * Moving a dataset without having enough permission fails with * PermissionException. * * @throws java.lang.Exception - * + * * Ignoring after permissions change in 47fb045. Did that change make this * case untestable? Unclear. */ @@ -305,7 +400,7 @@ void testAuthenticatedUserWithNoRole() { DataverseRequest aRequest = new DataverseRequest(nobody, httpRequest); assertThrows(IllegalCommandException.class, - () -> testEngine.submit(new MoveDatasetCommand(aRequest, moved, childA, null))); + () -> testEngine.submit(new MoveDatasetCommand(aRequest, moved, childA, null))); } /** @@ -319,22 +414,22 @@ void testNotAuthenticatedUser() { DataverseRequest aRequest = new DataverseRequest(GuestUser.get(), httpRequest); assertThrows(PermissionException.class, - () -> testEngine.submit(new MoveDatasetCommand(aRequest, moved, root, null))); + () -> testEngine.submit(new MoveDatasetCommand(aRequest, moved, root, null))); } - - /** - * Moving published DS to unpublished DV - * @throws IllegalCommandException - */ + + /** + * Moving published DS to unpublished DV + * + * @throws IllegalCommandException + */ @Test void testInvalidMovePublishedToUnpublished() { DataverseRequest aRequest = new DataverseRequest(auth, httpRequest); assertThrows(IllegalCommandException.class, - () -> testEngine.submit(new MoveDatasetCommand(aRequest, moved, childDraft, null))); + () -> testEngine.submit(new MoveDatasetCommand(aRequest, moved, childDraft, null))); } - - - private static class EntityManagerImpl implements EntityManager { + + private static class EntityManagerImpl implements EntityManager { @Override public void persist(Object entity) { @@ -591,8 +686,8 @@ public List> getEntityGraphs(Class entityClass) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } - } - + } + private static class MockEntityManager extends EntityManagerImpl { @Override diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommandTest.java new file mode 100644 index 00000000000..3f6b3b0f393 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommandTest.java @@ -0,0 +1,371 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.api.dto.UserDTO; +import edu.harvard.iq.dataverse.authorization.AuthenticatedUserDisplayInfo; +import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; +import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier; +import edu.harvard.iq.dataverse.authorization.exceptions.AuthorizationException; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.engine.command.exception.InvalidFieldsCommandException; +import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.List; + +import static edu.harvard.iq.dataverse.mocks.MocksFactory.makeRequest; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Mockito.*; + +@LocalJvmSettings +class RegisterOIDCUserCommandTest { + + private static final String TEST_BEARER_TOKEN = "Bearer test"; + private static final String TEST_USERNAME = "username"; + private static final AuthenticatedUserDisplayInfo TEST_MISSING_CLAIMS_DISPLAY_INFO = new AuthenticatedUserDisplayInfo( + null, + null, + null, + "", + "" + ); + private static final AuthenticatedUserDisplayInfo TEST_VALID_DISPLAY_INFO = new AuthenticatedUserDisplayInfo( + "FirstName", + "LastName", + "user@example.com", + "", + "" + ); + + private UserDTO testUserDTO; + + @Mock + private CommandContext contextStub; + + @Mock + private AuthenticationServiceBean authServiceStub; + + @InjectMocks + private RegisterOIDCUserCommand sut; + + private OAuth2UserRecord oAuth2UserRecordStub; + private UserRecordIdentifier userRecordIdentifierMock; + private AuthenticatedUser existingTestUser; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + setUpDefaultUserDTO(); + + userRecordIdentifierMock = mock(UserRecordIdentifier.class); + oAuth2UserRecordStub = mock(OAuth2UserRecord.class); + existingTestUser = new AuthenticatedUser(); + + when(oAuth2UserRecordStub.getUserRecordIdentifier()).thenReturn(userRecordIdentifierMock); + when(contextStub.authentication()).thenReturn(authServiceStub); + + sut = new RegisterOIDCUserCommand(makeRequest(), TEST_BEARER_TOKEN, testUserDTO); + } + + private void setUpDefaultUserDTO() { + testUserDTO = new UserDTO(); + testUserDTO.setTermsAccepted(true); + testUserDTO.setFirstName("FirstName"); + testUserDTO.setLastName("LastName"); + testUserDTO.setUsername("username"); + testUserDTO.setEmailAddress("user@example.com"); + } + + @Test + public void execute_completedUserDTOWithUnacceptedTerms_missingClaimsInProvider_provideMissingClaimsFeatureFlagDisabled() throws AuthorizationException { + testUserDTO.setTermsAccepted(false); + testUserDTO.setEmailAddress(null); + testUserDTO.setUsername(null); + testUserDTO.setFirstName(null); + testUserDTO.setLastName(null); + + when(authServiceStub.getAuthenticatedUserByEmail(testUserDTO.getEmailAddress())).thenReturn(null); + when(authServiceStub.getAuthenticatedUser(testUserDTO.getUsername())).thenReturn(null); + when(authServiceStub.verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)).thenReturn(oAuth2UserRecordStub); + + when(oAuth2UserRecordStub.getUsername()).thenReturn(null); + when(oAuth2UserRecordStub.getDisplayInfo()).thenReturn(TEST_MISSING_CLAIMS_DISPLAY_INFO); + + assertThatThrownBy(() -> sut.execute(contextStub)) + .isInstanceOf(InvalidFieldsCommandException.class) + .satisfies(exception -> { + InvalidFieldsCommandException ex = (InvalidFieldsCommandException) exception; + assertThat(ex.getFieldErrors()) + .containsEntry("termsAccepted", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.userShouldAcceptTerms")) + .containsEntry("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.fieldRequired", List.of("emailAddress"))) + .containsEntry("username", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.fieldRequired", List.of("username"))) + .containsEntry("firstName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.fieldRequired", List.of("firstName"))) + .containsEntry("lastName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.fieldRequired", List.of("lastName"))); + }); + } + + @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + public void execute_uncompletedUserDTOWithUnacceptedTerms_missingClaimsInProvider_provideMissingClaimsFeatureFlagEnabled() throws AuthorizationException { + testUserDTO.setTermsAccepted(false); + testUserDTO.setEmailAddress(null); + testUserDTO.setUsername(null); + testUserDTO.setFirstName(null); + testUserDTO.setLastName(null); + + when(oAuth2UserRecordStub.getUsername()).thenReturn(null); + when(oAuth2UserRecordStub.getDisplayInfo()).thenReturn(TEST_MISSING_CLAIMS_DISPLAY_INFO); + + when(authServiceStub.getAuthenticatedUserByEmail(testUserDTO.getEmailAddress())).thenReturn(null); + when(authServiceStub.getAuthenticatedUser(testUserDTO.getUsername())).thenReturn(null); + when(authServiceStub.verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)).thenReturn(oAuth2UserRecordStub); + + assertThatThrownBy(() -> sut.execute(contextStub)) + .isInstanceOf(InvalidFieldsCommandException.class) + .satisfies(exception -> { + InvalidFieldsCommandException ex = (InvalidFieldsCommandException) exception; + assertThat(ex.getFieldErrors()) + .containsEntry("termsAccepted", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.userShouldAcceptTerms")) + .containsEntry("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldRequired", List.of("emailAddress"))) + .containsEntry("username", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldRequired", List.of("username"))) + .containsEntry("firstName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldRequired", List.of("firstName"))) + .containsEntry("lastName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldRequired", List.of("lastName"))); + }); + } + + @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + public void execute_acceptedTerms_unavailableEmailAndUsername_missingClaimsInProvider_provideMissingClaimsFeatureFlagEnabled() throws AuthorizationException { + when(authServiceStub.getAuthenticatedUserByEmail(testUserDTO.getEmailAddress())).thenReturn(existingTestUser); + when(authServiceStub.getAuthenticatedUser(testUserDTO.getUsername())).thenReturn(existingTestUser); + when(authServiceStub.verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)).thenReturn(oAuth2UserRecordStub); + + when(oAuth2UserRecordStub.getUsername()).thenReturn(null); + when(oAuth2UserRecordStub.getDisplayInfo()).thenReturn(TEST_MISSING_CLAIMS_DISPLAY_INFO); + + assertThatThrownBy(() -> sut.execute(contextStub)) + .isInstanceOf(InvalidFieldsCommandException.class) + .satisfies(exception -> { + InvalidFieldsCommandException ex = (InvalidFieldsCommandException) exception; + assertThat(ex.getFieldErrors()) + .containsEntry("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.emailAddressInUse")) + .containsEntry("username", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.usernameInUse")) + .doesNotContainKey("termsAccepted"); + }); + } + + @Test + void execute_throwsPermissionException_onAuthorizationException() throws AuthorizationException { + String testAuthorizationExceptionMessage = "Authorization failed"; + when(contextStub.authentication().verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)) + .thenThrow(new AuthorizationException(testAuthorizationExceptionMessage)); + + assertThatThrownBy(() -> sut.execute(contextStub)) + .isInstanceOf(PermissionException.class) + .hasMessageContaining(testAuthorizationExceptionMessage); + + verify(contextStub.authentication(), times(1)).verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN); + } + + @Test + void execute_throwsIllegalCommandException_ifUserAlreadyRegisteredWithToken() throws AuthorizationException { + when(contextStub.authentication().verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)) + .thenReturn(oAuth2UserRecordStub); + when(contextStub.authentication().lookupUser(userRecordIdentifierMock)).thenReturn(new AuthenticatedUser()); + + assertThatThrownBy(() -> sut.execute(contextStub)) + .isInstanceOf(IllegalCommandException.class) + .hasMessageContaining(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.userAlreadyRegisteredWithToken")); + + verify(contextStub.authentication(), times(1)).lookupUser(userRecordIdentifierMock); + } + + @Test + void execute_throwsInvalidFieldsCommandException_ifUserDTOHasClaimsAndProvideMissingClaimsFeatureFlagIsDisabled() throws AuthorizationException { + when(contextStub.authentication().verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)) + .thenReturn(oAuth2UserRecordStub); + + assertThatThrownBy(() -> sut.execute(contextStub)) + .isInstanceOf(InvalidFieldsCommandException.class) + .satisfies(exception -> { + InvalidFieldsCommandException ex = (InvalidFieldsCommandException) exception; + assertThat(ex.getFieldErrors()) + .containsEntry("username", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.unableToSetFieldViaJSON", List.of("username"))) + .containsEntry("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.unableToSetFieldViaJSON", List.of("emailAddress"))) + .containsEntry("firstName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.unableToSetFieldViaJSON", List.of("firstName"))) + .containsEntry("lastName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.unableToSetFieldViaJSON", List.of("lastName"))) + .containsEntry("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.unableToSetFieldViaJSON", List.of("emailAddress"))); + }); + } + + @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + void execute_happyPath_withoutAffiliationAndPosition_missingClaimsInProvider_provideMissingClaimsFeatureFlagEnabled() throws AuthorizationException, CommandException { + when(authServiceStub.verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)).thenReturn(oAuth2UserRecordStub); + + when(oAuth2UserRecordStub.getUsername()).thenReturn(null); + when(oAuth2UserRecordStub.getDisplayInfo()).thenReturn(TEST_MISSING_CLAIMS_DISPLAY_INFO); + + sut.execute(contextStub); + + verify(authServiceStub, times(1)).createAuthenticatedUser( + eq(userRecordIdentifierMock), + eq(testUserDTO.getUsername()), + eq(new AuthenticatedUserDisplayInfo( + testUserDTO.getFirstName(), + testUserDTO.getLastName(), + testUserDTO.getEmailAddress(), + "", + "") + ), + eq(true) + ); + } + + @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + void execute_happyPath_withAffiliationAndPosition_missingClaimsInProvider_provideMissingClaimsFeatureFlagEnabled() throws AuthorizationException, CommandException { + testUserDTO.setPosition("test position"); + testUserDTO.setAffiliation("test affiliation"); + + when(authServiceStub.verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)).thenReturn(oAuth2UserRecordStub); + + when(oAuth2UserRecordStub.getUsername()).thenReturn(null); + when(oAuth2UserRecordStub.getDisplayInfo()).thenReturn(TEST_MISSING_CLAIMS_DISPLAY_INFO); + + sut.execute(contextStub); + + verify(authServiceStub, times(1)).createAuthenticatedUser( + eq(userRecordIdentifierMock), + eq(testUserDTO.getUsername()), + eq(new AuthenticatedUserDisplayInfo( + testUserDTO.getFirstName(), + testUserDTO.getLastName(), + testUserDTO.getEmailAddress(), + testUserDTO.getAffiliation(), + testUserDTO.getPosition()) + ), + eq(true) + ); + } + + @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + void execute_conflictingClaimsInProvider_provideMissingClaimsFeatureFlagEnabled() throws AuthorizationException { + when(authServiceStub.verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)).thenReturn(oAuth2UserRecordStub); + + when(oAuth2UserRecordStub.getUsername()).thenReturn(TEST_USERNAME); + when(oAuth2UserRecordStub.getDisplayInfo()).thenReturn(TEST_VALID_DISPLAY_INFO); + + testUserDTO.setUsername("conflictingUsername"); + testUserDTO.setFirstName("conflictingFirstName"); + testUserDTO.setLastName("conflictingLastName"); + testUserDTO.setEmailAddress("conflictingemail@example.com"); + + assertThatThrownBy(() -> sut.execute(contextStub)) + .isInstanceOf(InvalidFieldsCommandException.class) + .satisfies(exception -> { + InvalidFieldsCommandException ex = (InvalidFieldsCommandException) exception; + assertThat(ex.getFieldErrors()) + .containsEntry("username", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldAlreadyPresentInProvider", List.of("username"))) + .containsEntry("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldAlreadyPresentInProvider", List.of("emailAddress"))) + .containsEntry("firstName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldAlreadyPresentInProvider", List.of("firstName"))) + .containsEntry("lastName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldAlreadyPresentInProvider", List.of("lastName"))) + .containsEntry("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldAlreadyPresentInProvider", List.of("emailAddress"))); + }); + } + + @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + void execute_happyPath_withoutAffiliationAndPosition_claimsInProvider_provideMissingClaimsFeatureFlagEnabled() throws AuthorizationException, CommandException { + testUserDTO.setTermsAccepted(true); + testUserDTO.setEmailAddress(null); + testUserDTO.setUsername(null); + testUserDTO.setFirstName(null); + testUserDTO.setLastName(null); + + when(authServiceStub.verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)).thenReturn(oAuth2UserRecordStub); + + when(oAuth2UserRecordStub.getUsername()).thenReturn(TEST_USERNAME); + when(oAuth2UserRecordStub.getDisplayInfo()).thenReturn(TEST_VALID_DISPLAY_INFO); + + sut.execute(contextStub); + + verify(authServiceStub, times(1)).createAuthenticatedUser( + eq(userRecordIdentifierMock), + eq(TEST_USERNAME), + eq(new AuthenticatedUserDisplayInfo( + TEST_VALID_DISPLAY_INFO.getFirstName(), + TEST_VALID_DISPLAY_INFO.getLastName(), + TEST_VALID_DISPLAY_INFO.getEmailAddress(), + "", + "") + ), + eq(true) + ); + } + + @ParameterizedTest + @ValueSource(strings = {" ", ""}) + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + void execute_happyPath_withoutAffiliationAndPosition_blankClaimInProviderProvidedInJson_provideMissingClaimsFeatureFlagEnabled(String testBlankUsername) throws AuthorizationException, CommandException { + String testUsernameNotBlank = "usernameNotBlank"; + testUserDTO.setUsername(testUsernameNotBlank); + testUserDTO.setTermsAccepted(true); + testUserDTO.setEmailAddress(null); + testUserDTO.setFirstName(null); + testUserDTO.setLastName(null); + + when(authServiceStub.verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)).thenReturn(oAuth2UserRecordStub); + + when(oAuth2UserRecordStub.getUsername()).thenReturn(testBlankUsername); + when(oAuth2UserRecordStub.getDisplayInfo()).thenReturn(TEST_VALID_DISPLAY_INFO); + + sut.execute(contextStub); + + verify(authServiceStub, times(1)).createAuthenticatedUser( + eq(userRecordIdentifierMock), + eq(testUsernameNotBlank), + eq(new AuthenticatedUserDisplayInfo( + TEST_VALID_DISPLAY_INFO.getFirstName(), + TEST_VALID_DISPLAY_INFO.getLastName(), + TEST_VALID_DISPLAY_INFO.getEmailAddress(), + "", + "") + ), + eq(true) + ); + } + + @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-handle-tos-acceptance-in-idp") + void execute_doNotThrowUnacceptedTermsError_unacceptedTermsInUserDTOAndAllClaimsInProvider_handleTosAcceptanceInIdpFeatureFlagEnabled() throws AuthorizationException { + testUserDTO.setTermsAccepted(false); + testUserDTO.setEmailAddress(null); + testUserDTO.setUsername(null); + testUserDTO.setFirstName(null); + testUserDTO.setLastName(null); + + when(authServiceStub.verifyOIDCBearerTokenAndGetOAuth2UserRecord(TEST_BEARER_TOKEN)).thenReturn(oAuth2UserRecordStub); + + when(oAuth2UserRecordStub.getUsername()).thenReturn(TEST_USERNAME); + when(oAuth2UserRecordStub.getDisplayInfo()).thenReturn(TEST_VALID_DISPLAY_INFO); + + assertDoesNotThrow(() -> sut.execute(contextStub)); + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java index 2da15147255..8350c5d9875 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java @@ -617,7 +617,7 @@ public void testWriteRelatedIdentifierElement() throws XMLStreamException, IOExc //then assertEquals("" - + "" + + "" + "RelatedPublicationIDNumber1" + "" + "RelatedPublicationIDNumber2" diff --git a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java index 2139589b4c3..2121aa4d9f9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.branding.BrandingUtilTest; +import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; import io.gdcc.spi.export.ExportDataProvider; import io.gdcc.spi.export.XMLExporter; import edu.harvard.iq.dataverse.license.License; @@ -53,6 +54,7 @@ public class SchemaDotOrgExporterTest { private static final MockDatasetFieldSvc datasetFieldTypeSvc = new MockDatasetFieldSvc(); private static final SettingsServiceBean settingsService = Mockito.mock(SettingsServiceBean.class); private static final LicenseServiceBean licenseService = Mockito.mock(LicenseServiceBean.class); + private static final DatasetTypeServiceBean datasetTypeService = Mockito.mock(DatasetTypeServiceBean.class); private static final SchemaDotOrgExporter schemaDotOrgExporter = new SchemaDotOrgExporter(); @BeforeAll @@ -173,7 +175,7 @@ public void testExportDescriptionTruncation() throws JsonParseException, ParseEx private JsonObject createExportFromJson(ExportDataProvider provider) throws JsonParseException, ParseException { License license = new License("CC0 1.0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0/"), URI.create("/resources/images/cc0.png"), true, 1l); license.setDefault(true); - JsonParser jsonParser = new JsonParser(datasetFieldTypeSvc, null, settingsService, licenseService); + JsonParser jsonParser = new JsonParser(datasetFieldTypeSvc, null, settingsService, licenseService, datasetTypeService); DatasetVersion version = jsonParser.parseDatasetVersion(provider.getDatasetJson().getJsonObject("datasetVersion")); version.setVersionState(DatasetVersion.VersionState.RELEASED); SimpleDateFormat dateFmt = new SimpleDateFormat("yyyyMMdd"); @@ -251,7 +253,7 @@ public void testGetDisplayName() { */ @Test public void testIsXMLFormat() { - assertEquals(false, schemaDotOrgExporter instanceof XMLExporter); + assertFalse(schemaDotOrgExporter instanceof XMLExporter); } /** @@ -259,7 +261,7 @@ public void testIsXMLFormat() { */ @Test public void testIsHarvestable() { - assertEquals(false, schemaDotOrgExporter.isHarvestable()); + assertFalse(schemaDotOrgExporter.isHarvestable()); } /** @@ -267,7 +269,7 @@ public void testIsHarvestable() { */ @Test public void testIsAvailableToUsers() { - assertEquals(true, schemaDotOrgExporter.isAvailableToUsers()); + assertTrue(schemaDotOrgExporter.isAvailableToUsers()); } /** diff --git a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt index 431f069cb03..6b3c7689bbf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt +++ b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt @@ -324,6 +324,12 @@ "typeClass": "compound", "value": [ { + "publicationRelationType": { + "typeName": "publicationRelationType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "IsSupplementTo" + }, "publicationCitation": { "typeName": "publicationCitation", "multiple": false, diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java index 41e6be61bb8..f594de4757d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java @@ -64,6 +64,23 @@ public void testJson2DdiNoFiles() throws Exception { XmlAssert.assertThat(result).and(datasetAsDdi).ignoreWhitespace().areSimilar(); } + @Test + public void testJson2DdiNoFilesTermsOfUse() throws Exception { + // given + Path datasetVersionJson = Path.of("src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch-terms-of-use.json"); + String datasetVersionAsJson = Files.readString(datasetVersionJson, StandardCharsets.UTF_8); + Path ddiFile = Path.of("src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch-terms-of-use.xml"); + String datasetAsDdi = XmlPrinter.prettyPrintXml(Files.readString(ddiFile, StandardCharsets.UTF_8)); + logger.fine(datasetAsDdi); + + // when + String result = DdiExportUtil.datasetDtoAsJson2ddi(datasetVersionAsJson); + logger.fine(result); + + // then + XmlAssert.assertThat(result).and(datasetAsDdi).ignoreWhitespace().areSimilar(); + } + @Test public void testExportDDI() throws Exception { // given diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch-terms-of-use.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch-terms-of-use.json new file mode 100644 index 00000000000..b3d6caff2e9 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch-terms-of-use.json @@ -0,0 +1,404 @@ +{ + "id": 11, + "identifier": "PCA2E3", + "persistentUrl": "https://doi.org/10.5072/FK2/PCA2E3", + "protocol": "doi", + "authority": "10.5072/FK2", + "metadataLanguage": "en", + "datasetVersion": { + "id": 2, + "versionNumber": 1, + "versionMinorNumber": 0, + "versionState": "RELEASED", + "productionDate": "Production Date", + "lastUpdateTime": "2015-09-24T17:07:57Z", + "releaseTime": "2015-09-24T17:07:57Z", + "createTime": "2015-09-24T16:47:51Z", + "termsOfUse":"This dataset is made available without information on how it can be used. You should communicate with the Contact(s) specified before use.", + "metadataBlocks": { + "citation": { + "displayName": "Citation Metadata", + "name":"citation", + "fields": [ + { + "typeName": "title", + "multiple": false, + "typeClass": "primitive", + "value": "Darwin's Finches" + }, + { + "typeName": "alternativeTitle", + "multiple": true, + "typeClass": "primitive", + "value": ["Darwin's Finches Alternative Title1", "Darwin's Finches Alternative Title2"] + }, + { + "typeName": "author", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "authorName": { + "typeName": "authorName", + "multiple": false, + "typeClass": "primitive", + "value": "Finch, Fiona" + }, + "authorAffiliation": { + "typeName": "authorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "Birds Inc." + } + } + ] + }, + { + "typeName": "timePeriodCovered", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "timePeriodStart": { + "typeName": "timePeriodCoveredStart", + "multiple": false, + "typeClass": "primitive", + "value": "20020816" + }, + "timePeriodEnd": { + "typeName": "timePeriodCoveredEnd", + "multiple": false, + "typeClass": "primitive", + "value": "20160630" + } + } + ] + }, + { + "typeName": "dateOfCollection", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "timePeriodStart": { + "typeName": "dateOfCollectionStart", + "multiple": false, + "typeClass": "primitive", + "value": "20070831" + }, + "timePeriodEnd": { + "typeName": "dateOfCollectionEnd", + "multiple": false, + "typeClass": "primitive", + "value": "20130630" + } + } + ] + }, + { + "typeName": "datasetContact", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "datasetContactEmail": { + "typeName": "datasetContactEmail", + "multiple": false, + "typeClass": "primitive", + "value": "finch@mailinator.com" + }, + "datasetContactName": { + "typeName": "datasetContactName", + "multiple": false, + "typeClass": "primitive", + "value": "Jimmy Finch" + }, + "datasetContactAffiliation": { + "typeName": "datasetContactAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "Finch Academy" + } + } + ] + }, + { + "typeName": "producer", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "producerAbbreviation": { + "typeName": "producerAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "ProdAbb" + }, + "producerName": { + "typeName": "producerName", + "multiple": false, + "typeClass": "primitive", + "value": "Johnny Hawk" + }, + "producerAffiliation": { + "typeName": "producerAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "Hawk Institute" + }, + "producerURL": { + "typeName": "producerURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://www.hawk.edu/url" + }, + "producerLogoURL": { + "typeName": "producerLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://www.hawk.edu/logo" + } + } + ] + }, + { + "typeName": "distributor", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "distributorAbbreviation": { + "typeName": "distributorAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "Dist-Abb" + }, + "producerName": { + "typeName": "distributorName", + "multiple": false, + "typeClass": "primitive", + "value": "Odin Raven" + }, + "distributorAffiliation": { + "typeName": "distributorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "Valhalla Polytechnic" + }, + "distributorURL": { + "typeName": "distributorURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://www.valhalla.edu/url" + }, + "distributorLogoURL": { + "typeName": "distributorLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://www.valhalla.edu/logo" + } + } + ] + }, + { + "typeName": "dsDescription", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "dsDescriptionValue": { + "typeName": "dsDescriptionValue", + "multiple": false, + "typeClass": "primitive", + "value": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds." + } + } + ] + }, + { + "typeName": "subject", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Medicine, Health and Life Sciences" + ] + }, + { + "typeName": "keyword", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "keywordValue": { + "typeName": "keywordValue", + "multiple": false, + "typeClass": "primitive", + "value": "Keyword Value 1" + }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, + "keywordVocabulary": { + "typeName": "keywordVocabulary", + "multiple": false, + "typeClass": "primitive", + "value": "Keyword Vocabulary" + }, + "keywordVocabularyURI": { + "typeName": "keywordVocabularyURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://www.keyword.com/one" + } + }, + { + "keywordValue": { + "typeName": "keywordValue", + "multiple": false, + "typeClass": "primitive", + "value": "Keyword Value Two" + }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, + "keywordVocabulary": { + "typeName": "keywordVocabulary", + "multiple": false, + "typeClass": "primitive", + "value": "Keyword Vocabulary" + }, + "keywordVocabularyURI": { + "typeName": "keywordVocabularyURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://www.keyword.com/one" + } + } + ] + }, + { + "typeName": "topicClassification", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "topicClassValue": { + "typeName": "topicClassValue", + "multiple": false, + "typeClass": "primitive", + "value": "TC Value 1" + }, + "topicClassVocab": { + "typeName": "topicClassVocab", + "multiple": false, + "typeClass": "primitive", + "value": "TC Vocabulary" + }, + "topicClassVocabURI": { + "typeName": "topicClassVocabURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://www.topicClass.com/one" + } + } + ] + }, + { + "typeName": "kindOfData", + "multiple": true, + "typeClass": "primitive", + "value": [ + "Kind of Data" + ] + }, + { + "typeName": "depositor", + "multiple": false, + "typeClass": "primitive", + "value": "Added, Depositor" + } + ] + }, + "geospatial": { + "displayName": "Geospatial", + "name":"geospatial", + "fields": [ + { + "typeName": "geographicCoverage", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "country": { + "typeName": "country", + "multiple": false, + "typeClass": "primitive", + "value": "USA" + }, + "state": { + "typeName": "state", + "multiple": false, + "typeClass": "primitive", + "value": "MA" + }, + "city": { + "typeName": "city", + "multiple": false, + "typeClass": "primitive", + "value": "Cambridge" + }, + "otherGeographicCoverage": { + "typeName": "otherGeographicCoverage", + "multiple": false, + "typeClass": "primitive", + "value": "Other Geographic Coverage" + } + } + ] + }, + { + "typeName": "geographicBoundingBox", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "westLongitude": { + "typeName": "westLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "60.3" + }, + "eastLongitude": { + "typeName": "eastLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "59.8" + }, + "southLatitude": { + "typeName": "southLatitude", + "multiple": false, + "typeClass": "primitive", + "value": "41.6" + }, + "northLatitude": { + "typeName": "northLatitude", + "multiple": false, + "typeClass": "primitive", + "value": "43.8" + } + } + ] + } + ] + } + }, + "files": [], + "citation": "Finch, Fiona, 2015, \"Darwin's Finches\", https://doi.org/10.5072/FK2/PCA2E3, Root Dataverse, V1" + } +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch-terms-of-use.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch-terms-of-use.xml new file mode 100644 index 00000000000..d813d155a90 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch-terms-of-use.xml @@ -0,0 +1,78 @@ + + + + + + Darwin's Finches + doi:10.5072/FK2/PCA2E3 + + + + 1 + + Finch, Fiona, 2015, "Darwin's Finches", https://doi.org/10.5072/FK2/PCA2E3, Root Dataverse, V1 + + + + + + Darwin's Finches + Darwin's Finches Alternative Title1 + Darwin's Finches Alternative Title2 + doi:10.5072/FK2/PCA2E3 + + + Finch, Fiona + + + Johnny Hawk + + + Odin Raven + Jimmy Finch + Added, Depositor + + + + + + Medicine, Health and Life Sciences + Keyword Value 1 + Keyword Value Two + TC Value 1 + + Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds. + + 20020816 + 20160630 + 20070831 + 20130630 + USA + Cambridge + MA + Other Geographic Coverage + + 60.3 + 59.8 + 41.6 + 43.8 + + Kind of Data + + + + + + + + + + + + + This dataset is made available without information on how it can be used. You should communicate with the Contact(s) specified before use. + + + + + diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml index 6730c44603a..010a5db4f2b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml @@ -69,6 +69,7 @@ + <a href="http://creativecommons.org/publicdomain/zero/1.0">CC0 1.0</a> diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml index 507d752192d..e865dc0ffe4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml @@ -161,6 +161,7 @@ Disclaimer Terms of Access + <a href="http://creativecommons.org/publicdomain/zero/1.0">CC0 1.0</a> RelatedMaterial1 diff --git a/src/test/java/edu/harvard/iq/dataverse/feedback/FeedbackUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/feedback/FeedbackUtilTest.java index 7c31db5bee2..072be13dcec 100644 --- a/src/test/java/edu/harvard/iq/dataverse/feedback/FeedbackUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/feedback/FeedbackUtilTest.java @@ -13,6 +13,7 @@ import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; import edu.harvard.iq.dataverse.license.LicenseServiceBean; import edu.harvard.iq.dataverse.mocks.MockDatasetFieldSvc; import edu.harvard.iq.dataverse.mocks.MocksFactory; @@ -55,6 +56,7 @@ public class FeedbackUtilTest { private static InternetAddress systemAddress; private static final SettingsServiceBean settingsService = Mockito.mock(SettingsServiceBean.class); private static final LicenseServiceBean licenseService = Mockito.mock(LicenseServiceBean.class); + private static final DatasetTypeServiceBean datasetTypeService = Mockito.mock(DatasetTypeServiceBean.class); private static final String systemEmail = "support@librascholar.edu"; private static final boolean weKnowHowToCreateMockAuthenticatedUsers = false; @@ -144,7 +146,7 @@ public static void setUpClass() throws IOException, JsonParseException, AddressE JsonReader jsonReader1 = Json.createReader(new StringReader(datasetVersionAsJson)); JsonObject json1 = jsonReader1.readObject(); - JsonParser jsonParser = new JsonParser(datasetFieldTypeSvc, null, settingsService, licenseService); + JsonParser jsonParser = new JsonParser(datasetFieldTypeSvc, null, settingsService, licenseService, datasetTypeService); dsVersion = jsonParser.parseDatasetVersion(json1.getJsonObject("datasetVersion")); File datasetVersionJson2 = new File("tests/data/datasetContacts1.json"); @@ -153,14 +155,14 @@ public static void setUpClass() throws IOException, JsonParseException, AddressE JsonReader jsonReader12 = Json.createReader(new StringReader(datasetVersionAsJson2)); JsonObject json12 = jsonReader12.readObject(); - JsonParser jsonParser2 = new JsonParser(datasetFieldTypeSvc, null, settingsService, licenseService); + JsonParser jsonParser2 = new JsonParser(datasetFieldTypeSvc, null, settingsService, licenseService, datasetTypeService); dsVersion2 = jsonParser2.parseDatasetVersion(json12.getJsonObject("datasetVersion")); File datasetVersionJsonNoContacts = new File("tests/data/datasetNoContacts.json"); String datasetVersionAsJsonNoContacts = new String(Files.readAllBytes(Paths.get(datasetVersionJsonNoContacts.getAbsolutePath()))); JsonReader jsonReaderNoContacts = Json.createReader(new StringReader(datasetVersionAsJsonNoContacts)); JsonObject jsonNoContacts = jsonReaderNoContacts.readObject(); - JsonParser jsonParserNoContacts = new JsonParser(datasetFieldTypeSvc, null, settingsService, licenseService); + JsonParser jsonParserNoContacts = new JsonParser(datasetFieldTypeSvc, null, settingsService, licenseService, datasetTypeService); dsVersionNoContacts = jsonParserNoContacts.parseDatasetVersion(jsonNoContacts.getJsonObject("datasetVersion")); FeedbackUtil justForCodeCoverage = new FeedbackUtil(); diff --git a/src/test/java/edu/harvard/iq/dataverse/ingest/IngestUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/ingest/IngestUtilTest.java index 4dfedf5aa17..955070a662a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/ingest/IngestUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/ingest/IngestUtilTest.java @@ -112,8 +112,8 @@ public void testCheckForDuplicateFileNamesNoDirectories() throws Exception { } // check filenames are unique and altered - assertEquals(true, file1NameAltered); - assertEquals(true, file2NameAltered); + assertTrue(file1NameAltered); + assertTrue(file2NameAltered); // try to add data files with "-1" duplicates and see if it gets incremented to "-2" IngestUtil.checkForDuplicateFileNamesFinal(datasetVersion, dataFileList, null); @@ -128,8 +128,8 @@ public void testCheckForDuplicateFileNamesNoDirectories() throws Exception { } // check filenames are unique and altered - assertEquals(true, file1NameAltered); - assertEquals(true, file2NameAltered); + assertTrue(file1NameAltered); + assertTrue(file2NameAltered); } @Test @@ -218,8 +218,8 @@ public void testCheckForDuplicateFileNamesWithEmptyDirectoryLabels() throws Exce } // check filenames are unique and altered - assertEquals(true, file1NameAltered); - assertEquals(true, file2NameAltered); + assertTrue(file1NameAltered); + assertTrue(file2NameAltered); // try to add data files with "-1" duplicates and see if it gets incremented to "-2" IngestUtil.checkForDuplicateFileNamesFinal(datasetVersion, dataFileList, null); @@ -234,8 +234,8 @@ public void testCheckForDuplicateFileNamesWithEmptyDirectoryLabels() throws Exce } // check filenames are unique and altered - assertEquals(true, file1NameAltered); - assertEquals(true, file2NameAltered); + assertTrue(file1NameAltered); + assertTrue(file2NameAltered); } @Test @@ -347,9 +347,9 @@ public void testCheckForDuplicateFileNamesWithDirectories() throws Exception { } // check filenames are unique - assertEquals(true, file1NameAltered); - assertEquals(true, file2NameAltered); - assertEquals(false, file3NameAltered); + assertTrue(file1NameAltered); + assertTrue(file2NameAltered); + assertFalse(file3NameAltered); // add duplicate file in root datasetVersion.getFileMetadatas().add(fmd3); @@ -371,9 +371,9 @@ public void testCheckForDuplicateFileNamesWithDirectories() throws Exception { } // check filenames are unique - assertEquals(true, file1NameAltered); - assertEquals(true, file2NameAltered); - assertEquals(true, file3NameAltered); + assertTrue(file1NameAltered); + assertTrue(file2NameAltered); + assertTrue(file3NameAltered); } @Test @@ -457,7 +457,7 @@ public void testCheckForDuplicateFileNamesTabular() throws Exception { } // check filename is altered since tabular and will change to .tab after ingest - assertEquals(true, file2NameAltered); + assertTrue(file2NameAltered); } @@ -553,8 +553,8 @@ public void testCheckForDuplicateFileNamesWhenReplacing() throws Exception { } // check filenames are unique and unaltered - assertEquals(true, file1NameAltered); - assertEquals(false, file2NameAltered); + assertTrue(file1NameAltered); + assertFalse(file2NameAltered); } @Test @@ -657,7 +657,7 @@ public void testRecalculateDatasetVersionUNF() { DataTable dataTable = new DataTable(); dataTable.setUnf("unfOnDataTable"); datafile1.setDataTable(dataTable); - assertEquals(true, datafile1.isTabularData()); + assertTrue(datafile1.isTabularData()); FileMetadata fmd1 = new FileMetadata(); fmd1.setId(1L); @@ -692,7 +692,7 @@ public void testGetUnfValuesOfFiles() { @Test public void testshouldHaveUnf() { - assertEquals(false, IngestUtil.shouldHaveUnf(null)); + assertFalse(IngestUtil.shouldHaveUnf(null)); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/metrics/MetricsUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/metrics/MetricsUtilTest.java index 484ce2ebe47..ca662409a98 100644 --- a/src/test/java/edu/harvard/iq/dataverse/metrics/MetricsUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/metrics/MetricsUtilTest.java @@ -6,38 +6,36 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; -import java.util.Arrays; -import java.util.Collection; import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.fail; -public class MetricsUtilTest { +class MetricsUtilTest { - public static class MetricsUtilNoParamTest { + @Nested + class MetricsUtilNoParamTest { private static final long COUNT = 42l; @Test - public void testCountToJson() { - // This constructor is just here for code coverage. :) - MetricsUtil metricsUtil = new MetricsUtil(); + void testCountToJson() { JsonObject jsonObject = MetricsUtil.countToJson(COUNT).build(); System.out.println(JsonUtil.prettyPrint(jsonObject)); assertEquals(COUNT, jsonObject.getJsonNumber("count").longValue()); } @Test - public void testDataversesByCategoryToJson() { + void testDataversesByCategoryToJson() { List list = new ArrayList<>(); Object[] obj00 = { "RESEARCH_PROJECTS", 791l }; Object[] obj01 = { "RESEARCHERS", 745l }; @@ -66,7 +64,7 @@ public void testDataversesByCategoryToJson() { } @Test - public void testDatasetsBySubjectToJson() { + void testDatasetsBySubjectToJson() { List list = new ArrayList<>(); Object[] obj00 = { "Social Sciences", 24955l }; Object[] obj01 = { "Medicine, Health and Life Sciences", 2262l }; @@ -105,7 +103,7 @@ public void testDatasetsBySubjectToJson() { } @Test - public void testDataversesBySubjectToJson() { + void testDataversesBySubjectToJson() { List list = new ArrayList<>(); Object[] obj00 = { "Social Sciences", 24955l }; Object[] obj01 = { "Medicine, Health and Life Sciences", 2262l }; @@ -164,7 +162,7 @@ void testSanitizeYearMonthUserInputIsAfterCurrentDate() { } @Test - public void testGetCurrentMonth() { + void testGetCurrentMonth() { String expectedMonth = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM")); String currentMonth = MetricsUtil.getCurrentMonth(); assertEquals(expectedMonth, currentMonth); @@ -173,7 +171,7 @@ public void testGetCurrentMonth() { // Create JsonArray, turn into string and back into array to confirm data // integrity @Test - public void testStringToJsonArrayBuilder() { + void testStringToJsonArrayBuilder() { System.out.println("testStringToJsonArrayBuilder"); List list = new ArrayList<>(); Object[] obj00 = { "Social Sciences", 24955l }; @@ -192,7 +190,7 @@ public void testStringToJsonArrayBuilder() { // Create JsonObject, turn into string and back into array to confirm data // integrity @Test - public void testStringToJsonObjectBuilder() { + void testStringToJsonObjectBuilder() { System.out.println("testStringToJsonObjectBuilder"); JsonObject jsonObjBefore = Json.createObjectBuilder().add("Test", "result").build(); @@ -204,6 +202,12 @@ public void testStringToJsonObjectBuilder() { assertEquals(jsonObjBefore.getString("Test"), jsonObjAfter.getString("Test")); } + @Test + void testStringToJsonWithNull() { + assertNull(MetricsUtil.stringToJsonArray(null)); + assertNull(MetricsUtil.stringToJsonObject(null)); + } + } @ParameterizedTest diff --git a/src/test/java/edu/harvard/iq/dataverse/mocks/MocksFactory.java b/src/test/java/edu/harvard/iq/dataverse/mocks/MocksFactory.java index 927d288d660..9bda917a9bc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/mocks/MocksFactory.java +++ b/src/test/java/edu/harvard/iq/dataverse/mocks/MocksFactory.java @@ -196,12 +196,17 @@ public static DatasetFieldType makeDatasetFieldType() { DatasetFieldType retVal = new DatasetFieldType("SampleType-"+id, FieldType.TEXT, false); retVal.setId(id); MetadataBlock mdb = new MetadataBlock(); + mdb.setId(new Random().nextLong()); mdb.setName("Test"); retVal.setMetadataBlock(mdb); return retVal; } - public static DataverseRole makeRole( String name ) { + public static DataverseRole makeRole( String name ) { + return makeRole(name, true); + } + + public static DataverseRole makeRole( String name, Boolean includePublishDataset ) { DataverseRole dvr = new DataverseRole(); dvr.setId( nextId() ); @@ -211,7 +216,10 @@ public static DataverseRole makeRole( String name ) { dvr.addPermission(Permission.ManageDatasetPermissions); dvr.addPermission(Permission.EditDataset); - dvr.addPermission(Permission.PublishDataset); + if (includePublishDataset){ + dvr.addPermission(Permission.PublishDataset); + } + dvr.addPermission(Permission.ViewUnpublishedDataset); return dvr; diff --git a/src/test/java/edu/harvard/iq/dataverse/mydata/PagerTest.java b/src/test/java/edu/harvard/iq/dataverse/mydata/PagerTest.java index a0ac22f99f3..a2729ce7514 100644 --- a/src/test/java/edu/harvard/iq/dataverse/mydata/PagerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/mydata/PagerTest.java @@ -2,6 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import java.util.Arrays; @@ -171,13 +173,13 @@ public void testBasics() { pager1 = new Pager(102, 10, 1); msgt("Test: 102 results, 10 per page, page 1"); - assertEquals(true, pager1.isPagerNecessary()); + assertTrue(pager1.isPagerNecessary()); assertEquals(102, pager1.getNumResults()); assertEquals(1, pager1.getPreviousPageNumber()); assertEquals(2, pager1.getNextPageNumber()); - assertEquals(false, pager1.hasPreviousPageNumber()); - assertEquals(true, pager1.hasNextPageNumber()); + assertFalse(pager1.hasPreviousPageNumber()); + assertTrue(pager1.hasNextPageNumber()); msg("page list: " + Arrays.toString(pager1.getPageNumberList())); //assertEquals(new int[]{1, 2, 3, 4, 5}, pager1.getPageNumberList()); @@ -232,13 +234,13 @@ public void testNoResults() { System.out.println("getNumResults"); Pager pager1 = new Pager(0, 10, 1); - assertEquals(false, pager1.isPagerNecessary()); + assertFalse(pager1.isPagerNecessary()); assertEquals(0, pager1.getNumResults()); assertEquals(0, pager1.getPreviousPageNumber()); assertEquals(0, pager1.getNextPageNumber()); - assertEquals(false, pager1.hasPreviousPageNumber()); - assertEquals(false, pager1.hasNextPageNumber()); + assertFalse(pager1.hasPreviousPageNumber()); + assertFalse(pager1.hasNextPageNumber()); msgt("page list: " + Arrays.toString(pager1.getPageNumberList())); //assertEquals(null, pager1.getPageNumberList()); diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java index cffac741c78..bacb231b4d5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java @@ -5,6 +5,8 @@ import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import edu.harvard.iq.dataverse.pidproviders.doi.UnmanagedDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.crossref.CrossRefDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.crossref.CrossRefDOIProviderFactory; import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteDOIProvider; import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteProviderFactory; import edu.harvard.iq.dataverse.pidproviders.doi.ezid.EZIdDOIProvider; @@ -25,7 +27,6 @@ import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -41,7 +42,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; @@ -67,7 +67,7 @@ @JvmSetting(key = JvmSettings.PID_PROVIDER_SHOULDER, value = "QE", varArgs = "perma2") @JvmSetting(key = JvmSettings.PID_PROVIDER_MANAGED_LIST, value = "perma:LINKIT/FK2ABCDEF", varArgs ="perma2") @JvmSetting(key = JvmSettings.PERMALINK_SEPARATOR, value = "/", varArgs = "perma2") -@JvmSetting(key = JvmSettings.PERMALINK_BASE_URL, value = "https://example.org/123", varArgs = "perma2") +@JvmSetting(key = JvmSettings.PERMALINK_BASE_URL, value = "https://example.org/123/citation?persistentId=perma:", varArgs = "perma2") // Datacite 1 @JvmSetting(key = JvmSettings.PID_PROVIDER_LABEL, value = "dataCite 1", varArgs = "dc1") @JvmSetting(key = JvmSettings.PID_PROVIDER_TYPE, value = DataCiteDOIProvider.TYPE, varArgs = "dc1") @@ -99,7 +99,7 @@ @JvmSetting(key = JvmSettings.PID_PROVIDER_LABEL, value = "FAKE 1", varArgs = "fake1") @JvmSetting(key = JvmSettings.PID_PROVIDER_TYPE, value = FakeDOIProvider.TYPE, varArgs = "fake1") @JvmSetting(key = JvmSettings.PID_PROVIDER_AUTHORITY, value = "10.5074", varArgs = "fake1") -@JvmSetting(key = JvmSettings.PID_PROVIDER_SHOULDER, value = "FK", varArgs = "fake1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_SHOULDER, value = "fk", varArgs = "fake1") @JvmSetting(key = JvmSettings.PID_PROVIDER_MANAGED_LIST, value = "doi:10.5073/FK3ABCDEF", varArgs ="fake1") //HANDLE 1 @@ -114,8 +114,20 @@ @JvmSetting(key = JvmSettings.HANDLENET_KEY_PASSPHRASE, value = "passphrase", varArgs ="hdl1") @JvmSetting(key = JvmSettings.HANDLENET_KEY_PATH, value = "/tmp/cred", varArgs ="hdl1") +// CrossRef 1 +@JvmSetting(key = JvmSettings.PID_PROVIDER_LABEL, value = "CrossRef 1", varArgs = "crossref1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_TYPE, value = CrossRefDOIProvider.TYPE, varArgs = "crossref1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_AUTHORITY, value = "10.11111", varArgs = "crossref1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_SHOULDER, value = "DVN/", varArgs = "crossref1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_MANAGED_LIST, value = "", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_URL, value = "https://doi.crossref.org", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_REST_API_URL, value = "https://test.crossref.org", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_USERNAME, value = "crusername", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_PASSWORD, value = "secret", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_DEPOSITOR, value = "xyz", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_DEPOSITOR_EMAIL, value = "xyz@example.com", varArgs ="crossref1") //List to instantiate -@JvmSetting(key = JvmSettings.PID_PROVIDERS, value = "perma1, perma2, dc1, dc2, ez1, fake1, hdl1") +@JvmSetting(key = JvmSettings.PID_PROVIDERS, value = "perma1, perma2, dc1, dc2, ez1, fake1, hdl1, crossref1") public class PidUtilTest { @@ -133,6 +145,7 @@ public static void setUpClass() throws Exception { pidProviderFactoryMap.put(HandlePidProvider.TYPE, new HandleProviderFactory()); pidProviderFactoryMap.put(FakeDOIProvider.TYPE, new FakeProviderFactory()); pidProviderFactoryMap.put(EZIdDOIProvider.TYPE, new EZIdProviderFactory()); + pidProviderFactoryMap.put(CrossRefDOIProvider.TYPE, new CrossRefDOIProviderFactory()); PidUtil.clearPidProviders(); @@ -191,7 +204,7 @@ public void testFactories() throws IOException { assertEquals("-", p.getSeparator()); assertTrue(p.getUrlPrefix().startsWith(SystemConfig.getDataverseSiteUrlStatic())); p = PidUtil.getPidProvider("perma2"); - assertTrue(p.getUrlPrefix().startsWith("https://example.org/123")); + assertTrue(p.getUrlPrefix().startsWith("https://example.org/123/citation?persistentId=")); p = PidUtil.getPidProvider("dc2"); assertEquals("FK3", p.getShoulder()); @@ -237,9 +250,12 @@ public void testDOIParsing() throws IOException { assertEquals(pid1String, pid3.asString()); assertEquals("dc1", pid3.getProviderId()); - String pid4String = "doi:10.5072/FK3ABCDEF"; + //Also test case insensitive + String pid4String = "doi:10.5072/fk3ABCDEF"; GlobalId pid4 = PidUtil.parseAsGlobalID(pid4String); - assertEquals(pid4String, pid4.asString()); + // Lower case is recognized by converting to upper case internally, so we need to test vs. the upper case identifier + // I.e. we are verifying that the lower case string is parsed the same as the upper case string, both give an internal upper case PID representation + assertEquals("doi:10.5072/FK3ABCDEF", pid4.asString()); assertEquals("dc2", pid4.getProviderId()); String pid5String = "doi:10.5072/FK2ABCDEF"; @@ -252,7 +268,10 @@ public void testDOIParsing() throws IOException { assertEquals(pid6String, pid6.asString()); assertEquals("fake1", pid6.getProviderId()); - + String pid7String = "doi:10.11111/DVN/ABCDEF"; + GlobalId pid7 = PidUtil.parseAsGlobalID(pid7String); + assertEquals(pid7String, pid7.asString()); + assertEquals("crossref1", pid7.getProviderId()); } @Test @@ -296,6 +315,13 @@ public void testUnmanagedParsing() throws IOException { GlobalId pid6 = PidUtil.parseAsGlobalID(pid6String); assertEquals(pid6String, pid6.asString()); assertEquals(UnmanagedPermaLinkPidProvider.ID, pid6.getProviderId()); + + //Lowercase test for unmanaged DOIs + String pid7String = "doi:10.5281/zenodo.6381129"; + GlobalId pid7 = PidUtil.parseAsGlobalID(pid7String); + assertEquals(UnmanagedDOIProvider.ID, pid5.getProviderId()); + assertEquals(pid7String.toUpperCase().replace("DOI", "doi"), pid7.asString()); + } @@ -334,15 +360,15 @@ public void testExcludedSetParsing() throws IOException { @Test public void testManagedSetParsing() throws IOException { - String pid1String = "doi:10.5073/FK3ABCDEF"; + String pid1String = "doi:10.5073/fk3ABCDEF"; GlobalId pid2 = PidUtil.parseAsGlobalID(pid1String); - assertEquals(pid1String, pid2.asString()); + assertEquals(pid1String.toUpperCase().replace("DOI", "doi"), pid2.asString()); assertEquals("fake1", pid2.getProviderId()); assertEquals("https://doi.org/" + pid2.getAuthority() + PidUtil.getPidProvider(pid2.getProviderId()).getSeparator() + pid2.getIdentifier(),pid2.asURL()); assertEquals("10.5073", pid2.getAuthority()); assertEquals(AbstractDOIProvider.DOI_PROTOCOL, pid2.getProtocol()); GlobalId pid3 = PidUtil.parseAsGlobalID(pid2.asURL()); - assertEquals(pid1String, pid3.asString()); + assertEquals(pid1String.toUpperCase().replace("DOI", "doi"), pid3.asString()); assertEquals("fake1", pid3.getProviderId()); assertFalse(PidUtil.getPidProvider(pid3.getProviderId()).canCreatePidsLike(pid3)); diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java new file mode 100644 index 00000000000..2bd6818821d --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java @@ -0,0 +1,447 @@ +package edu.harvard.iq.dataverse.pidproviders.doi.datacite; + +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.DataCitation; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetAuthor; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; +import edu.harvard.iq.dataverse.DatasetFieldType.FieldType; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.MetadataBlock; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.branding.BrandingUtil; +import edu.harvard.iq.dataverse.dataset.DatasetType; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactoryBean; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; +import edu.harvard.iq.dataverse.pidproviders.doi.DoiMetadata; +import edu.harvard.iq.dataverse.pidproviders.doi.XmlMetadataTemplate; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.json.CompoundVocabularyException; +import edu.harvard.iq.dataverse.util.json.ControlledVocabularyException; +import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonParser; +import edu.harvard.iq.dataverse.util.json.JsonUtil; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; +import edu.harvard.iq.dataverse.util.xml.XmlValidator; +import jakarta.json.JsonArray; +import jakarta.json.JsonObject; +import jakarta.json.JsonString; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.xml.transform.stream.StreamSource; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; + +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.xml.sax.SAXException; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; + +@ExtendWith(MockitoExtension.class) +@LocalJvmSettings +@JvmSetting(key = JvmSettings.SITE_URL, value = "https://example.com") + +public class XmlMetadataTemplateTest { + + static DataverseServiceBean dataverseSvc; + static SettingsServiceBean settingsSvc; + static PidProviderFactoryBean pidService; + static final String DEFAULT_NAME = "LibraScholar"; + + @BeforeAll + public static void setupMocks() { + dataverseSvc = Mockito.mock(DataverseServiceBean.class); + settingsSvc = Mockito.mock(SettingsServiceBean.class); + BrandingUtil.injectServices(dataverseSvc, settingsSvc); + + // initial values (needed here for other tests where this method is reused!) + Mockito.when(settingsSvc.getValueForKey(SettingsServiceBean.Key.InstallationName)).thenReturn(DEFAULT_NAME); + Mockito.when(dataverseSvc.getRootDataverseName()).thenReturn(DEFAULT_NAME); + + pidService = Mockito.mock(PidProviderFactoryBean.class); + Mockito.when(pidService.isGlobalIdLocallyUnique(any(GlobalId.class))).thenReturn(true); + Mockito.when(pidService.getProducer()).thenReturn("RootDataverse"); + + } + + /** + * A minimal example to assure that the XMLMetadataTemplate generates output + * consistent with the DataCite XML v4.5 schema. + */ + @Test + public void testDataCiteXMLCreation() throws IOException { + DoiMetadata doiMetadata = new DoiMetadata(); + doiMetadata.setTitle("A Title"); + DatasetFieldType dft = new DatasetFieldType(DatasetFieldConstant.authorName, FieldType.TEXT, false); + dft.setDisplayFormat("#VALUE"); + DatasetFieldType dft2 = new DatasetFieldType(DatasetFieldConstant.authorAffiliation, FieldType.TEXT, false); + dft2.setDisplayFormat("#VALUE"); + DatasetAuthor alice = new DatasetAuthor(); + DatasetField df1 = new DatasetField(); + df1.setDatasetFieldType(dft); + df1.setSingleValue("Alice"); + alice.setName(df1); + DatasetField df2 = new DatasetField(); + df2.setDatasetFieldType(dft2); + df2.setSingleValue("Harvard University"); + alice.setAffiliation(df2); + DatasetAuthor bob = new DatasetAuthor(); + DatasetField df3 = new DatasetField(); + df3.setDatasetFieldType(dft); + df3.setSingleValue("Bob"); + bob.setName(df3); + DatasetField df4 = new DatasetField(); + df4.setDatasetFieldType(dft2); + df4.setSingleValue("QDR"); + bob.setAffiliation(df4); + List authors = new ArrayList<>(); + authors.add(alice); + authors.add(bob); + doiMetadata.setAuthors(authors); + doiMetadata.setPublisher("Dataverse"); + XmlMetadataTemplate template = new XmlMetadataTemplate(doiMetadata); + + Dataset d = new Dataset(); + GlobalId doi = new GlobalId("doi", "10.5072", "FK2/ABCDEF", null, null, null); + d.setGlobalId(doi); + DatasetVersion dv = new DatasetVersion(); + TermsOfUseAndAccess toa = new TermsOfUseAndAccess(); + toa.setTermsOfUse("Some terms"); + dv.setTermsOfUseAndAccess(toa); + dv.setDataset(d); + DatasetFieldType primitiveDSFType = new DatasetFieldType(DatasetFieldConstant.title, + DatasetFieldType.FieldType.TEXT, false); + DatasetField testDatasetField = new DatasetField(); + + dv.setVersionState(VersionState.DRAFT); + + testDatasetField.setDatasetVersion(dv); + testDatasetField.setDatasetFieldType(primitiveDSFType); + testDatasetField.setSingleValue("First Title"); + List fields = new ArrayList<>(); + fields.add(testDatasetField); + dv.setDatasetFields(fields); + ArrayList dsvs = new ArrayList<>(); + dsvs.add(0, dv); + d.setVersions(dsvs); + DatasetType dType = new DatasetType(); + dType.setName(DatasetType.DATASET_TYPE_DATASET); + d.setDatasetType(dType); + + String xml = template.generateXML(d); + System.out.println("Output from minimal example is " + xml); + try { + StreamSource source = new StreamSource(new StringReader(xml)); + source.setSystemId("DataCite XML for test dataset"); + assertTrue(XmlValidator.validateXmlSchema(source, + new URL("https://schema.datacite.org/meta/kernel-4/metadata.xsd"))); + } catch (SAXException e) { + System.out.println("Invalid schema: " + e.getMessage()); + } + + } + + /** + * This tests a more complete example based off of the dataset-all-defaults + * file, again checking for conformance of the result with the DataCite XML v4.5 + * schema. + */ + @Test + public void testDataCiteXMLCreationAllFields() throws IOException { + Dataverse collection = new Dataverse(); + collection.setCitationDatasetFieldTypes(new ArrayList<>()); + Dataset d = new Dataset(); + d.setOwner(collection); + DatasetVersion dv = new DatasetVersion(); + TermsOfUseAndAccess toa = new TermsOfUseAndAccess(); + toa.setTermsOfUse("Some terms"); + dv.setTermsOfUseAndAccess(toa); + dv.setDataset(d); + DatasetFieldType primitiveDSFType = new DatasetFieldType(DatasetFieldConstant.title, + DatasetFieldType.FieldType.TEXT, false); + DatasetField testDatasetField = new DatasetField(); + + dv.setVersionState(VersionState.DRAFT); + + testDatasetField.setDatasetVersion(dv); + + File datasetVersionJson = new File("src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt"); + String datasetVersionAsJson = new String(Files.readAllBytes(Paths.get(datasetVersionJson.getAbsolutePath()))); + JsonObject datasetJson = JsonUtil.getJsonObject(datasetVersionAsJson); + + GlobalId doi = new GlobalId("doi", datasetJson.getString("authority"), datasetJson.getString("identifier"), + null, null, null); + d.setGlobalId(doi); + + List fields = assertDoesNotThrow(() -> XmlMetadataTemplateTest + .parseMetadataBlocks(datasetJson.getJsonObject("datasetVersion").getJsonObject("metadataBlocks"))); + dv.setDatasetFields(fields); + + ArrayList dsvs = new ArrayList<>(); + dsvs.add(0, dv); + d.setVersions(dsvs); + DatasetType dType = new DatasetType(); + dType.setName(DatasetType.DATASET_TYPE_DATASET); + d.setDatasetType(dType); + String xml = DOIDataCiteRegisterService.getMetadataFromDvObject(dv.getDataset().getGlobalId().asString(), + new DataCitation(dv).getDataCiteMetadata(), dv.getDataset()); + System.out.println("Output from dataset-all-defaults is " + xml); + try { + StreamSource source = new StreamSource(new StringReader(xml)); + source.setSystemId("DataCite XML for test dataset"); + assertTrue(XmlValidator.validateXmlSchema(source, + new URL("https://schema.datacite.org/meta/kernel-4/metadata.xsd"))); + } catch (SAXException e) { + System.out.println("Invalid schema: " + e.getMessage()); + } + + } + + /** + * Mock Utility Methods - These methods support importing DatasetFields from the + * Dataverse JSON export format. They assume that any DatasetFieldType + * referenced exists, that any Controlled Vocabulary value exists, etc. which + * avoids having to do database lookups or read metadatablock tsv files. They + * are derived from the JsonParser methods of the same names with any db + * references and DatasetFieldType-related error checking removed. + */ + public static List parseMetadataBlocks(JsonObject json) throws JsonParseException { + + Map existingTypes = new HashMap<>(); + + Set keys = json.keySet(); + List fields = new LinkedList<>(); + + for (String blockName : keys) { + MetadataBlock block = new MetadataBlock(); + block.setName(blockName); + JsonObject blockJson = json.getJsonObject(blockName); + JsonArray fieldsJson = blockJson.getJsonArray("fields"); + fields.addAll(parseFieldsFromArray(fieldsJson, true, block, existingTypes)); + } + return fields; + } + + private static List parseFieldsFromArray(JsonArray fieldsArray, Boolean testType, MetadataBlock block, + Map existingTypes) throws JsonParseException { + List fields = new LinkedList<>(); + for (JsonObject fieldJson : fieldsArray.getValuesAs(JsonObject.class)) { + + DatasetField field = parseField(fieldJson, testType, block, existingTypes); + if (field != null) { + fields.add(field); + } + + } + return fields; + + } + + public static DatasetField parseField(JsonObject json, Boolean testType, MetadataBlock block, + Map existingTypes) throws JsonParseException { + if (json == null) { + return null; + } + + DatasetField ret = new DatasetField(); + String fieldName = json.getString("typeName", ""); + String typeClass = json.getString("typeClass", ""); + if (!existingTypes.containsKey(fieldName)) { + boolean multiple = json.getBoolean("multiple"); + DatasetFieldType fieldType = new DatasetFieldType(); + fieldType.setName(fieldName); + fieldType.setAllowMultiples(multiple); + fieldType.setAllowControlledVocabulary(typeClass.equals("controlledVocabulary")); + fieldType.setFieldType(FieldType.TEXT); + fieldType.setMetadataBlock(block); + fieldType.setChildDatasetFieldTypes(new ArrayList<>()); + existingTypes.put(fieldName, fieldType); + } + DatasetFieldType type = existingTypes.get(fieldName); + ret.setDatasetFieldType(type); + + if (typeClass.equals("compound")) { + parseCompoundValue(ret, type, json, testType, block, existingTypes); + } else if (type.isControlledVocabulary()) { + parseControlledVocabularyValue(ret, type, json); + } else { + parsePrimitiveValue(ret, type, json); + } + + return ret; + } + + public static void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, JsonObject json, + Boolean testType, MetadataBlock block, Map existingTypes) + throws JsonParseException { + List vocabExceptions = new ArrayList<>(); + List vals = new LinkedList<>(); + if (compoundType.isAllowMultiples()) { + int order = 0; + try { + json.getJsonArray("value").getValuesAs(JsonObject.class); + } catch (ClassCastException cce) { + throw new JsonParseException("Invalid values submitted for " + compoundType.getName() + + ". It should be an array of values."); + } + for (JsonObject obj : json.getJsonArray("value").getValuesAs(JsonObject.class)) { + DatasetFieldCompoundValue cv = new DatasetFieldCompoundValue(); + List fields = new LinkedList<>(); + for (String fieldName : obj.keySet()) { + JsonObject childFieldJson = obj.getJsonObject(fieldName); + DatasetField f = null; + try { + f = parseField(childFieldJson, testType, block, existingTypes); + } catch (ControlledVocabularyException ex) { + vocabExceptions.add(ex); + } + + if (f != null) { + f.setParentDatasetFieldCompoundValue(cv); + fields.add(f); + } + } + if (!fields.isEmpty()) { + cv.setChildDatasetFields(fields); + cv.setDisplayOrder(order); + vals.add(cv); + } + order++; + } + + } else { + + DatasetFieldCompoundValue cv = new DatasetFieldCompoundValue(); + List fields = new LinkedList<>(); + JsonObject value = json.getJsonObject("value"); + for (String key : value.keySet()) { + JsonObject childFieldJson = value.getJsonObject(key); + DatasetField f = null; + try { + f = parseField(childFieldJson, testType, block, existingTypes); + } catch (ControlledVocabularyException ex) { + vocabExceptions.add(ex); + } + if (f != null) { + f.setParentDatasetFieldCompoundValue(cv); + fields.add(f); + } + } + if (!fields.isEmpty()) { + cv.setChildDatasetFields(fields); + vals.add(cv); + } + + } + if (!vocabExceptions.isEmpty()) { + throw new CompoundVocabularyException("Invalid controlled vocabulary in compound field ", vocabExceptions, + vals); + } + + for (DatasetFieldCompoundValue dsfcv : vals) { + dsfcv.setParentDatasetField(dsf); + } + dsf.setDatasetFieldCompoundValues(vals); + } + + public static void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft, JsonObject json) + throws JsonParseException { + List vals = new LinkedList<>(); + if (dft.isAllowMultiples()) { + try { + json.getJsonArray("value").getValuesAs(JsonObject.class); + } catch (ClassCastException cce) { + throw new JsonParseException( + "Invalid values submitted for " + dft.getName() + ". It should be an array of values."); + } + for (JsonString val : json.getJsonArray("value").getValuesAs(JsonString.class)) { + DatasetFieldValue datasetFieldValue = new DatasetFieldValue(dsf); + datasetFieldValue.setDisplayOrder(vals.size() - 1); + datasetFieldValue.setValue(val.getString().trim()); + vals.add(datasetFieldValue); + } + + } else { + try { + json.getString("value"); + } catch (ClassCastException cce) { + throw new JsonParseException( + "Invalid value submitted for " + dft.getName() + ". It should be a single value."); + } + DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); + datasetFieldValue.setValue(json.getString("value", "").trim()); + datasetFieldValue.setDatasetField(dsf); + vals.add(datasetFieldValue); + } + + dsf.setDatasetFieldValues(vals); + } + + public static void parseControlledVocabularyValue(DatasetField dsf, DatasetFieldType cvvType, JsonObject json) + throws JsonParseException { + List vals = new LinkedList<>(); + try { + if (cvvType.isAllowMultiples()) { + try { + json.getJsonArray("value").getValuesAs(JsonObject.class); + } catch (ClassCastException cce) { + throw new JsonParseException( + "Invalid values submitted for " + cvvType.getName() + ". It should be an array of values."); + } + for (JsonString strVal : json.getJsonArray("value").getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + ControlledVocabularyValue cvv = new ControlledVocabularyValue(); + cvv.setDatasetFieldType(cvvType); + cvv.setStrValue(strVal.getString()); + vals.add(cvv); + } + + } else { + try { + json.getString("value"); + } catch (ClassCastException cce) { + throw new JsonParseException( + "Invalid value submitted for " + cvvType.getName() + ". It should be a single value."); + } + String strValue = json.getString("value", ""); + ControlledVocabularyValue cvv = new ControlledVocabularyValue(); + cvv.setDatasetFieldType(cvvType); + cvv.setStrValue(strValue); + vals.add(cvv); + } + } catch (ClassCastException cce) { + throw new JsonParseException("Invalid values submitted for " + cvvType.getName()); + } + + dsf.setControlledVocabularyValues(vals); + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java index da94b288bee..f06be37578d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java @@ -277,7 +277,7 @@ public void testGetPrivateUrlFromRoleAssignmentSuccess() { PrivateUrl privateUrl = PrivateUrlUtil.getPrivateUrlFromRoleAssignment(ra, dataverseSiteUrl); assertNotNull(privateUrl); assertEquals(new Long(42), privateUrl.getDataset().getId()); - assertEquals("https://dataverse.example.edu/privateurl.xhtml?token=cd71e9d7-73a7-4ec8-b890-3d00499e8693", privateUrl.getLink()); + assertEquals("https://dataverse.example.edu/previewurl.xhtml?token=cd71e9d7-73a7-4ec8-b890-3d00499e8693", privateUrl.getLink()); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java index 92b06e5936f..8e24c546556 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.Dataverse.DataverseType; import edu.harvard.iq.dataverse.branding.BrandingUtil; +import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.mocks.MocksFactory; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import edu.harvard.iq.dataverse.settings.JvmSettings; @@ -51,6 +52,8 @@ public void setUp() { indexService.settingsService = Mockito.mock(SettingsServiceBean.class); indexService.dataverseService = Mockito.mock(DataverseServiceBean.class); indexService.datasetFieldService = Mockito.mock(DatasetFieldServiceBean.class); + indexService.datasetVersionService = Mockito.mock(DatasetVersionServiceBean.class); + indexService.datasetVersionFilesServiceBean = Mockito.mock(DatasetVersionFilesServiceBean.class); BrandingUtil.injectServices(indexService.dataverseService, indexService.settingsService); Mockito.when(indexService.dataverseService.findRootDataverse()).thenReturn(dataverse); @@ -141,6 +144,9 @@ private IndexableDataset createIndexableDataset() { datasetVersion.getDatasetFields().add(field); final IndexableDataset indexableDataset = new IndexableDataset(datasetVersion); indexableDataset.getDatasetVersion().getDataset().setOwner(dataverse); + DatasetType datasetType = new DatasetType(); + datasetType.setName(DatasetType.DEFAULT_DATASET_TYPE); + indexableDataset.getDatasetVersion().getDataset().setDatasetType(datasetType); return indexableDataset; } diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrSearchResultTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrSearchResultTest.java index 4fb29869db7..d7deaa2dbc1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SolrSearchResultTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrSearchResultTest.java @@ -225,6 +225,29 @@ public void testSetPublicationStatuses14() { assertTrue(this.solrSearchResult.isDeaccessionedState()); } + @Test + public void testSetPublicationStatusesJson() { + + boolean showRelevance = false; + boolean showEntityIds = false; + boolean showApiUrls = false; + + SolrSearchResult result01 = new SolrSearchResult("myQuery", "myName"); + result01.setType(SearchConstants.DATAVERSES); + result01.setPublicationStatuses(List.of("Unpublished", "Draft")); + JsonObjectBuilder actual01 = result01.json(showRelevance, showEntityIds, showApiUrls); + JsonObject actual = actual01.build(); + System.out.println("actual: " + actual); + + JsonObjectBuilder expResult = Json.createObjectBuilder(); + expResult.add("type", SearchConstants.DATAVERSE); + expResult.add("publicationStatuses", Json.createArrayBuilder().add("Unpublished").add("Draft").build()); + JsonObject expected = expResult.build(); + System.out.println("expect: " + expected); + + assertEquals(expected, actual); + } + @Test public void testJson() { diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java index ce8698c95eb..46359d7b02c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java @@ -434,4 +434,11 @@ public void testDetermineFileTypeROCrate() { assertEquals("Code", FileUtil.getIndexableFacetFileType(dockerDataFile)); } + @Test + public void testSanitizeFileName() { + assertEquals(null, FileUtil.sanitizeFileName(null)); + assertEquals("with_space", FileUtil.sanitizeFileName("with space")); + assertEquals("withcomma", FileUtil.sanitizeFileName("with,comma")); + assertEquals("with.txt", FileUtil.sanitizeFileName("with,\\?:;,.txt")); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java index c3d9fd8fcd3..d772ba2b9da 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java @@ -89,6 +89,8 @@ public void testName() { verifyIsPerson("kcjim11, kcjim11", "kcjim11", "kcjim11"); verifyIsPerson("Bartholomew 3, James", "James", "Bartholomew 3"); + verifyIsPerson("Smith, ", null, "Smith"); + verifyIsPerson("Smith,", null, "Smith"); } private void verifyIsOrganization(String fullName) { @@ -106,7 +108,7 @@ private void verifyIsPerson(String fullName, String givenName, String familyName private void verifyIsPerson(String fullName, String givenName, String familyName, boolean isPerson) { JsonObject obj = PersonOrOrgUtil.getPersonOrOrganization(fullName, false, isPerson); System.out.println(JsonUtil.prettyPrint(obj)); - assertEquals(obj.getString("fullName"),fullName); + assertEquals(obj.getString("fullName"), StringUtil.normalize(fullName)); assertTrue(obj.getBoolean("isPerson")); assertEquals(obj.containsKey("givenName"), givenName != null); if(obj.containsKey("givenName") && givenName != null) { diff --git a/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java b/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java index f43a0c78284..9fac4d42bcd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java @@ -23,21 +23,21 @@ public class FileDataProviderFactoryTest { public void should_return_FolderDataProvider_when_parameter_is_path() { FileDataProvider result = target.getFileDataProvider(Path.of(UUID.randomUUID().toString())); - MatcherAssert.assertThat(result.getClass().getName(), Matchers.is(FolderDataProvider.class.getName())); + MatcherAssert.assertThat("should return FolderDataProvider when parameter is path", result instanceof FolderDataProvider); } @Test public void should_return_ZipFileDataProvider_when_parameter_is_file() throws IOException { FileDataProvider result = target.getFileDataProvider(Path.of(FIXTURE_DIRECTORY, "FileDataProviderFactoryTest.zip").toFile()); - MatcherAssert.assertThat(result.getClass().getName(), Matchers.is(ZipFileDataProvider.class.getName())); + MatcherAssert.assertThat("should return ZipFileDataProvider when parameter is file", result instanceof ZipFileDataProvider); } @Test public void should_return_DataFileDataProvider_when_parameter_is_datafiles() { FileDataProvider result = target.getFileDataProvider("test-name", Collections.emptyList()); - MatcherAssert.assertThat(result.getClass().getName(), Matchers.is(DataFileDataProvider.class.getName())); + MatcherAssert.assertThat("should return DataFileDataProvider when parameter is datafiles", result instanceof DataFileDataProvider); } } \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java index 972fc9c41cd..d1cb30e2bc3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java @@ -4,18 +4,11 @@ package edu.harvard.iq.dataverse.util.json; -import edu.harvard.iq.dataverse.ControlledVocabularyValue; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; -import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetFieldType.FieldType; -import edu.harvard.iq.dataverse.DatasetFieldValue; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseTheme.Alignment; -import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.UserNotification.Type; +import edu.harvard.iq.dataverse.api.dto.DataverseDTO; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroupProvider; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; @@ -23,6 +16,8 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroupTest; import edu.harvard.iq.dataverse.authorization.users.GuestUser; +import edu.harvard.iq.dataverse.dataset.DatasetType; +import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.license.LicenseServiceBean; import edu.harvard.iq.dataverse.mocks.MockDatasetFieldSvc; @@ -48,16 +43,7 @@ import java.io.StringReader; import java.math.BigDecimal; import java.text.ParseException; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.TimeZone; +import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.*; @@ -72,6 +58,7 @@ public class JsonParserTest { MockDatasetFieldSvc datasetFieldTypeSvc = null; MockSettingsSvc settingsSvc = null; LicenseServiceBean licenseService = Mockito.mock(LicenseServiceBean.class); + DatasetTypeServiceBean datasetTypeService = Mockito.mock(DatasetTypeServiceBean.class); DatasetFieldType keywordType; DatasetFieldType descriptionType; DatasetFieldType subjectType; @@ -124,7 +111,11 @@ public void setUp() { } compoundSingleType.setChildDatasetFieldTypes(childTypes); settingsSvc = new MockSettingsSvc(); - sut = new JsonParser(datasetFieldTypeSvc, null, settingsSvc, licenseService); + DatasetType datasetType = new DatasetType(); + datasetType.setName(DatasetType.DEFAULT_DATASET_TYPE); + datasetType.setId(1l); + Mockito.when(datasetTypeService.getByName(DatasetType.DEFAULT_DATASET_TYPE)).thenReturn(datasetType); + sut = new JsonParser(datasetFieldTypeSvc, null, settingsSvc, licenseService, datasetTypeService); } @Test @@ -274,6 +265,33 @@ public void testParseCompleteDataverse() throws JsonParseException { throw new JsonParseException("Couldn't read test file", ioe); } } + + /** + * Test that a JSON object passed for a DataverseDTO is correctly parsed. + * This checks that all properties are parsed into the correct DataverseDTO properties. + * @throws JsonParseException when this test is broken. + */ + @Test + public void parseDataverseDTO() throws JsonParseException { + JsonObject dvJson; + try (FileReader reader = new FileReader("doc/sphinx-guides/source/_static/api/dataverse-complete.json")) { + dvJson = Json.createReader(reader).readObject(); + DataverseDTO actual = sut.parseDataverseDTO(dvJson); + List actualDataverseContacts = actual.getDataverseContacts(); + assertEquals("Scientific Research", actual.getName()); + assertEquals("science", actual.getAlias()); + assertEquals("Scientific Research University", actual.getAffiliation()); + assertEquals("We do all the science.", actual.getDescription()); + assertEquals("LABORATORY", actual.getDataverseType().toString()); + assertEquals(2, actualDataverseContacts.size()); + assertEquals("pi@example.edu", actualDataverseContacts.get(0).getContactEmail()); + assertEquals("student@example.edu", actualDataverseContacts.get(1).getContactEmail()); + assertEquals(0, actualDataverseContacts.get(0).getDisplayOrder()); + assertEquals(1, actualDataverseContacts.get(1).getDisplayOrder()); + } catch (IOException ioe) { + throw new JsonParseException("Couldn't read test file", ioe); + } + } @Test public void testParseThemeDataverse() throws JsonParseException { diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java index 11da71e1980..7ec8e0b25f3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java @@ -133,7 +133,7 @@ public void testJson_PrivateUrl() { assertNotNull(job); JsonObject jsonObject = job.build(); assertEquals("e1d53cf6-794a-457a-9709-7c07629a8267", jsonObject.getString("token")); - assertEquals("https://dataverse.example.edu/privateurl.xhtml?token=e1d53cf6-794a-457a-9709-7c07629a8267", jsonObject.getString("link")); + assertEquals("https://dataverse.example.edu/previewurl.xhtml?token=e1d53cf6-794a-457a-9709-7c07629a8267", jsonObject.getString("link")); assertEquals("e1d53cf6-794a-457a-9709-7c07629a8267", jsonObject.getJsonObject("roleAssignment").getString("privateUrlToken")); assertEquals(PrivateUrlUser.PREFIX + "42", jsonObject.getJsonObject("roleAssignment").getString("assignee")); } @@ -290,7 +290,7 @@ public void testDataversePrinter() { assertEquals("42 Inc.", jsonObject.getString("affiliation")); assertEquals(0, jsonObject.getJsonArray("dataverseContacts").getJsonObject(0).getInt("displayOrder")); assertEquals("dv42@mailinator.com", jsonObject.getJsonArray("dataverseContacts").getJsonObject(0).getString("contactEmail")); - assertEquals(false, jsonObject.getBoolean("permissionRoot")); + assertFalse(jsonObject.getBoolean("permissionRoot")); assertEquals("Description for Dataverse 42.", jsonObject.getString("description")); assertEquals("UNCATEGORIZED", jsonObject.getString("dataverseType")); } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java index f0e538616b2..c4ee4547ed7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java @@ -63,22 +63,22 @@ private File createBlankFile(String filename) throws IOException { } return Files.createFile(tempFolder.resolve(filename)).toFile(); } - + private FileInputStream createZipReturnFilestream(List file_names, String zipfile_name) throws IOException{ - + File zip_file_obj = this.createAndZipFiles(file_names, zipfile_name); if (zip_file_obj == null){ return null; } - + FileInputStream file_input_stream = new FileInputStream(zip_file_obj); return file_input_stream; - + } - + /* - Convenience class to create .zip file and return a FileInputStream + Convenience method to create .zip file and return a File @param List file_names - List of filenames to add to .zip. These names will be used to create 0 length files @param String zipfile_name - Name of .zip file to create @@ -98,13 +98,13 @@ private File createAndZipFiles(List file_names, String zipfile_name) thr } Path zip_file_obj = this.tempFolder.resolve(zipfile_name); - ZipOutputStream zip_stream = new ZipOutputStream(new FileOutputStream(zip_file_obj.toFile())); + try (ZipOutputStream zip_stream = new ZipOutputStream(new FileOutputStream(zip_file_obj.toFile()))) { - // Iterate through File objects and add them to the ZipOutputStream - for (File file_obj : fileCollection) { - this.addToZipFile(file_obj.getName(), file_obj, zip_stream); + // Iterate through File objects and add them to the ZipOutputStream + for (File file_obj : fileCollection) { + this.addToZipFile(file_obj.getName(), file_obj, zip_stream); + } } - /* ----------------------------------- Cleanup: Delete single files that were added to .zip ----------------------------------- */ @@ -126,7 +126,7 @@ public void testCreateZippedNonShapefile() throws IOException{ File zipfile_obj = createAndZipFiles(file_names, "not-quite-a-shape.zip"); // Pass the .zip to the ShapefileHandler - ShapefileHandler shp_handler = new ShapefileHandler(new FileInputStream(zipfile_obj)); + ShapefileHandler shp_handler = new ShapefileHandler(zipfile_obj); shp_handler.DEBUG= true; // Contains shapefile? @@ -157,7 +157,7 @@ public void testShapefileWithQpjAndQmd() throws IOException { File zipFile = createAndZipFiles(fileNames, "testShapeWithNewExtensions.zip"); // Pass the zip to the ShapefileHandler - ShapefileHandler shpHandler = new ShapefileHandler(new FileInputStream(zipFile)); + ShapefileHandler shpHandler = new ShapefileHandler(zipFile); shpHandler.DEBUG = true; // Check if it is recognized as a shapefile @@ -191,7 +191,7 @@ public void testZippedTwoShapefiles() throws IOException{ File zipfile_obj = createAndZipFiles(file_names, "two-shapes.zip"); // Pass the .zip to the ShapefileHandler - ShapefileHandler shp_handler = new ShapefileHandler(new FileInputStream(zipfile_obj)); + ShapefileHandler shp_handler = new ShapefileHandler(zipfile_obj); shp_handler.DEBUG= true; assertTrue(shp_handler.containsShapefile(), "verify shapefile existance"); @@ -217,7 +217,7 @@ public void testZippedTwoShapefiles() throws IOException{ // Rezip/Reorder the files File test_unzip_folder = Files.createDirectory(this.tempFolder.resolve("test_unzip")).toFile(); //File test_unzip_folder = new File("/Users/rmp553/Desktop/blah"); - shp_handler.rezipShapefileSets(new FileInputStream(zipfile_obj), test_unzip_folder ); + shp_handler.rezipShapefileSets(test_unzip_folder ); // Does the re-ordering do what we wanted? @@ -244,7 +244,7 @@ public void testZippedShapefileWithExtraFiles() throws IOException{ File zipfile_obj = createAndZipFiles(file_names, "shape-plus.zip"); // Pass the .zip to the ShapefileHandler - ShapefileHandler shp_handler = new ShapefileHandler(new FileInputStream(zipfile_obj)); + ShapefileHandler shp_handler = new ShapefileHandler(zipfile_obj); shp_handler.DEBUG= true; assertTrue(shp_handler.containsShapefile(), "verify shapefile existance"); @@ -264,7 +264,7 @@ public void testZippedShapefileWithExtraFiles() throws IOException{ File unzip2Folder = Files.createDirectory(this.tempFolder.resolve("test_unzip2")).toFile(); // Rezip/Reorder the files - shp_handler.rezipShapefileSets(new FileInputStream(zipfile_obj), unzip2Folder); + shp_handler.rezipShapefileSets(unzip2Folder); //shp_handler.rezipShapefileSets(new FileInputStream(zipfile_obj), new File("/Users/rmp553/Desktop/blah")); @@ -282,8 +282,14 @@ public void testZippedShapefileWithExtraFiles() throws IOException{ msg("Passed!"); } - + @Test + public void testHiddenFiles() throws IOException { + // test with shapefiles in hidden directory + ShapefileHandler shp_handler = new ShapefileHandler(new File("src/test/resources/hiddenShapefiles.zip")); + shp_handler.DEBUG= true; + assertFalse(shp_handler.containsShapefile()); + } diff --git a/src/test/java/edu/harvard/iq/dataverse/validation/JSONDataValidationTest.java b/src/test/java/edu/harvard/iq/dataverse/validation/JSONDataValidationTest.java new file mode 100644 index 00000000000..107dcecba35 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/validation/JSONDataValidationTest.java @@ -0,0 +1,237 @@ +package edu.harvard.iq.dataverse.validation; + +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DatasetFieldType; +import org.everit.json.schema.Schema; +import org.everit.json.schema.ValidationException; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; + +public class JSONDataValidationTest { + + @Mock + static DatasetFieldServiceBean datasetFieldServiceMock; + @Mock + static DatasetFieldType datasetFieldTypeMock; + static ControlledVocabularyValue cvv = new ControlledVocabularyValue(); + static Map>> schemaChildMap = new HashMap<>(); + + static JSONObject rawSchema = null; + static JSONObject rawSchema() throws IOException { + if (rawSchema == null) { + Path file = Path.of("doc/sphinx-guides/source/_static/api/dataset-schema.json"); + String schema = Files.readString(file, StandardCharsets.UTF_8); + rawSchema = new JSONObject(schema); + } + return rawSchema; + } + + static String jsonInput() { + return """ + { + "datasetVersion": { + "license": { + "name": "CC0 1.0", + "uri": "http://creativecommons.org/publicdomain/zero/1.0" + }, + "metadataBlocks": { + "citation": { + "fields": [ + { + "value": "Darwin's Finches", + "typeClass": "primitive", + "multiple": false, + "typeName": "title" + }, + { + "value": [ + { + "authorName": { + "value": "Finch, Fiona", + "typeClass": "primitive", + "multiple": false, + "typeName": "authorName" + }, + "authorAffiliation": { + "value": "Birds Inc.", + "typeClass": "primitive", + "multiple": false, + "typeName": "authorAffiliation" + } + } + ], + "typeClass": "compound", + "multiple": true, + "typeName": "author" + }, + { + "value": [ + { "datasetContactEmail" : { + "typeClass": "primitive", + "multiple": false, + "typeName": "datasetContactEmail", + "value" : "finch@mailinator.com" + }, + "datasetContactName" : { + "typeClass": "primitive", + "multiple": false, + "typeName": "datasetContactName", + "value": "Finch, Fiona" + } + }], + "typeClass": "compound", + "multiple": true, + "typeName": "datasetContact" + }, + { + "value": [{ + "dsDescriptionValue":{ + "value": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds.", + "multiple": false, + "typeClass": "primitive", + "typeName": "dsDescriptionValue" + }, + "dsDescriptionDate": { + "typeName": "dsDescriptionDate", + "multiple": false, + "typeClass": "primitive", + "value": "2021-07-13" + } + }], + "typeClass": "compound", + "multiple": true, + "typeName": "dsDescription" + }, + { + "value": [ + "Medicine, Health and Life Sciences", + "Social Sciences" + ], + "typeClass": "controlledVocabulary", + "multiple": true, + "typeName": "subject" + } + ], + "displayName": "Citation Metadata" + } + } + } + } + """; + } + + @BeforeAll + static void setup() throws NoSuchFieldException, IllegalAccessException { + datasetFieldServiceMock = Mockito.mock(DatasetFieldServiceBean.class); + datasetFieldTypeMock = Mockito.mock(DatasetFieldType.class); + Field datasetFieldServiceField = JSONDataValidation.class.getDeclaredField("datasetFieldService"); + datasetFieldServiceField.setAccessible(true); + datasetFieldServiceField.set(JSONDataValidation.class, datasetFieldServiceMock); + + Mockito.when(datasetFieldServiceMock.findByName(any(String.class))).thenReturn(datasetFieldTypeMock); + List cvvList = List.of("Medicine, Health and Life Sciences", "Social Sciences"); + cvvList.forEach(i -> { + Mockito.when(datasetFieldServiceMock.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(datasetFieldTypeMock, i,true)).thenReturn(cvv); + }); + Mockito.when(datasetFieldServiceMock.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(datasetFieldTypeMock, "Bad",true)).thenReturn(null); + + Map> datasetContact = new HashMap<>(); + datasetContact.put("required", List.of("datasetContactName")); + datasetContact.put("allowed", List.of("datasetContactName", "datasetContactEmail","datasetContactAffiliation")); + schemaChildMap.put("datasetContact",datasetContact); + Map> dsDescription = new HashMap<>(); + dsDescription.put("required", List.of("dsDescriptionValue")); + dsDescription.put("allowed", List.of("dsDescriptionValue", "dsDescriptionDate")); + schemaChildMap.put("dsDescription",dsDescription); + + } + @Test + public void testSchema() throws IOException { + Schema schema = SchemaLoader.load(rawSchema()); + schema.validate(new JSONObject(jsonInput())); + } + @Test + public void testValid() throws IOException { + Schema schema = SchemaLoader.load(rawSchema()); + JSONDataValidation.validate(schema, schemaChildMap, jsonInput()); + } + @Test + public void testInvalid() throws IOException { + Schema schema = SchemaLoader.load(rawSchema()); + try { + JSONDataValidation.validate(schema, schemaChildMap, jsonInput().replace("\"Social Sciences\"", "\"Social Sciences\",\"Bad\"")); + fail(); + } catch (ValidationException e) { + System.out.println(e.getMessage()); + System.out.println(e.getStackTrace()); + } + + try { + // test multiple = false but value is list + JSONDataValidation.validate(schema, schemaChildMap, jsonInput().replaceAll("true", "false")); + fail(); + } catch (ValidationException e) { + System.out.println(e.getMessage()); + } + + // verify that child objects are also validated + String childTest = "\"multiple\": false, \"typeName\": \"authorAffiliation\""; + try { + String trimmedStr = jsonInput().replaceAll("\\s{2,}", " "); + // test child object with multiple set to true + JSONDataValidation.validate(schema, schemaChildMap, trimmedStr.replace(childTest, childTest.replace("false", "true"))); + fail(); + } catch (ValidationException e) { + System.out.println(e.getMessage()); + } + + try { + // test dsDescription but dsDescriptionValue missing + JSONDataValidation.validate(schema, schemaChildMap, jsonInput().replace("typeName\": \"dsDescriptionValue", "typeName\": \"notdsDescriptionValue")); + fail(); + } catch (ValidationException e) { + System.out.println(e.getMessage()); + } + + try { + // test dsDescription but child dsDescriptionValue missing + JSONDataValidation.validate(schema, schemaChildMap, jsonInput().replace("dsDescriptionValue\":{", "notdsDescriptionValue\":{")); + fail(); + } catch (ValidationException e) { + System.out.println(e.getMessage()); + } + + try { + // test required dataType missing + JSONDataValidation.validate(schema, schemaChildMap, jsonInput().replaceAll("\"datasetContactName\"", "\"datasetContactAffiliation\"")); + fail(); + } catch (ValidationException e) { + System.out.println(e.getMessage()); + } + + try { + // test dataType not allowed + JSONDataValidation.validate(schema, schemaChildMap, jsonInput().replaceAll("\"datasetContactEmail\"", "\"datasetContactNotAllowed\"")); + fail(); + } catch (ValidationException e) { + System.out.println(e.getMessage()); + } + } +} diff --git a/src/test/resources/hiddenShapefiles.zip b/src/test/resources/hiddenShapefiles.zip new file mode 100644 index 00000000000..c733a2083ae Binary files /dev/null and b/src/test/resources/hiddenShapefiles.zip differ diff --git a/src/test/resources/json/export-formats.json b/src/test/resources/json/export-formats.json new file mode 100644 index 00000000000..b4dc0168629 --- /dev/null +++ b/src/test/resources/json/export-formats.json @@ -0,0 +1,83 @@ +{ + "status": "OK", + "data": { + "OAI_ORE": { + "displayName": "OAI_ORE", + "mediaType": "application/json", + "isHarvestable": false, + "isVisibleInUserInterface": true + }, + "Datacite": { + "displayName": "DataCite", + "mediaType": "application/xml", + "isHarvestable": true, + "isVisibleInUserInterface": true, + "XMLNameSpace": "http://datacite.org/schema/kernel-4", + "XMLSchemaLocation": "http://datacite.org/schema/kernel-4 http://schema.datacite.org/meta/kernel-4.5/metadata.xsd", + "XMLSchemaVersion": "4.5" + }, + "oai_dc": { + "displayName": "Dublin Core", + "mediaType": "application/xml", + "isHarvestable": true, + "isVisibleInUserInterface": false, + "XMLNameSpace": "http://www.openarchives.org/OAI/2.0/oai_dc/", + "XMLSchemaLocation": "http://www.openarchives.org/OAI/2.0/oai_dc.xsd", + "XMLSchemaVersion": "2.0" + }, + "oai_datacite": { + "displayName": "OpenAIRE", + "mediaType": "application/xml", + "isHarvestable": true, + "isVisibleInUserInterface": true, + "XMLNameSpace": "http://datacite.org/schema/kernel-4", + "XMLSchemaLocation": "http://schema.datacite.org/meta/kernel-4.1/metadata.xsd", + "XMLSchemaVersion": "4.1" + }, + "schema.org": { + "displayName": "Schema.org JSON-LD", + "mediaType": "application/json", + "isHarvestable": false, + "isVisibleInUserInterface": true + }, + "ddi": { + "displayName": "DDI Codebook v2", + "mediaType": "application/xml", + "isHarvestable": false, + "isVisibleInUserInterface": true, + "XMLNameSpace": "ddi:codebook:2_5", + "XMLSchemaLocation": "https://ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd", + "XMLSchemaVersion": "2.5" + }, + "dcterms": { + "displayName": "Dublin Core", + "mediaType": "application/xml", + "isHarvestable": false, + "isVisibleInUserInterface": true, + "XMLNameSpace": "http://purl.org/dc/terms/", + "XMLSchemaLocation": "http://dublincore.org/schemas/xmls/qdc/dcterms.xsd", + "XMLSchemaVersion": "2.0" + }, + "html": { + "displayName": "DDI HTML Codebook", + "mediaType": "text/html", + "isHarvestable": false, + "isVisibleInUserInterface": true + }, + "dataverse_json": { + "displayName": "JSON", + "mediaType": "application/json", + "isHarvestable": true, + "isVisibleInUserInterface": true + }, + "oai_ddi": { + "displayName": "DDI Codebook v2", + "mediaType": "application/xml", + "isHarvestable": true, + "isVisibleInUserInterface": false, + "XMLNameSpace": "ddi:codebook:2_5", + "XMLSchemaLocation": "https://ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd", + "XMLSchemaVersion": "2.5" + } + } +} diff --git a/src/test/resources/json/importGenericWithOtherId.json b/src/test/resources/json/importGenericWithOtherId.json new file mode 100644 index 00000000000..af9241393e9 --- /dev/null +++ b/src/test/resources/json/importGenericWithOtherId.json @@ -0,0 +1,307 @@ +{ + "UNF": "UNF", + "createTime": "2014-11-12 12:17:55 -05", + "distributionDate": "Distribution Date", + "id": 2, + "lastUpdateTime": "2014-11-12 12:20:32 -05", + "metadataBlocks": { + "astrophysics": { + "displayName": "Astronomy and Astrophysics Metadata", + "fields": [ + { + "multiple": true, + "typeClass": "controlledVocabulary", + "typeName": "astroType", + "value": [ + "Image", + "Mosaic", + "EventList" + ] + } + ] + }, + "citation": { + "displayName": "Citation Metadata", + "fields": [ + { + "multiple": false, + "typeClass": "primitive", + "typeName": "title", + "value": "My Dataset" + }, + { + "multiple": true, + "typeClass": "compound", + "typeName": "author", + "value": [ + { + "authorAffiliation": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorAffiliation", + "value": "Top" + }, + "authorIdentifier": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorIdentifier", + "value": "ellenid" + }, + "authorIdentifierScheme": { + "multiple": false, + "typeClass": "controlledVocabulary", + "typeName": "authorIdentifierScheme", + "value": "ORCID" + }, + "authorName": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorName", + "value": "Privileged, Pete" + } + }, + { + "authorAffiliation": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorAffiliation", + "value": "Bottom" + }, + "authorIdentifier": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorIdentifier", + "value": "audreyId" + }, + "authorIdentifierScheme": { + "multiple": false, + "typeClass": "controlledVocabulary", + "typeName": "authorIdentifierScheme", + "value": "DAISY" + }, + "authorName": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorName", + "value": "Awesome, Audrey" + } + } + ] + }, + { + "multiple": true, + "typeClass": "primitive", + "typeName": "datasetContact", + "value": [ + "pete@malinator.com" + ] + }, + { + "multiple": false, + "typeClass": "primitive", + "typeName": "dsDescription", + "value": "Here is my description" + }, + { + "multiple": true, + "typeClass": "controlledVocabulary", + "typeName": "subject", + "value": [ + "Arts and Humanities", + "Astronomy and Astrophysics", + "Business and Management" + ] + }, + { + "multiple": true, + "typeClass": "primitive", + "typeName": "keyword", + "value": [ + "keyword1", + "keyword2" + ] + }, + { + "multiple": true, + "typeClass": "compound", + "typeName": "otherId", + "value": [ + { + "otherIdAgency": { + "multiple": false, + "typeClass": "primitive", + "typeName": "otherIdAgency", + "value": "my agency" + }, + "otherIdValue": { + "multiple": false, + "typeClass": "primitive", + "typeName": "otherIdValue", + "value": "otherId" + } + }, + { + "otherIdAgency": { + "multiple": false, + "typeClass": "primitive", + "typeName": "otherIdAgency", + "value": "another agency" + }, + "otherIdValue": { + "multiple": false, + "typeClass": "primitive", + "typeName": "otherIdValue", + "value": "otherId2" + } + }, + { + "otherIdAgency": { + "multiple": false, + "typeClass": "primitive", + "typeName": "otherIdAgency", + "value": "another agency" + }, + "otherIdValue": { + "multiple": false, + "typeClass": "primitive", + "typeName": "otherIdValue", + "value": "doi:10.7910/DVN/TJCLKP" + } + } + ] + }, + { + "multiple": false, + "typeClass": "primitive", + "typeName": "depositor", + "value": "Ellen K" + }, + { + "multiple": false, + "typeClass": "primitive", + "typeName": "dateOfDeposit", + "value": "2014-11-12" + } + ] + }, + "geospatial": { + "displayName": "Geospatial Metadata", + "fields": [ + { + "multiple": true, + "typeClass": "compound", + "typeName": "geographicCoverage", + "value": [ + { + "city": { + "multiple": false, + "typeClass": "primitive", + "typeName": "city", + "value": "Arlington" + }, + "country": { + "multiple": false, + "typeClass": "controlledVocabulary", + "typeName": "country", + "value": "United States" + }, + "state": { + "multiple": false, + "typeClass": "primitive", + "typeName": "state", + "value": "MA" + } + }, + { + "city": { + "multiple": false, + "typeClass": "primitive", + "typeName": "city", + "value": "beachcity" + }, + "country": { + "multiple": false, + "typeClass": "controlledVocabulary", + "typeName": "country", + "value": "Aruba" + }, + "state": { + "multiple": false, + "typeClass": "primitive", + "typeName": "state", + "value": "beach" + } + } + ] + }, + { + "multiple": false, + "typeClass": "compound", + "typeName": "geographicBoundingBox", + "value": + { + "eastLongitude": { + "multiple": false, + "typeClass": "primitive", + "typeName": "eastLongitude", + "value": "23" + }, + "northLatitude": { + "multiple": false, + "typeClass": "primitive", + "typeName": "northLatitude", + "value": "786" + }, + "southLatitude": { + "multiple": false, + "typeClass": "primitive", + "typeName": "southLatitude", + "value": "34" + }, + "westLongitude": { + "multiple": false, + "typeClass": "primitive", + "typeName": "westLongitude", + "value": "45" + } + } + + } + ] + }, + "socialscience": { + "displayName": "Social Science and Humanities Metadata", + "fields": [ + { + "multiple": true, + "typeClass": "compound", + "typeName": "software", + "value": [ + { + "softwareName": { + "multiple": false, + "typeClass": "primitive", + "typeName": "softwareName", + "value": "softwareName" + }, + "softwareVersion": { + "multiple": false, + "typeClass": "primitive", + "typeName": "softwareVersion", + "value": "software version" + } + } + ] + }, + { + "multiple": false, + "typeClass": "primitive", + "typeName": "unitOfAnalysis", + "value": "unit of analysis" + } + ] + } + }, + "productionDate": "Production Date", + "versionState": "DRAFT" + } diff --git a/src/test/resources/json/importGenericWithoutOtherId.json b/src/test/resources/json/importGenericWithoutOtherId.json new file mode 100644 index 00000000000..ceb2263c2cf --- /dev/null +++ b/src/test/resources/json/importGenericWithoutOtherId.json @@ -0,0 +1,258 @@ +{ + "UNF": "UNF", + "createTime": "2014-11-12 12:17:55 -05", + "distributionDate": "Distribution Date", + "id": 2, + "lastUpdateTime": "2014-11-12 12:20:32 -05", + "metadataBlocks": { + "astrophysics": { + "displayName": "Astronomy and Astrophysics Metadata", + "fields": [ + { + "multiple": true, + "typeClass": "controlledVocabulary", + "typeName": "astroType", + "value": [ + "Image", + "Mosaic", + "EventList" + ] + } + ] + }, + "citation": { + "displayName": "Citation Metadata", + "fields": [ + { + "multiple": false, + "typeClass": "primitive", + "typeName": "title", + "value": "My Dataset" + }, + { + "multiple": true, + "typeClass": "compound", + "typeName": "author", + "value": [ + { + "authorAffiliation": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorAffiliation", + "value": "Top" + }, + "authorIdentifier": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorIdentifier", + "value": "ellenid" + }, + "authorIdentifierScheme": { + "multiple": false, + "typeClass": "controlledVocabulary", + "typeName": "authorIdentifierScheme", + "value": "ORCID" + }, + "authorName": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorName", + "value": "Privileged, Pete" + } + }, + { + "authorAffiliation": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorAffiliation", + "value": "Bottom" + }, + "authorIdentifier": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorIdentifier", + "value": "audreyId" + }, + "authorIdentifierScheme": { + "multiple": false, + "typeClass": "controlledVocabulary", + "typeName": "authorIdentifierScheme", + "value": "DAISY" + }, + "authorName": { + "multiple": false, + "typeClass": "primitive", + "typeName": "authorName", + "value": "Awesome, Audrey" + } + } + ] + }, + { + "multiple": true, + "typeClass": "primitive", + "typeName": "datasetContact", + "value": [ + "pete@malinator.com" + ] + }, + { + "multiple": false, + "typeClass": "primitive", + "typeName": "dsDescription", + "value": "Here is my description" + }, + { + "multiple": true, + "typeClass": "controlledVocabulary", + "typeName": "subject", + "value": [ + "Arts and Humanities", + "Astronomy and Astrophysics", + "Business and Management" + ] + }, + { + "multiple": true, + "typeClass": "primitive", + "typeName": "keyword", + "value": [ + "keyword1", + "keyword2" + ] + }, + { + "multiple": false, + "typeClass": "primitive", + "typeName": "depositor", + "value": "Ellen K" + }, + { + "multiple": false, + "typeClass": "primitive", + "typeName": "dateOfDeposit", + "value": "2014-11-12" + } + ] + }, + "geospatial": { + "displayName": "Geospatial Metadata", + "fields": [ + { + "multiple": true, + "typeClass": "compound", + "typeName": "geographicCoverage", + "value": [ + { + "city": { + "multiple": false, + "typeClass": "primitive", + "typeName": "city", + "value": "Arlington" + }, + "country": { + "multiple": false, + "typeClass": "controlledVocabulary", + "typeName": "country", + "value": "United States" + }, + "state": { + "multiple": false, + "typeClass": "primitive", + "typeName": "state", + "value": "MA" + } + }, + { + "city": { + "multiple": false, + "typeClass": "primitive", + "typeName": "city", + "value": "beachcity" + }, + "country": { + "multiple": false, + "typeClass": "controlledVocabulary", + "typeName": "country", + "value": "Aruba" + }, + "state": { + "multiple": false, + "typeClass": "primitive", + "typeName": "state", + "value": "beach" + } + } + ] + }, + { + "multiple": false, + "typeClass": "compound", + "typeName": "geographicBoundingBox", + "value": + { + "eastLongitude": { + "multiple": false, + "typeClass": "primitive", + "typeName": "eastLongitude", + "value": "23" + }, + "northLatitude": { + "multiple": false, + "typeClass": "primitive", + "typeName": "northLatitude", + "value": "786" + }, + "southLatitude": { + "multiple": false, + "typeClass": "primitive", + "typeName": "southLatitude", + "value": "34" + }, + "westLongitude": { + "multiple": false, + "typeClass": "primitive", + "typeName": "westLongitude", + "value": "45" + } + } + + } + ] + }, + "socialscience": { + "displayName": "Social Science and Humanities Metadata", + "fields": [ + { + "multiple": true, + "typeClass": "compound", + "typeName": "software", + "value": [ + { + "softwareName": { + "multiple": false, + "typeClass": "primitive", + "typeName": "softwareName", + "value": "softwareName" + }, + "softwareVersion": { + "multiple": false, + "typeClass": "primitive", + "typeName": "softwareVersion", + "value": "software version" + } + } + ] + }, + { + "multiple": false, + "typeClass": "primitive", + "typeName": "unitOfAnalysis", + "value": "unit of analysis" + } + ] + } + }, + "productionDate": "Production Date", + "versionState": "DRAFT" + } diff --git a/src/test/resources/own-cloud-downloads/greetings.zip b/src/test/resources/own-cloud-downloads/greetings.zip new file mode 100644 index 00000000000..6e166d385d1 Binary files /dev/null and b/src/test/resources/own-cloud-downloads/greetings.zip differ diff --git a/src/test/resources/own-cloud-downloads/shapes.zip b/src/test/resources/own-cloud-downloads/shapes.zip new file mode 100644 index 00000000000..99d5f36c895 Binary files /dev/null and b/src/test/resources/own-cloud-downloads/shapes.zip differ diff --git a/src/test/resources/tsv/whitespace-test.tsv b/src/test/resources/tsv/whitespace-test.tsv new file mode 100644 index 00000000000..5485c948825 --- /dev/null +++ b/src/test/resources/tsv/whitespace-test.tsv @@ -0,0 +1,10 @@ +#metadataBlock name dataverseAlias displayName + whitespaceDemo Whitespace Demo +#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id + whitespaceDemoOne One Trailing Space text 0 TRUE TRUE TRUE FALSE TRUE FALSE whitespaceDemo + whitespaceDemoTwo Two Leading Space text 1 TRUE TRUE TRUE FALSE TRUE FALSE whitespaceDemo + whitespaceDemoThree Three CV with errors text 2 TRUE TRUE TRUE FALSE TRUE FALSE whitespaceDemo +#controlledVocabulary DatasetField Value identifier displayOrder + whitespaceDemoThree CV1 0 + whitespaceDemoThree CV2 1 + whitespaceDemoThree CV3 2 diff --git a/tests/check_duplicate_properties.sh b/tests/check_duplicate_properties.sh new file mode 100755 index 00000000000..7d053bdba4b --- /dev/null +++ b/tests/check_duplicate_properties.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# This script will check Java *.properties files within the src dir for duplicates +# and print logs with file annotations about it. + +set -euo pipefail + +FAIL=0 + +while IFS= read -r -d '' FILE; do + + # Scan the whole file for duplicates + FILTER=$(grep -a -v -E "^(#.*|\s*$)" "$FILE" | cut -d"=" -f1 | sort | uniq -c | tr -s " " | { grep -vs "^ 1 " || true; }) + + # If there are any duplicates present, analyse further to point people to the source + if [ -n "$FILTER" ]; then + FAIL=1 + + echo "::group::$FILE" + for KEY in $(echo "$FILTER" | cut -d" " -f3); do + # Find duplicate lines' numbers by grepping for the KEY and cutting the number from the output + DUPLICATE_LINES=$(grep -n -E -e "^$KEY=" "$FILE" | cut -d":" -f1) + # Join the found line numbers for better error log + DUPLICATE_NUMBERS=$(echo "$DUPLICATE_LINES" | paste -sd ',') + + # This form will make Github annotate the lines in the PR that changes the properties file + for LINE_NUMBER in $DUPLICATE_LINES; do + echo "::error file=$FILE,line=$LINE_NUMBER::Found duplicate for key '$KEY' in lines $DUPLICATE_NUMBERS" + done + done + echo "::endgroup::" + fi +done < <( find "$(git rev-parse --show-toplevel)" -wholename "*/src/*.properties" -print0 ) + +if [ "$FAIL" -eq 1 ]; then + exit 1 +fi diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt index 44bbfdcceb7..e1dad7a75b1 100644 --- a/tests/integration-tests.txt +++ b/tests/integration-tests.txt @@ -1 +1 @@ -DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,DataRetrieverApiIT,ProvIT,S3AccessIT,OpenApiIT,InfoIT +DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,DataRetrieverApiIT,ProvIT,S3AccessIT,OpenApiIT,InfoIT,DatasetFieldsIT,SavedSearchIT,DatasetTypesIT \ No newline at end of file diff --git a/tests/shell/spec/spec_helper.sh b/tests/shell/spec/spec_helper.sh index 93f19083cd2..0cf2106f5e0 100644 --- a/tests/shell/spec/spec_helper.sh +++ b/tests/shell/spec/spec_helper.sh @@ -22,3 +22,4 @@ spec_helper_configure() { # Available functions: import, before_each, after_each, before_all, after_all : import 'support/custom_matcher' } + diff --git a/tests/verify_mdb_properties.sh b/tests/verify_mdb_properties.sh new file mode 100755 index 00000000000..bc62c0f503a --- /dev/null +++ b/tests/verify_mdb_properties.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +# This script will check our metadata block files and scan if the properties files contain all the matching keys. + +set -euo pipefail + +if ! which jbang > /dev/null 2>&1; then + echo "Cannot find jbang on path. Did you install it?" >&2 + exit 1 +fi +if ! which native-image > /dev/null 2>&1; then + echo "Cannot find GraalVM native-image on path. Did you install it?" >&2 + exit 1 +fi + +FAIL=0 + +# We need a small Java app here, replacing spaces, converting to lower case but especially to replace UTF-8 chars with nearest ascii / strip accents because of +# https://github.com/IQSS/dataverse/blob/dddcf29188a5c35174f3c94ffc1c4cb1d7fc0552/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java#L139-L140 +# This cannot be replaced by another tool, as it behaves rather individually. +DIR=$(mktemp -d) +SOURCE="$DIR/stripaccents.java" +STRIP_BIN="$(dirname "$0")/stripaccents" +cat > "$SOURCE" << EOF +///usr/bin/env jbang "\$0" "\$@" ; exit \$? +//JAVA 11+ +//DEPS org.apache.commons:commons-lang3:3.12.0 +import org.apache.commons.lang3.StringUtils; +import java.nio.charset.StandardCharsets; +import java.io.IOException; +class stripaccents { + public static void main(String[] args) throws IOException { + String input = new String(System.in.readAllBytes(), StandardCharsets.UTF_8).toLowerCase().replace(" ", "_"); + System.out.println(StringUtils.stripAccents(input)); + } +} +EOF +jbang export native --force --fresh -O "$STRIP_BIN" "$SOURCE" + +while IFS= read -r -d '' MDB; do + + echo "::group::$MDB" + BLOCK_NAME=$(sed -n "2p" "$MDB" | cut -f2) + BLOCK_DISPLAY_NAME=$(sed -n "2p" "$MDB" | cut -f4) + PROPERTIES_FILE="$(git rev-parse --show-toplevel)/src/main/java/propertyFiles/$BLOCK_NAME.properties" + + # Check correct file exists + if [ ! -r "$PROPERTIES_FILE" ]; then + echo "::error::Missing properties file for metadata block '$BLOCK_NAME', expected at '$PROPERTIES_FILE'" + FAIL=1 + continue + fi + + # Check metadata block properties exist and are equal to TSV source + if ! grep -a -q -e "^metadatablock.name=$BLOCK_NAME$" "$PROPERTIES_FILE"; then + echo "::error::Missing 'metadatablock.name=$BLOCK_NAME' or different from TSV source in $PROPERTIES_FILE" + FAIL=1 + fi + if ! grep -a -q -e "^metadatablock.displayName=$BLOCK_DISPLAY_NAME$" "$PROPERTIES_FILE"; then + echo "::error::Missing 'metadatablock.displayName=$BLOCK_DISPLAY_NAME' or different from TSV source in $PROPERTIES_FILE" + FAIL=1 + fi + if ! grep -a -q -e "^metadatablock.displayFacet=" "$PROPERTIES_FILE"; then + echo "::error::Missing 'metadatablock.displayFacet=...' in $PROPERTIES_FILE" + FAIL=1 + fi + + # Check dataset fields + for FIELD in $(grep -a -A1000 "^#datasetField" "$MDB" | tail -n+2 | grep -a -B1000 "^#controlledVocabulary" | head -n-1 | cut -f2); do + for ENTRY in title description watermark; do + if ! grep -a -q -e "^datasetfieldtype.$FIELD.$ENTRY=" "$PROPERTIES_FILE"; then + echo "::error::Missing key 'datasetfieldtype.$FIELD.$ENTRY=...' in $PROPERTIES_FILE" + FAIL=1 + fi + done + done + + # Check CV entries + while read -r LINE; do + FIELD_NAME=$(echo "$LINE" | cut -f1) + # See https://github.com/IQSS/dataverse/blob/dddcf29188a5c35174f3c94ffc1c4cb1d7fc0552/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java#L139-L140 + # Square brackets are special in grep with expressions activated, so escape them if present! + FIELD_VALUE=$(echo "$LINE" | cut -f2 | "$STRIP_BIN" | sed -e 's/\([][]\)/\\\1/g' ) + + if ! grep -q -a -e "^controlledvocabulary.$FIELD_NAME.$FIELD_VALUE=" "$PROPERTIES_FILE"; then + echo "::error::Missing key 'controlledvocabulary.$FIELD_NAME.$FIELD_VALUE=...' in $PROPERTIES_FILE" + FAIL=1 + fi + done < <(grep -a -A1000 "^#controlledVocabulary" "$MDB" | tail -n+2) + + echo "::endgroup::" + +done < <( find "$(git rev-parse --show-toplevel)/scripts/api/data/metadatablocks" -name '*.tsv' -print0 ) + +rm "$SOURCE" "$STRIP_BIN" + +if [ "$FAIL" -eq 1 ]; then + exit 1 +fi