Skip to content

Commit

Permalink
merged develop branch
Browse files Browse the repository at this point in the history
  • Loading branch information
ErykKul committed Apr 17, 2024
2 parents 53d0d85 + d9a7922 commit 9599f44
Show file tree
Hide file tree
Showing 597 changed files with 33,362 additions and 13,079 deletions.
3 changes: 2 additions & 1 deletion .env
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
APP_IMAGE=gdcc/dataverse:unstable
POSTGRES_VERSION=13
POSTGRES_VERSION=16
DATAVERSE_DB_USER=dataverse
SOLR_VERSION=9.3.0
SKIP_DEPLOY=0
4 changes: 4 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# https://www.git-scm.com/docs/gitattributes

# This set mandatory LF line endings for .sh files preventing from windows users to having to change the value of their git config --global core.autocrlf to 'false' or 'input'
*.sh text eol=lf
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/bug_report.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: Bug report
about: Did you encounter something unexpected or incorrect in the Dataverse software?
We'd like to hear about it!
title: ''
labels: ''
labels: 'Type: Bug'
assignees: ''

---
Expand Down
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/feature_request.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
name: Feature request
about: Suggest an idea or new feature for the Dataverse software!
title: 'Feature Request/Idea:'
labels: ''
labels: 'Type: Feature'
assignees: ''

---
Expand Down
101 changes: 101 additions & 0 deletions .github/workflows/maven_cache_management.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
name: Maven Cache Management

on:
# Every push to develop should trigger cache rejuvenation (dependencies might have changed)
push:
branches:
- develop
# According to https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy
# all caches are deleted after 7 days of no access. Make sure we rejuvenate every 7 days to keep it available.
schedule:
- cron: '23 2 * * 0' # Run for 'develop' every Sunday at 02:23 UTC (3:23 CET, 21:23 ET)
# Enable manual cache management
workflow_dispatch:
# Delete branch caches once a PR is merged
pull_request:
types:
- closed

env:
COMMON_CACHE_KEY: "dataverse-maven-cache"
COMMON_CACHE_PATH: "~/.m2/repository"

jobs:
seed:
name: Drop and Re-Seed Local Repository
runs-on: ubuntu-latest
if: ${{ github.event_name != 'pull_request' }}
permissions:
# Write permission needed to delete caches
# See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id
actions: write
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Determine Java version from Parent POM
run: echo "JAVA_VERSION=$(grep '<target.java.version>' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV}
- name: Set up JDK ${{ env.JAVA_VERSION }}
uses: actions/setup-java@v4
with:
java-version: ${{ env.JAVA_VERSION }}
distribution: temurin
- name: Seed common cache
run: |
mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins
# This non-obvious order is due to the fact that the download via Maven above will take a very long time (7-8 min).
# Jobs should not be left without a cache. Deleting and saving in one go leaves only a small chance for a cache miss.
- name: Drop common cache
run: |
gh extension install actions/gh-actions-cache
echo "🛒 Fetching list of cache keys"
cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -B develop | cut -f 1 )
## Setting this to not fail the workflow while deleting cache keys.
set +e
echo "🗑️ Deleting caches..."
for cacheKey in $cacheKeys
do
gh actions-cache delete $cacheKey -R ${{ github.repository }} -B develop --confirm
done
echo "✅ Done"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Save the common cache
uses: actions/cache@v4
with:
path: ${{ env.COMMON_CACHE_PATH }}
key: ${{ env.COMMON_CACHE_KEY }}
enableCrossOsArchive: true

# Let's delete feature branch caches once their PR is merged - we only have 10 GB of space before eviction kicks in
deplete:
name: Deplete feature branch caches
runs-on: ubuntu-latest
if: ${{ github.event_name == 'pull_request' }}
permissions:
# `actions:write` permission is required to delete caches
# See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id
actions: write
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Cleanup caches
run: |
gh extension install actions/gh-actions-cache
BRANCH=refs/pull/${{ github.event.pull_request.number }}/merge
echo "🛒 Fetching list of cache keys"
cacheKeysForPR=$(gh actions-cache list -R ${{ github.repository }} -B $BRANCH | cut -f 1 )
## Setting this to not fail the workflow while deleting cache keys.
set +e
echo "🗑️ Deleting caches..."
for cacheKey in $cacheKeysForPR
do
gh actions-cache delete $cacheKey -R ${{ github.repository }} -B $BRANCH --confirm
done
echo "✅ Done"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
117 changes: 103 additions & 14 deletions .github/workflows/maven_unit_test.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
name: Maven Unit Tests
name: Maven Tests

on:
push:
paths:
- "**.java"
- "**.sql"
- "pom.xml"
- "modules/**/pom.xml"
- "!modules/container-base/**"
- "!modules/dataverse-spi/**"
pull_request:
paths:
- "**.java"
- "**.sql"
- "pom.xml"
- "modules/**/pom.xml"
- "!modules/container-base/**"
Expand All @@ -28,6 +30,7 @@ jobs:
continue-on-error: ${{ matrix.experimental }}
runs-on: ubuntu-latest
steps:
# Basic setup chores
- uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.jdk }}
uses: actions/setup-java@v3
Expand All @@ -37,16 +40,110 @@ jobs:
cache: maven

# The reason why we use "install" here is that we want the submodules to be available in the next step.
# Also, we can cache them this way for jobs triggered by this one.
- name: Build with Maven
# Also, we can cache them this way for jobs triggered by this one. We need to skip ITs here, as we run
# them in the next job - but install usually runs through verify phase.
- name: Build with Maven and run unit tests
run: >
mvn -B -f modules/dataverse-parent
-Dtarget.java.version=${{ matrix.jdk }}
-DcompilerArgument=-Xlint:unchecked -P all-unit-tests
-DskipIntegrationTests
-pl edu.harvard.iq:dataverse -am
install
- name: Maven Code Coverage
# We don't want to cache the WAR file, so delete it
- run: rm -rf ~/.m2/repository/edu/harvard/iq/dataverse

# Upload the built war file. For download, it will be wrapped in a ZIP by GitHub.
# See also https://github.com/actions/upload-artifact#zipped-artifact-downloads
- uses: actions/upload-artifact@v3
with:
name: dataverse-java${{ matrix.jdk }}.war
path: target/dataverse*.war
retention-days: 7

# Store the build for the next step (integration test) to avoid recompilation and to transfer coverage reports
- run: |
tar -cvf java-builddir.tar target
tar -cvf java-m2-selection.tar ~/.m2/repository/io/gdcc/dataverse-*
- uses: actions/upload-artifact@v3
with:
name: java-artifacts
path: |
java-builddir.tar
java-m2-selection.tar
retention-days: 3

integration-test:
runs-on: ubuntu-latest
needs: unittest
name: (${{ matrix.status}} / JDK ${{ matrix.jdk }}) Integration Tests
strategy:
fail-fast: false
matrix:
jdk: [ '17' ]
experimental: [ false ]
status: [ "Stable" ]
#
# JDK 17 builds disabled due to non-essential fails marking CI jobs as completely failed within
# Github Projects, PR lists etc. This was consensus on Slack #dv-tech. See issue #8094
# (This is a limitation of how Github is currently handling these things.)
#
#include:
# - jdk: '17'
# experimental: true
# status: "Experimental"
continue-on-error: ${{ matrix.experimental }}
steps:
# Basic setup chores
- uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.jdk }}
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.jdk }}
distribution: temurin
cache: maven

# Get the build output from the unit test job
- uses: actions/download-artifact@v3
with:
name: java-artifacts
- run: |
tar -xvf java-builddir.tar
tar -xvf java-m2-selection.tar -C /
# Run integration tests (but not unit tests again)
- run: mvn -DskipUnitTests -Dtarget.java.version=${{ matrix.jdk }} verify

# Wrap up and send to coverage job
- run: tar -cvf java-reportdir.tar target/site
- uses: actions/upload-artifact@v3
with:
name: java-reportdir
path: java-reportdir.tar
retention-days: 3

coverage-report:
runs-on: ubuntu-latest
needs: integration-test
name: Coverage Report Submission
steps:
# Basic setup chores
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
with:
java-version: '17'
distribution: temurin
cache: maven

# Get the build output from the integration test job
- uses: actions/download-artifact@v3
with:
name: java-reportdir
- run: tar -xvf java-reportdir.tar

# Deposit Code Coverage
- name: Deposit Code Coverage
env:
CI_NAME: github
COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }}
Expand All @@ -57,22 +154,14 @@ jobs:
-DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
jacoco:report coveralls:report
# We don't want to cache the WAR file, so delete it
- run: rm -rf ~/.m2/repository/edu/harvard/iq/dataverse
# NOTE: this may be extended with adding a report to the build output, leave a comment, send to Sonarcloud, ...

# Upload the built war file. For download, it will be wrapped in a ZIP by GitHub.
# See also https://github.com/actions/upload-artifact#zipped-artifact-downloads
- uses: actions/upload-artifact@v3
with:
name: dataverse-java${{ matrix.jdk }}.war
path: target/dataverse*.war
retention-days: 7
push-app-img:
name: Publish App Image
permissions:
contents: read
packages: write
pull-requests: write
needs: unittest
needs: integration-test
uses: ./.github/workflows/container_app_push.yml
secrets: inherit
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -61,3 +61,4 @@ src/main/webapp/resources/images/dataverseproject.png.thumb140

# Docker development volumes
/docker-dev-volumes
/.vs
4 changes: 2 additions & 2 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,12 @@ If you are interested in working on the main Dataverse code, great! Before you s

Please read http://guides.dataverse.org/en/latest/developers/version-control.html to understand how we use the "git flow" model of development and how we will encourage you to create a GitHub issue (if it doesn't exist already) to associate with your pull request. That page also includes tips on making a pull request.

After making your pull request, your goal should be to help it advance through our kanban board at https://github.com/orgs/IQSS/projects/2 . If no one has moved your pull request to the code review column in a timely manner, please reach out. Note that once a pull request is created for an issue, we'll remove the issue from the board so that we only track one card (the pull request).
After making your pull request, your goal should be to help it advance through our kanban board at https://github.com/orgs/IQSS/projects/34 . If no one has moved your pull request to the code review column in a timely manner, please reach out. Note that once a pull request is created for an issue, we'll remove the issue from the board so that we only track one card (the pull request).

Thanks for your contribution!

[dataverse-community Google Group]: https://groups.google.com/group/dataverse-community
[Community Call]: https://dataverse.org/community-calls
[dataverse-dev Google Group]: https://groups.google.com/group/dataverse-dev
[community contributors]: https://docs.google.com/spreadsheets/d/1o9DD-MQ0WkrYaEFTD5rF_NtyL8aUISgURsAXSL7Budk/edit?usp=sharing
[dev efforts]: https://github.com/orgs/IQSS/projects/2#column-5298405
[dev efforts]: https://github.com/orgs/IQSS/projects/34/views/6
13 changes: 7 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
Dataverse&#174;
===============

Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Data Science and Products team](http://www.iq.harvard.edu/people/people/data-science-products) at the [Institute for Quantitative Social Science](http://iq.harvard.edu/) and the [Dataverse community][]).
Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Dataverse team](https://dataverse.org/about) at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][]).

[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our development [roadmap][], and more.
[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our [project board][], our development [roadmap][], and more.

We maintain a demo site at [demo.dataverse.org][] which you are welcome to use for testing and evaluating Dataverse.

Expand All @@ -15,7 +15,7 @@ We love contributors! Please see our [Contributing Guide][] for ways you can hel

Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States.

[![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg?raw=true "Dataverse Project")](http://dataverse.org)
[![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg "Dataverse Project")](http://dataverse.org)

[![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/)
[![API Test Coverage](https://img.shields.io/jenkins/coverage/jacoco?jobUrl=https%3A%2F%2Fjenkins.dataverse.org%2Fjob%2FIQSS-dataverse-develop&label=API%20Test%20Coverage)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/ws/target/coverage-it/index.html)
Expand All @@ -26,15 +26,16 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi
[dataverse.org]: https://dataverse.org
[demo.dataverse.org]: https://demo.dataverse.org
[Dataverse community]: https://dataverse.org/developers
[Installation Guide]: http://guides.dataverse.org/en/latest/installation/index.html
[Installation Guide]: https://guides.dataverse.org/en/latest/installation/index.html
[latest release]: https://github.com/IQSS/dataverse/releases
[features]: https://dataverse.org/software-features
[project board]: https://github.com/orgs/IQSS/projects/34
[roadmap]: https://www.iq.harvard.edu/roadmap-dataverse-project
[integrations]: https://dataverse.org/integrations
[REST APIs]: http://guides.dataverse.org/en/latest/api/index.html
[REST APIs]: https://guides.dataverse.org/en/latest/api/index.html
[Contributing Guide]: CONTRIBUTING.md
[mailing list]: https://groups.google.com/group/dataverse-community
[community call]: https://dataverse.org/community-calls
[chat.dataverse.org]: http://chat.dataverse.org
[chat.dataverse.org]: https://chat.dataverse.org
[Dataverse Community Meeting]: https://dataverse.org/events
[open source]: LICENSE.md
12 changes: 7 additions & 5 deletions conf/keycloak/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ version: "3.9"
services:

keycloak:
image: 'jboss/keycloak:16.1.1'
image: 'quay.io/keycloak/keycloak:21.0'
command:
- "start-dev"
- "--import-realm"
environment:
- KEYCLOAK_USER=kcadmin
- KEYCLOAK_PASSWORD=kcpassword
- KEYCLOAK_IMPORT=/tmp/oidc-realm.json
- KEYCLOAK_ADMIN=kcadmin
- KEYCLOAK_ADMIN_PASSWORD=kcpassword
- KEYCLOAK_LOGLEVEL=DEBUG
ports:
- "8090:8080"
volumes:
- './oidc-realm.json:/tmp/oidc-realm.json'
- './test-realm.json:/opt/keycloak/data/import/test-realm.json'
2 changes: 1 addition & 1 deletion conf/keycloak/oidc-keycloak-auth-provider.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
"factoryAlias": "oidc",
"title": "OIDC-Keycloak",
"subtitle": "OIDC-Keycloak",
"factoryData": "type: oidc | issuer: http://keycloak.mydomain.com:8090/realms/oidc-realm | clientId: oidc-client | clientSecret: ss6gE8mODCDfqesQaSG3gwUwZqZt547E",
"factoryData": "type: oidc | issuer: http://keycloak.mydomain.com:8090/realms/test | clientId: test | clientSecret: 94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8",
"enabled": true
}
Loading

0 comments on commit 9599f44

Please sign in to comment.