add S3 tests

.env.s3

@@ -0,0 +1,4 @@
+POSTGRES_VERSION=13
+DATAVERSE_DB_USER=dataverse
+SOLR_VERSION=9.3.0
+DATAVERSE_BOOTSTRAP_TIMEOUT=5m

.github/workflows/s3.yml

@@ -0,0 +1,82 @@
+name: S3
+
+on: workflow_dispatch
+
+jobs:
+  api:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone this repo
+        uses: actions/checkout@v3
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+      - name: Clone Dataverse repo
+        uses: actions/checkout@v3
+        with:
+          repository: IQSS/dataverse
+          path: dataverse
+          ref: develop
+      - name: Start containers
+        run: |
+          docker compose -f docker-compose-s3.yml --env-file .env.s3 up -d
+      - uses: emilioschepis/wait-for-endpoint@v1.0.3
+        with:
+          url: http://localhost:8080/api/info/version
+          timeout: 120000
+          interval: 10000
+      - name: Test metadatablocks
+        run: |
+          curl localhost:8080/api/metadatablocks
+      - uses: emilioschepis/wait-for-endpoint@v1.0.3
+        with:
+          url: http://localhost:8080/api/metadatablocks
+          timeout: 120000
+          interval: 10000
+      - name: Put SUSHI config file in place
+        run: |
+          docker ps
+          # When tests are run on localhost, Dataverse can access /tmp/sushi_sample_logs.json
+          # See https://github.com/IQSS/dataverse/blob/develop/src/test/java/edu/harvard/iq/dataverse/api/MakeDataCountApiIT.java#L56
+          docker exec test_dataverse sh -c 'curl https://raw.githubusercontent.com/IQSS/dataverse/develop/src/test/java/edu/harvard/iq/dataverse/makedatacount/sushi_sample_logs.json > /tmp/sushi_sample_logs.json && head /tmp/sushi_sample_logs.json'
+      - name: Test Search API
+        run: |
+          sleep 30
+          curl 'http://localhost:8080/api/search?q=*'
+      - name: Test Solr
+        run: |
+          curl http://localhost:8983/solr/collection1/schema/fields
+          curl 'http://localhost:8983/solr/collection1/select?q=*%3A*&wt=json&indent=true'
+      - name: Run database sequence
+        run: |
+          cd dataverse
+          export PGPASSWORD=secret
+          # use correct database name
+          sed -i 's/dvnapp/dataverse/g' doc/sphinx-guides/source/_static/util/createsequence.sql
+          psql -h localhost -U dataverse dataverse -f doc/sphinx-guides/source/_static/util/createsequence.sql
+      - name: Run API tests
+        run: |
+          cd dataverse
+          INTEGRATION_TESTS=$(<tests/integration-tests.txt)
+          # TODO: re-enable full test suite
+          #mvn test -Dtest=$INTEGRATION_TESTS -Ddataverse.test.baseurl=http://localhost:8080
+          mvn test -Dtest=S3AccessIT -Ddataverse.test.baseurl=http://localhost:8080
+      - name: Dump docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          dest: './logs'
+      - name: Tar logs
+        if: failure()
+        run: tar cvzf ./logs.tgz ./logs
+      - name: Upload logs to GitHub
+        if: failure()
+        uses: actions/upload-artifact@master
+        with:
+          name: logs.tgz
+          path: ./logs.tgz
+      - name: Stop containers
+        if: always()
+        run: docker compose -f docker-compose-s3.yml --env-file .env.s3 down

docker-compose-s3.yml

@@ -0,0 +1,168 @@
+version: '2.4'
+
+services:
+  test_dataverse:
+    container_name: 'test_dataverse'
+    hostname: dataverse
+    image: gdcc/dataverse:unstable
+    restart: on-failure
+    user: payara
+    environment:
+      DATAVERSE_DB_HOST: postgres
+      DATAVERSE_DB_PASSWORD: secret
+      DATAVERSE_DB_USER: ${DATAVERSE_DB_USER}
+      ENABLE_JDWP: "1"
+      DATAVERSE_FEATURE_API_BEARER_AUTH: "1"
+      DATAVERSE_AUTH_OIDC_ENABLED: "1"
+      DATAVERSE_AUTH_OIDC_CLIENT_ID: test
+      DATAVERSE_AUTH_OIDC_CLIENT_SECRET: 94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8
+      DATAVERSE_AUTH_OIDC_AUTH_SERVER_URL: http://keycloak.mydomain.com:8090/realms/test
+      DATAVERSE_JSF_REFRESH_PERIOD: "1"
+      # to get HarvestingServerIT to pass
+      dataverse_oai_server_maxidentifiers: 2
+      dataverse_oai_server_maxrecords: 2
+      JVM_ARGS: -Ddataverse.files.storage-driver-id=file1
+        -Ddataverse.files.file1.type=file
+        -Ddataverse.files.file1.label=Filesystem
+        -Ddataverse.files.file1.directory=${STORAGE_DIR}/store
+        -Ddataverse.files.localstack1.type=s3
+        -Ddataverse.files.localstack1.label=LocalStack
+        -Ddataverse.files.localstack1.custom-endpoint-url=http://localstack:4566
+        -Ddataverse.files.localstack1.custom-endpoint-region=us-east-2
+        -Ddataverse.files.localstack1.bucket-name=mybucket
+        -Ddataverse.files.localstack1.path-style-access=true
+        -Ddataverse.files.localstack1.upload-redirect=true
+        -Ddataverse.files.localstack1.access-key=default
+        -Ddataverse.files.localstack1.secret-key=default
+        -Ddataverse.files.minio1.type=s3
+        -Ddataverse.files.minio1.label=MinIO
+        -Ddataverse.files.minio1.custom-endpoint-url=http://minio:9000
+        -Ddataverse.files.minio1.custom-endpoint-region=us-east-1
+        -Ddataverse.files.minio1.bucket-name=mybucket
+        -Ddataverse.files.minio1.path-style-access=true
+        -Ddataverse.files.minio1.upload-redirect=false
+        -Ddataverse.files.minio1.access-key=minioadmin
+        -Ddataverse.files.minio1.secret-key=minioadmin
+    ports:
+      - '8080:8080'
+    networks:
+      - dataverse
+    depends_on:
+      - test_postgres
+      - test_solr
+    tmpfs:
+      - /dumps:mode=770,size=2052M,uid=1000,gid=1000
+      - /tmp:mode=770,size=2052M,uid=1000,gid=1000
+    mem_limit: 2147483648 # 2 GiB
+    mem_reservation: 1024m
+    privileged: false
+
+  test_dataverse_bootstrap:
+    container_name: 'test_dataverse_bootstrap'
+    image: gdcc/configbaker:unstable
+    restart: 'no'
+    environment:
+      - TIMEOUT=${DATAVERSE_BOOTSTRAP_TIMEOUT}
+    command:
+      - bootstrap.sh
+      - dev
+    networks:
+      - dataverse
+
+  test_postgres:
+    container_name: 'test_postgres'
+    hostname: postgres
+    image: postgres:${POSTGRES_VERSION}
+    restart: on-failure
+    environment:
+      - POSTGRES_USER=${DATAVERSE_DB_USER}
+      - POSTGRES_PASSWORD=secret
+    ports:
+      - '5432:5432'
+    networks:
+      - dataverse
+
+  test_solr_initializer:
+    container_name: 'test_solr_initializer'
+    image: gdcc/configbaker:unstable
+    restart: 'no'
+    command:
+      - sh
+      - -c
+      - 'fix-fs-perms.sh solr && cp -a /template/* /solr-template'
+    volumes:
+      - ${RUNNER_TEMP}/solr/data:/var/solr
+      - ${RUNNER_TEMP}/solr/conf:/solr-template
+
+  test_solr:
+    container_name: 'test_solr'
+    hostname: 'solr'
+    image: solr:${SOLR_VERSION}
+    depends_on:
+      - test_solr_initializer
+    restart: on-failure
+    ports:
+      - '8983:8983'
+    networks:
+      - dataverse
+    command:
+      - 'solr-precreate'
+      - 'collection1'
+      - '/template'
+    volumes:
+      - ${RUNNER_TEMP}/solr/data:/var/solr
+      - ${RUNNER_TEMP}/solr/conf:/template
+
+  test_smtp:
+    container_name: 'test_smtp'
+    hostname: 'smtp'
+    image: maildev/maildev:2.0.5
+    restart: on-failure
+    expose:
+      - '25' # smtp server
+    environment:
+      - MAILDEV_SMTP_PORT=25
+      - MAILDEV_MAIL_DIRECTORY=/mail
+    networks:
+      - dataverse
+    tmpfs:
+      - /mail:mode=770,size=128M,uid=1000,gid=1000
+  dev_localstack:
+    container_name: "dev_localstack"
+    hostname: "localstack"
+    image: localstack/localstack:2.3.2
+    restart: on-failure
+    ports:
+      - "127.0.0.1:4566:4566"
+    environment:
+      - DEBUG=${DEBUG-}
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - HOSTNAME_EXTERNAL=localstack
+    networks:
+      - dataverse
+    volumes:
+      - ./conf/localstack:/etc/localstack/init/ready.d
+    tmpfs:
+      - /localstack:mode=770,size=128M,uid=1000,gid=1000
+
+  dev_minio:
+    container_name: "dev_minio"
+    hostname: "minio"
+    image: minio/minio
+    restart: on-failure
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    networks:
+      - dataverse
+    volumes:
+      - minio_storage:/data
+    environment:
+      # these are the defaults but are here for clarity
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    command: server /data
+
+networks:
+  dataverse:
+    driver: bridge