From cc92f9c0e7e626c095510fd5cc50d0cc410bb4b8 Mon Sep 17 00:00:00 2001 From: pallinger Date: Tue, 19 Sep 2023 14:08:51 +0200 Subject: [PATCH 01/22] Language update fix (#236) * fix updating languages for multiple servers -- language files will be downloaded and packed locally on each server * bundle.properties source is remote --------- Co-authored-by: Peter Pallinger --- tasks/dataverse-languages.yml | 43 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/tasks/dataverse-languages.yml b/tasks/dataverse-languages.yml index 155de396..e970d8a7 100644 --- a/tasks/dataverse-languages.yml +++ b/tasks/dataverse-languages.yml @@ -4,6 +4,9 @@ debug: msg: '##### DATAVERSE LANGUAGES #####' +- set_fact: + lang_git_dir: /usr/local/src/dataverse_language_packs + - name: get dataverse language file path shell: "{{ payara_dir }}/bin/asadmin list-jvm-options | grep dataverse.lang.directory | sed 's/.*=//'" register: dataverse_lang_directory @@ -16,46 +19,46 @@ owner: '{{ dataverse.payara.user }}' when: (dataverse_lang_directory.stdout | trim) == '' -- name: set dataverse language file path if not set - shell: "{{ payara_dir }}/bin/asadmin create-jvm-options -Ddataverse.lang.directory={{ dataverse.language.lang_directory }}" - when: (dataverse_lang_directory.stdout | trim) == '' - -- name: clone dataverse language packs - local_action: - module: git - repo: "{{ dataverse.language.language_packs.source }}" - dest: "/tmp/dataverse_language_packs" - version: "{{ dataverse.language.language_packs.version }}" - run_once: true - - name: copy default bundle to the language directory if it was just created copy: - src: /tmp/dataverse_language_packs/en_US/Bundle.properties + src: "{{ lang_git_dir }}/en_US/Bundle.properties" dest: "{{ dataverse.language.lang_directory }}" owner: '{{ dataverse.payara.user }}' + remote_src: yes + when: (dataverse_lang_directory.stdout | trim) == '' + +- name: set dataverse language file path if not set + shell: "{{ payara_dir }}/bin/asadmin create-jvm-options -Ddataverse.lang.directory={{ dataverse.language.lang_directory }}" when: (dataverse_lang_directory.stdout | trim) == '' - name: restart payara after setting language directory service: name=payara state=restarted when: (dataverse_lang_directory.stdout | trim) == '' +- name: clone dataverse language packs + git: + repo: "{{ dataverse.language.language_packs.source }}" + dest: "{{ lang_git_dir }}" + version: "{{ dataverse.language.language_packs.version }}" + run_once: true + - name: prepare language file temporary directory - local_action: shell cd /tmp/dataverse_language_packs ; rm -rf tmp.bak ; [ -d tmp ] && mv tmp tmp.bak && rm tmp.bak/*.zip ; mkdir tmp + shell: cd {{ lang_git_dir }} ; rm -rf tmp.bak ; [ -d tmp ] && mv tmp tmp.bak && rm tmp.bak/*.zip ; mkdir tmp changed_when: false - name: copy language files to temporary directory - local_action: shell cd /tmp/dataverse_language_packs ; cp -R {{ item.locale }}*/*.properties tmp/ + shell: cd {{ lang_git_dir }} ; cp -R {{ item.locale }}*/*.properties tmp/ changed_when: false with_items: "{{ dataverse.language.languages }}" - name: check if there was a change in the temporary directory - local_action: shell cd /tmp/dataverse_language_packs ; diff -r tmp tmp.bak + shell: cd {{ lang_git_dir }} ; diff -r tmp tmp.bak register: diff changed_when: diff.rc != 0 failed_when: diff.rc > 2 - name: create language pack - local_action: shell cd /tmp/dataverse_language_packs/tmp ; zip languages.zip *.properties + shell: cd {{ lang_git_dir }}/tmp ; zip languages.zip *.properties when: diff.changed - name: upload language pack @@ -64,11 +67,13 @@ method: POST headers: Content-type: "application/zip" - Accept: application/json - src: /tmp/dataverse_language_packs/tmp/languages.zip +# Accept: application/json + src: "{{ lang_git_dir }}/tmp/languages.zip" + remote_src: yes status_code: 200 body_format: raw when: diff.changed + notify: enable and restart payara - name: configure available languages uri: From 90f53e37f0b189635855d2c3583880d2d2929d81 Mon Sep 17 00:00:00 2001 From: pallinger Date: Tue, 19 Sep 2023 14:12:09 +0200 Subject: [PATCH 02/22] Dataverse deploy upgrade only (#287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * allow for quick upgrade for dataverse war, with fewer side effects - introduce dataverse.deployment.upgrade_only variable, false by default - if it is false, everything goes on as before - if it is set to true, the install script is not called, and a payara redeploy is called instead * duplicate new group vars to tests/ * fix upgrade_only variable name --------- Co-authored-by: Pallinger Péter Co-authored-by: Don Sizemore --- defaults/main.yml | 2 ++ tasks/dataverse-install.yml | 8 ++++++++ tasks/dataverse-postinstall.yml | 1 + tests/group_vars/jenkins.yml | 2 ++ tests/group_vars/memorytests.yml | 2 ++ tests/group_vars/vagrant.yml | 2 ++ 6 files changed, 17 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index 09851aad..368984b3 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -257,6 +257,8 @@ dataverse: enabled: false argument: '-DcompilerArgument=-Xlint:unchecked test -P all-unit-tests' usermgmtkey: burrito + deployment: + upgrade_only: false version: '6.0' # robots_txt_url: https://guides.dataverse.org/en/latest/_downloads/3a5cd7a283eecd5e93289e30af713554/robots.txt diff --git a/tasks/dataverse-install.yml b/tasks/dataverse-install.yml index de9eb9e0..7ffc5ca8 100644 --- a/tasks/dataverse-install.yml +++ b/tasks/dataverse-install.yml @@ -143,12 +143,20 @@ become_user: '{{ dataverse.payara.user }}' args: chdir: /tmp/dvinstall + when: not dataverse.deployment.upgrade_only + +- name: upgrade only -- deploy dataverse war + shell: '{{ payara_dir }}/bin/asadmin redeploy --name dataverse /tmp/dvinstall/dataverse.war' + become: yes + become_user: '{{ dataverse.payara.user }}' + when: dataverse.deployment.upgrade_only - name: stop payara manually (systemd gums up the works) become: yes become_user: "{{ dataverse.payara.user }}" shell: '{{ payara_dir }}/bin/asadmin stop-domain {{ dataverse.payara.domain }}' ignore_errors: yes + when: not dataverse.deployment.upgrade_only - name: now start payara with systemd service: diff --git a/tasks/dataverse-postinstall.yml b/tasks/dataverse-postinstall.yml index aaab598b..93961cee 100644 --- a/tasks/dataverse-postinstall.yml +++ b/tasks/dataverse-postinstall.yml @@ -98,3 +98,4 @@ ansible.builtin.get_url: url: http://localhost:8080/api/admin/index dest: /tmp/index_status.out + when: not dataverse.deployment.upgrade_only diff --git a/tests/group_vars/jenkins.yml b/tests/group_vars/jenkins.yml index 0929cb4f..dadd97a2 100644 --- a/tests/group_vars/jenkins.yml +++ b/tests/group_vars/jenkins.yml @@ -248,6 +248,8 @@ dataverse: enabled: true argument: '-DcompilerArgument=-Xlint:unchecked test -P all-unit-tests' usermgmtkey: burrito + deployment: + upgrade_only: false version: '6.0' build_guides: false diff --git a/tests/group_vars/memorytests.yml b/tests/group_vars/memorytests.yml index 12f105ca..2b509cc2 100644 --- a/tests/group_vars/memorytests.yml +++ b/tests/group_vars/memorytests.yml @@ -250,6 +250,8 @@ dataverse: enabled: false argument: '-DcompilerArgument=-Xlint:unchecked test -P all-unit-tests' usermgmtkey: burrito + deployment: + upgrade_only: false version: '5.14' build_guides: false diff --git a/tests/group_vars/vagrant.yml b/tests/group_vars/vagrant.yml index 6663d363..95bf672e 100644 --- a/tests/group_vars/vagrant.yml +++ b/tests/group_vars/vagrant.yml @@ -253,6 +253,8 @@ dataverse: enabled: true argument: '-DcompilerArgument=-Xlint:unchecked test -P all-unit-tests' usermgmtkey: burrito + deployment: + upgrade_only: false version: '6.0' build_guides: false From 7d8a425a8e1293717f31c2fd6d01c1d1801cf625 Mon Sep 17 00:00:00 2001 From: pallinger Date: Tue, 19 Sep 2023 14:12:57 +0200 Subject: [PATCH 03/22] fail early when user set to update licenses is not a superuser (#324) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Pallinger Péter --- tasks/dataverse-licenses.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/dataverse-licenses.yml b/tasks/dataverse-licenses.yml index 8a82e672..2499dc07 100644 --- a/tasks/dataverse-licenses.yml +++ b/tasks/dataverse-licenses.yml @@ -11,7 +11,7 @@ login_user: "{{ db.postgres.user }}" login_password: "{{ db.postgres.pass }}" login_host: "{{ db.postgres.host }}" - query: SELECT tokenstring FROM apitoken a JOIN authenticateduser u ON a.authenticateduser_id=u.id WHERE useridentifier='{{ dataverse.licenses.user }}' + query: SELECT tokenstring FROM apitoken a JOIN authenticateduser u ON a.authenticateduser_id=u.id WHERE useridentifier='{{ dataverse.licenses.user }}' AND superuser changed_when: false register: token_result failed_when: token_result.rowcount != 1 From 6cfe5cf57527bab6fc119f1b90501d3d5edf2b82 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 3 Oct 2023 10:14:07 -0400 Subject: [PATCH 04/22] update jacoco path for unit tests #325 --- tasks/jacoco-generate_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/jacoco-generate_report.yml b/tasks/jacoco-generate_report.yml index 694fbd20..1fe05132 100644 --- a/tasks/jacoco-generate_report.yml +++ b/tasks/jacoco-generate_report.yml @@ -16,7 +16,7 @@ mode: '0755' - name: merge jacoco exec files - shell: 'java -jar {{ dataverse.jacoco.home }}/lib/jacococli.jar merge {{ payara_dir }}/glassfish/domains/{{ dataverse.payara.domain }}/config/jacoco.exec {{ dataverse.srcdir }}/target/jacoco.exec --destfile {{ dataverse.srcdir }}/target/jacoco_merged.exec' + shell: 'java -jar {{ dataverse.jacoco.home }}/lib/jacococli.jar merge {{ payara_dir }}/glassfish/domains/{{ dataverse.payara.domain }}/config/jacoco.exec {{ dataverse.srcdir }}/target/coverage-reports/jacoco-unit.exec --destfile {{ dataverse.srcdir }}/target/jacoco_merged.exec' args: chdir: '{{ dataverse.jacoco.home }}' From d83118388dab48353be16aac7c3aa524e0fd1283 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 3 Oct 2023 16:11:42 -0400 Subject: [PATCH 05/22] Update ec2-create-instance.sh, fix coverage-reports/jacoco-unit.exec --- ec2/ec2-create-instance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ec2/ec2-create-instance.sh b/ec2/ec2-create-instance.sh index bdb58e19..63912bc7 100755 --- a/ec2/ec2-create-instance.sh +++ b/ec2/ec2-create-instance.sh @@ -257,7 +257,7 @@ if [ ! -z "$LOCAL_LOG_PATH" ]; then rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/mvn.out $LOCAL_LOG_PATH/ # 4 jacoco rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/coverage-it $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/*.exec $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/coverage-reports/jacoco-unit.exec $LOCAL_LOG_PATH/ rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/classes $LOCAL_LOG_PATH/ rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/src $LOCAL_LOG_PATH/ # 5 server.logs From 382ff0dd1b36d6c588bd7bfce341f6ae47d225aa Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 4 Oct 2023 15:02:05 -0400 Subject: [PATCH 06/22] Update ec2-create-instance.sh get merged file --- ec2/ec2-create-instance.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ec2/ec2-create-instance.sh b/ec2/ec2-create-instance.sh index 63912bc7..abd68827 100755 --- a/ec2/ec2-create-instance.sh +++ b/ec2/ec2-create-instance.sh @@ -258,6 +258,7 @@ if [ ! -z "$LOCAL_LOG_PATH" ]; then # 4 jacoco rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/coverage-it $LOCAL_LOG_PATH/ rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/coverage-reports/jacoco-unit.exec $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/jacoco_merged.exec $LOCAL_LOG_PATH/ rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/classes $LOCAL_LOG_PATH/ rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/src $LOCAL_LOG_PATH/ # 5 server.logs From 61f0317b06365195d51ea9d6265df2b2beec2e3e Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 25 Oct 2023 07:53:07 -0400 Subject: [PATCH 07/22] bump Jenkins maven version to 3.9.5 --- tests/group_vars/jenkins.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/group_vars/jenkins.yml b/tests/group_vars/jenkins.yml index dadd97a2..b0295d72 100644 --- a/tests/group_vars/jenkins.yml +++ b/tests/group_vars/jenkins.yml @@ -306,7 +306,7 @@ localstack: web_ui: 8888 maven: - version: 3.8.8 + version: 3.9.5 munin: install: false From 1eafc8197f678affc47b48e54635b5dd669be14e Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 25 Oct 2023 13:21:59 -0400 Subject: [PATCH 08/22] #330 accomodate oliver's new FS checks during warfile packing and installation --- defaults/main.yml | 4 ++-- tasks/dataverse-branch.yml | 12 ++++++++++++ tasks/dataverse-build.yml | 8 ++++++++ tasks/dataverse-install.yml | 8 ++++++++ tasks/payara.yml | 18 +++--------------- tasks/payara_service_account.yml | 11 +++++++++++ tests/group_vars/jenkins.yml | 2 +- tests/group_vars/vagrant.yml | 4 ++-- 8 files changed, 47 insertions(+), 20 deletions(-) create mode 100644 tasks/payara_service_account.yml diff --git a/defaults/main.yml b/defaults/main.yml index 368984b3..646b4cd3 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -244,14 +244,14 @@ dataverse: service_email: noreply@dataverse.yourinstitution.edu smtp: localhost # or the FQDN of your organization's SMTP relay solr: - download_url: https://dlcdn.apache.org/solr/solr/9.3.0/solr-9.3.0.tgz + download_url: https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz group: solr root: /usr/local/solr user: solr version: 9.3.0 listen: 127.0.0.1 checksum: sha256:2a924a776d7c06dea561f12bdb17291dbaecc9e9a06f363d4599acb7fb7bfa71 - srcdir: /tmp/dataverse + srcdir: /home/dataverse/dataverse thumbnails: false unittests: enabled: false diff --git a/tasks/dataverse-branch.yml b/tasks/dataverse-branch.yml index 88251a88..bb83e4ed 100644 --- a/tasks/dataverse-branch.yml +++ b/tasks/dataverse-branch.yml @@ -4,9 +4,21 @@ debug: msg: '##### CLONE BRANCH #####' +- name: ensure service account and group exist + ansible.builtin.import_tasks: payara_service_account.yml + +- name: ensure target dir exists + ansible.builtin.file: + path: '{{ dataverse.srcdir }}' + state: directory + owner: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.user }}' + - name: clone specified branch git: repo: "{{ dataverse_repo }}" dest: "{{ dataverse.srcdir }}" version: "{{ dataverse_branch }}" when: dataverse_branch != "release" + become: true + become_user: '{{ dataverse.payara.user }}' diff --git a/tasks/dataverse-build.yml b/tasks/dataverse-build.yml index c6872b97..22b9696b 100644 --- a/tasks/dataverse-build.yml +++ b/tasks/dataverse-build.yml @@ -32,6 +32,8 @@ chdir: "{{ dataverse.srcdir }}" when: dataverse.unittests.enabled == false and maven.version == 'default' + become: true + become_user: '{{ dataverse.payara.user }}' - name: build warfile without tests and custom maven. tail /tmp/dataverse/mvn.out for gory details. shell: "source /etc/profile.d/maven.sh && mvn -T 2C clean package -Dmaven.test.skip=true > mvn.out" @@ -39,6 +41,8 @@ chdir: "{{ dataverse.srcdir }}" when: dataverse.unittests.enabled == false and maven.version != 'default' + become: true + become_user: '{{ dataverse.payara.user }}' - name: build warfile with specified tests. tail /tmp/dataverse/mvn.out for gory details. shell: "mvn {{ dataverse.unittests.argument }} {{ jacoco_arg }} -T 2C clean package > mvn.out" @@ -46,6 +50,8 @@ chdir: "{{ dataverse.srcdir }}" when: dataverse.unittests.enabled == true and maven.version == 'default' + become: true + become_user: '{{ dataverse.payara.user }}' - name: build warfile with specified tests using custom maven. tail /tmp/dataverse/mvn.out for gory details. shell: "source /etc/profile.d/maven.sh && mvn {{ dataverse.unittests.argument }} {{ jacoco_arg }} -T 2C clean package > mvn.out" @@ -53,6 +59,8 @@ chdir: "{{ dataverse.srcdir }}" when: dataverse.unittests.enabled == true and maven.version != 'default' + become: true + become_user: '{{ dataverse.payara.user }}' - name: copy warfile over release copy: diff --git a/tasks/dataverse-install.yml b/tasks/dataverse-install.yml index 7ffc5ca8..110b72d4 100644 --- a/tasks/dataverse-install.yml +++ b/tasks/dataverse-install.yml @@ -10,6 +10,14 @@ debug: msg: '##### DATAVERSE INSTALLER #####' +# this is necessary for the installer. dataverse-storage.yml does the real work later. +- name: filesdir must pre-exist + ansible.builtin.file: + path: '{{ dataverse.filesdirs[0].path }}' + state: directory + owner: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.group }}' + - ansible.builtin.import_tasks: dataverse-build.yml when: dataverse_branch != "release" diff --git a/tasks/payara.yml b/tasks/payara.yml index 65dae6bf..a2d605e7 100644 --- a/tasks/payara.yml +++ b/tasks/payara.yml @@ -35,11 +35,8 @@ when: payara_zip_download.changed notify: enable and restart payara -- name: create payara service group - group: name={{ dataverse.payara.group }} state=present - -- name: create payara service account - user: name={{ dataverse.payara.user }} group={{ dataverse.payara.group }} +- name: ensure service account and group exist + ansible.builtin.import_tasks: payara_service_account.yml - name: payara service account should own {{ dataverse.payara.domain }} file: path={{ payara_dir }}/glassfish/domains/{{ dataverse.payara.domain }} owner={{ dataverse.payara.user }} @@ -136,16 +133,7 @@ mode: '0750' state: directory -- name: workaround for FISH-7665 - lineinfile: - path: '{{ payara_dir }}/glassfish/config/asenv.conf' - line: '{{ item }}' - with_items: - - 'export JAVA_TOOL_OPTIONS' - - 'JAVA_TOOL_OPTIONS="-Djdk.util.zip.disableZip64ExtraFieldValidation=true --add-opens=java.base/java.io=ALL-UNNAMED"' - when: dataverse.payara.zipurl is match(".*payara-6.*") - -- name: workaround for FISH-7665 +- name: so installations can still install payara5 if they want template: src: payara6.service.j2 dest: /usr/lib/systemd/system/payara.service diff --git a/tasks/payara_service_account.yml b/tasks/payara_service_account.yml new file mode 100644 index 00000000..4b05d613 --- /dev/null +++ b/tasks/payara_service_account.yml @@ -0,0 +1,11 @@ +--- + +- name: ensure payara service group exists + ansible.builtin.group: + name: '{{ dataverse.payara.group }}' + state: present + +- name: ensure payara service account exists + ansible.builtin.user: + name: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.group }}' diff --git a/tests/group_vars/jenkins.yml b/tests/group_vars/jenkins.yml index b0295d72..263536fc 100644 --- a/tests/group_vars/jenkins.yml +++ b/tests/group_vars/jenkins.yml @@ -242,7 +242,7 @@ dataverse: version: 9.3.0 listen: 127.0.0.1 checksum: sha256:2a924a776d7c06dea561f12bdb17291dbaecc9e9a06f363d4599acb7fb7bfa71 - srcdir: /tmp/dataverse + srcdir: /home/dataverse/dataverse thumbnails: false unittests: enabled: true diff --git a/tests/group_vars/vagrant.yml b/tests/group_vars/vagrant.yml index 95bf672e..77727bdd 100644 --- a/tests/group_vars/vagrant.yml +++ b/tests/group_vars/vagrant.yml @@ -240,14 +240,14 @@ dataverse: service_email: noreply@dataverse.yourinstitution.edu smtp: localhost solr: - download_url: https://dlcdn.apache.org/solr/solr/9.3.0/solr-9.3.0.tgz + download_url: https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz group: solr root: /usr/local/solr user: solr version: 9.3.0 listen: 127.0.0.1 checksum: sha256:2a924a776d7c06dea561f12bdb17291dbaecc9e9a06f363d4599acb7fb7bfa71 - srcdir: /tmp/dataverse + srcdir: /home/dataverse/dataverse thumbnails: false unittests: enabled: true From 2ad51836245690c26cb1daa5c9154245aa241bcf Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 25 Oct 2023 15:28:11 -0400 Subject: [PATCH 09/22] Update tasks/dataverse-build.yml write mvn.out to correct location Co-authored-by: Philip Durbin --- tasks/dataverse-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/dataverse-build.yml b/tasks/dataverse-build.yml index 22b9696b..66757d56 100644 --- a/tasks/dataverse-build.yml +++ b/tasks/dataverse-build.yml @@ -35,7 +35,7 @@ become: true become_user: '{{ dataverse.payara.user }}' -- name: build warfile without tests and custom maven. tail /tmp/dataverse/mvn.out for gory details. +- name: build warfile without tests and custom maven. tail /home/dataverse/dataverse/mvn.out for gory details. shell: "source /etc/profile.d/maven.sh && mvn -T 2C clean package -Dmaven.test.skip=true > mvn.out" args: chdir: "{{ dataverse.srcdir }}" From 6dcf9c7389e16a0bb461687cf266167a209c88b8 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 25 Oct 2023 15:28:28 -0400 Subject: [PATCH 10/22] Update tasks/dataverse-build.yml Co-authored-by: Philip Durbin --- tasks/dataverse-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/dataverse-build.yml b/tasks/dataverse-build.yml index 66757d56..4c332b46 100644 --- a/tasks/dataverse-build.yml +++ b/tasks/dataverse-build.yml @@ -53,7 +53,7 @@ become: true become_user: '{{ dataverse.payara.user }}' -- name: build warfile with specified tests using custom maven. tail /tmp/dataverse/mvn.out for gory details. +- name: build warfile with specified tests using custom maven. tail /home/dataverse/dataverse/mvn.out for gory details. shell: "source /etc/profile.d/maven.sh && mvn {{ dataverse.unittests.argument }} {{ jacoco_arg }} -T 2C clean package > mvn.out" args: chdir: "{{ dataverse.srcdir }}" From aee8c8584dbfdb8645561ad2cf5cd9701ae0e940 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 25 Oct 2023 15:31:12 -0400 Subject: [PATCH 11/22] #330 whoops, keep FISH-7665 fix --- tasks/payara.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tasks/payara.yml b/tasks/payara.yml index a2d605e7..00081e52 100644 --- a/tasks/payara.yml +++ b/tasks/payara.yml @@ -133,6 +133,15 @@ mode: '0750' state: directory +- name: workaround for FISH-7665 + lineinfile: + path: '{{ payara_dir }}/glassfish/config/asenv.conf' + line: '{{ item }}' + with_items: + - 'export JAVA_TOOL_OPTIONS' + - 'JAVA_TOOL_OPTIONS="-Djdk.util.zip.disableZip64ExtraFieldValidation=true --add-opens=java.base/java.io=ALL-UNNAMED"' + when: dataverse.payara.zipurl is match(".*payara-6.*") + - name: so installations can still install payara5 if they want template: src: payara6.service.j2 From 83c70992dd20380facd61b074ce6cfc1773a9f87 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 25 Oct 2023 15:32:24 -0400 Subject: [PATCH 12/22] accomodate oliver's new FS checks during warfile packing and installation (#331) * #330 accomodate oliver's new FS checks during warfile packing and installation * Update tasks/dataverse-build.yml write mvn.out to correct location Co-authored-by: Philip Durbin * Update tasks/dataverse-build.yml Co-authored-by: Philip Durbin * #330 whoops, keep FISH-7665 fix --------- Co-authored-by: Don Sizemore Co-authored-by: Philip Durbin --- defaults/main.yml | 4 ++-- tasks/dataverse-branch.yml | 12 ++++++++++++ tasks/dataverse-build.yml | 12 ++++++++++-- tasks/dataverse-install.yml | 8 ++++++++ tasks/payara.yml | 9 +++------ tasks/payara_service_account.yml | 11 +++++++++++ tests/group_vars/jenkins.yml | 2 +- tests/group_vars/vagrant.yml | 4 ++-- 8 files changed, 49 insertions(+), 13 deletions(-) create mode 100644 tasks/payara_service_account.yml diff --git a/defaults/main.yml b/defaults/main.yml index 368984b3..646b4cd3 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -244,14 +244,14 @@ dataverse: service_email: noreply@dataverse.yourinstitution.edu smtp: localhost # or the FQDN of your organization's SMTP relay solr: - download_url: https://dlcdn.apache.org/solr/solr/9.3.0/solr-9.3.0.tgz + download_url: https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz group: solr root: /usr/local/solr user: solr version: 9.3.0 listen: 127.0.0.1 checksum: sha256:2a924a776d7c06dea561f12bdb17291dbaecc9e9a06f363d4599acb7fb7bfa71 - srcdir: /tmp/dataverse + srcdir: /home/dataverse/dataverse thumbnails: false unittests: enabled: false diff --git a/tasks/dataverse-branch.yml b/tasks/dataverse-branch.yml index 88251a88..bb83e4ed 100644 --- a/tasks/dataverse-branch.yml +++ b/tasks/dataverse-branch.yml @@ -4,9 +4,21 @@ debug: msg: '##### CLONE BRANCH #####' +- name: ensure service account and group exist + ansible.builtin.import_tasks: payara_service_account.yml + +- name: ensure target dir exists + ansible.builtin.file: + path: '{{ dataverse.srcdir }}' + state: directory + owner: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.user }}' + - name: clone specified branch git: repo: "{{ dataverse_repo }}" dest: "{{ dataverse.srcdir }}" version: "{{ dataverse_branch }}" when: dataverse_branch != "release" + become: true + become_user: '{{ dataverse.payara.user }}' diff --git a/tasks/dataverse-build.yml b/tasks/dataverse-build.yml index c6872b97..4c332b46 100644 --- a/tasks/dataverse-build.yml +++ b/tasks/dataverse-build.yml @@ -32,13 +32,17 @@ chdir: "{{ dataverse.srcdir }}" when: dataverse.unittests.enabled == false and maven.version == 'default' + become: true + become_user: '{{ dataverse.payara.user }}' -- name: build warfile without tests and custom maven. tail /tmp/dataverse/mvn.out for gory details. +- name: build warfile without tests and custom maven. tail /home/dataverse/dataverse/mvn.out for gory details. shell: "source /etc/profile.d/maven.sh && mvn -T 2C clean package -Dmaven.test.skip=true > mvn.out" args: chdir: "{{ dataverse.srcdir }}" when: dataverse.unittests.enabled == false and maven.version != 'default' + become: true + become_user: '{{ dataverse.payara.user }}' - name: build warfile with specified tests. tail /tmp/dataverse/mvn.out for gory details. shell: "mvn {{ dataverse.unittests.argument }} {{ jacoco_arg }} -T 2C clean package > mvn.out" @@ -46,13 +50,17 @@ chdir: "{{ dataverse.srcdir }}" when: dataverse.unittests.enabled == true and maven.version == 'default' + become: true + become_user: '{{ dataverse.payara.user }}' -- name: build warfile with specified tests using custom maven. tail /tmp/dataverse/mvn.out for gory details. +- name: build warfile with specified tests using custom maven. tail /home/dataverse/dataverse/mvn.out for gory details. shell: "source /etc/profile.d/maven.sh && mvn {{ dataverse.unittests.argument }} {{ jacoco_arg }} -T 2C clean package > mvn.out" args: chdir: "{{ dataverse.srcdir }}" when: dataverse.unittests.enabled == true and maven.version != 'default' + become: true + become_user: '{{ dataverse.payara.user }}' - name: copy warfile over release copy: diff --git a/tasks/dataverse-install.yml b/tasks/dataverse-install.yml index 7ffc5ca8..110b72d4 100644 --- a/tasks/dataverse-install.yml +++ b/tasks/dataverse-install.yml @@ -10,6 +10,14 @@ debug: msg: '##### DATAVERSE INSTALLER #####' +# this is necessary for the installer. dataverse-storage.yml does the real work later. +- name: filesdir must pre-exist + ansible.builtin.file: + path: '{{ dataverse.filesdirs[0].path }}' + state: directory + owner: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.group }}' + - ansible.builtin.import_tasks: dataverse-build.yml when: dataverse_branch != "release" diff --git a/tasks/payara.yml b/tasks/payara.yml index 65dae6bf..00081e52 100644 --- a/tasks/payara.yml +++ b/tasks/payara.yml @@ -35,11 +35,8 @@ when: payara_zip_download.changed notify: enable and restart payara -- name: create payara service group - group: name={{ dataverse.payara.group }} state=present - -- name: create payara service account - user: name={{ dataverse.payara.user }} group={{ dataverse.payara.group }} +- name: ensure service account and group exist + ansible.builtin.import_tasks: payara_service_account.yml - name: payara service account should own {{ dataverse.payara.domain }} file: path={{ payara_dir }}/glassfish/domains/{{ dataverse.payara.domain }} owner={{ dataverse.payara.user }} @@ -145,7 +142,7 @@ - 'JAVA_TOOL_OPTIONS="-Djdk.util.zip.disableZip64ExtraFieldValidation=true --add-opens=java.base/java.io=ALL-UNNAMED"' when: dataverse.payara.zipurl is match(".*payara-6.*") -- name: workaround for FISH-7665 +- name: so installations can still install payara5 if they want template: src: payara6.service.j2 dest: /usr/lib/systemd/system/payara.service diff --git a/tasks/payara_service_account.yml b/tasks/payara_service_account.yml new file mode 100644 index 00000000..4b05d613 --- /dev/null +++ b/tasks/payara_service_account.yml @@ -0,0 +1,11 @@ +--- + +- name: ensure payara service group exists + ansible.builtin.group: + name: '{{ dataverse.payara.group }}' + state: present + +- name: ensure payara service account exists + ansible.builtin.user: + name: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.group }}' diff --git a/tests/group_vars/jenkins.yml b/tests/group_vars/jenkins.yml index b0295d72..263536fc 100644 --- a/tests/group_vars/jenkins.yml +++ b/tests/group_vars/jenkins.yml @@ -242,7 +242,7 @@ dataverse: version: 9.3.0 listen: 127.0.0.1 checksum: sha256:2a924a776d7c06dea561f12bdb17291dbaecc9e9a06f363d4599acb7fb7bfa71 - srcdir: /tmp/dataverse + srcdir: /home/dataverse/dataverse thumbnails: false unittests: enabled: true diff --git a/tests/group_vars/vagrant.yml b/tests/group_vars/vagrant.yml index 95bf672e..77727bdd 100644 --- a/tests/group_vars/vagrant.yml +++ b/tests/group_vars/vagrant.yml @@ -240,14 +240,14 @@ dataverse: service_email: noreply@dataverse.yourinstitution.edu smtp: localhost solr: - download_url: https://dlcdn.apache.org/solr/solr/9.3.0/solr-9.3.0.tgz + download_url: https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz group: solr root: /usr/local/solr user: solr version: 9.3.0 listen: 127.0.0.1 checksum: sha256:2a924a776d7c06dea561f12bdb17291dbaecc9e9a06f363d4599acb7fb7bfa71 - srcdir: /tmp/dataverse + srcdir: /home/dataverse/dataverse thumbnails: false unittests: enabled: true From 44320f3ae77c74762c77e2800e9a01a0c31b9657 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 25 Oct 2023 16:22:08 -0400 Subject: [PATCH 13/22] correct for new Jenkins build location --- ec2/ec2-create-instance.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ec2/ec2-create-instance.sh b/ec2/ec2-create-instance.sh index abd68827..c160fb73 100755 --- a/ec2/ec2-create-instance.sh +++ b/ec2/ec2-create-instance.sh @@ -251,16 +251,16 @@ if [ ! -z "$LOCAL_LOG_PATH" ]; then # 1 logdir should exist mkdir -p $LOCAL_LOG_PATH # 2 grab logs for local processing in jenkins - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/site $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/surefire-reports $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/site $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/surefire-reports $LOCAL_LOG_PATH/ # 3 grab mvn.out - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/mvn.out $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/mvn.out $LOCAL_LOG_PATH/ # 4 jacoco - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/coverage-it $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/coverage-reports/jacoco-unit.exec $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/jacoco_merged.exec $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/classes $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/src $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/coverage-it $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/coverage-reports/jacoco-unit.exec $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/jacoco_merged.exec $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/classes $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/src $LOCAL_LOG_PATH/ # 5 server.logs rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/usr/local/payara*/glassfish/domains/domain1/logs/server.log* $LOCAL_LOG_PATH/ # 6 query_count.out From 39066f86f9d64bfd8bf1892686d3183d7f51158f Mon Sep 17 00:00:00 2001 From: pallinger Date: Thu, 26 Oct 2023 09:31:44 +0200 Subject: [PATCH 14/22] =?UTF-8?q?introduce=20dataverse=5Fmisc=5Ffiles=5Fdi?= =?UTF-8?q?r=20variable=20for=20/opt/dv=20and=20create=20it=E2=80=A6=20(#3?= =?UTF-8?q?23)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * introduce dataverse_misc_files_dir variable for /opt/dv and create it during prereq, else language and handle config will fail later * duplicate group_vars to tests YML * add dataverse_misc_files_dir variable to test variable files --------- Co-authored-by: Pallinger Péter Co-authored-by: Don Sizemore --- defaults/main.yml | 6 ++++-- tasks/dataverse-prereqs.yml | 7 +++++++ tests/group_vars/java17.yml | 2 ++ tests/group_vars/jenkins.yml | 6 ++++-- tests/group_vars/memorytests.yml | 6 ++++-- tests/group_vars/solr9.yml | 2 ++ tests/group_vars/vagrant.yml | 6 ++++-- 7 files changed, 27 insertions(+), 8 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 646b4cd3..148972b0 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -41,6 +41,8 @@ letsencrypt: email: user: certbot +dataverse_misc_files_dir: '/opt/dv' + dataverse: adminpass: admin1 allow_signups: true @@ -78,7 +80,7 @@ dataverse: language_packs: source: https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs.git version: develop - lang_directory: /opt/dv/lang + lang_directory: "{{ dataverse_misc_files_dir }}/lang" licenses: enabled: false user: dataverseAdmin @@ -165,7 +167,7 @@ dataverse: handlenet: independenthandleservice: 'false' # handleauthhandle: YOUR:HANDLE/USERNAME - admcredfile: /opt/dv/admpriv.bin + admcredfile: "{{ dataverse_misc_files_dir }}/admpriv.bin" admcredfile_source: files/handlenet/admcredfile # admprivphrase: # index: 300 diff --git a/tasks/dataverse-prereqs.yml b/tasks/dataverse-prereqs.yml index b7bc65bb..020da44a 100644 --- a/tasks/dataverse-prereqs.yml +++ b/tasks/dataverse-prereqs.yml @@ -85,3 +85,10 @@ name: curl when: - ansible_os_family == "Debian" + +- name: create dataverse misc files directory for language and handle and other similar auxilliary files + file: + path: "{{ dataverse_misc_files_dir }}" + state: directory + owner: "{{ dataverse.payara.user }}" + group: "{{ dataverse.payara.group }}" diff --git a/tests/group_vars/java17.yml b/tests/group_vars/java17.yml index 1369af31..295b72ef 100644 --- a/tests/group_vars/java17.yml +++ b/tests/group_vars/java17.yml @@ -32,6 +32,8 @@ letsencrypt: email: user: certbot +dataverse_misc_files_dir: '/opt/dv' + dataverse: adminpass: admin1 allow_signups: true diff --git a/tests/group_vars/jenkins.yml b/tests/group_vars/jenkins.yml index 263536fc..7a7406a9 100644 --- a/tests/group_vars/jenkins.yml +++ b/tests/group_vars/jenkins.yml @@ -32,6 +32,8 @@ letsencrypt: email: user: certbot +dataverse_misc_files_dir: '/opt/dv' + dataverse: adminpass: admin1 allow_signups: true @@ -99,7 +101,7 @@ dataverse: handlenet: independenthandleservice: 'false' # handleauthhandle: YOUR:HANDLE/USERNAME - admcredfile: /opt/dv/admpriv.bin + admcredfile: "{{ dataverse_misc_files_dir }}/admpriv.bin" admcredfile_source: files/handlenet/admcredfile # admprivphrase: # index: 300 @@ -161,7 +163,7 @@ dataverse: language_packs: source: https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs.git version: develop - lang_directory: /opt/dv/lang + lang_directory: "{{ dataverse_misc_files_dir }}/lang" licenses: enabled: false user: dataverseAdmin diff --git a/tests/group_vars/memorytests.yml b/tests/group_vars/memorytests.yml index 2b509cc2..7e205235 100644 --- a/tests/group_vars/memorytests.yml +++ b/tests/group_vars/memorytests.yml @@ -37,6 +37,8 @@ letsencrypt: email: user: certbot +dataverse_misc_files_dir: '/opt/dv' + dataverse: adminpass: admin1 allow_signups: true @@ -102,7 +104,7 @@ dataverse: handlenet: independenthandleservice: 'false' # handleauthhandle: YOUR:HANDLE/USERNAME - admcredfile: /opt/dv/admpriv.bin + admcredfile: "{{ dataverse_misc_files_dir }}/admpriv.bin" admcredfile_source: files/handlenet/admcredfile # admprivphrase: # index: 300 @@ -164,7 +166,7 @@ dataverse: language_packs: source: https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs.git version: develop - lang_directory: /opt/dv/lang + lang_directory: "{{ dataverse_misc_files_dir }}/lang" licenses: enabled: false user: dataverseAdmin diff --git a/tests/group_vars/solr9.yml b/tests/group_vars/solr9.yml index 65116e0e..0eec5f33 100644 --- a/tests/group_vars/solr9.yml +++ b/tests/group_vars/solr9.yml @@ -32,6 +32,8 @@ letsencrypt: email: user: certbot +dataverse_misc_files_dir: '/opt/dv' + dataverse: adminpass: admin1 allow_signups: true diff --git a/tests/group_vars/vagrant.yml b/tests/group_vars/vagrant.yml index 77727bdd..98f8b18f 100644 --- a/tests/group_vars/vagrant.yml +++ b/tests/group_vars/vagrant.yml @@ -37,6 +37,8 @@ letsencrypt: email: user: certbot +dataverse_misc_files_dir: '/opt/dv' + dataverse: adminpass: admin1 allow_signups: true @@ -104,7 +106,7 @@ dataverse: handlenet: independenthandleservice: 'false' # handleauthhandle: YOUR:HANDLE/USERNAME - admcredfile: /opt/dv/admpriv.bin + admcredfile: "{{ dataverse_misc_files_dir }}/admpriv.bin" admcredfile_source: files/handlenet/admcredfile # admprivphrase: # index: 300 @@ -166,7 +168,7 @@ dataverse: language_packs: source: https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs.git version: develop - lang_directory: /opt/dv/lang + lang_directory: "{{ dataverse_misc_files_dir }}/lang" licenses: enabled: false user: dataverseAdmin From 2a001dbde1a332174505db99965c15499d0c0f04 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Thu, 26 Oct 2023 06:14:04 -0400 Subject: [PATCH 15/22] #330 oliver suggests /opt/dataverse since containers use /opt/payara --- defaults/main.yml | 2 +- ec2/ec2-create-instance.sh | 16 ++++++++-------- tests/group_vars/jenkins.yml | 2 +- tests/group_vars/vagrant.yml | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 148972b0..3af2593e 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -253,7 +253,7 @@ dataverse: version: 9.3.0 listen: 127.0.0.1 checksum: sha256:2a924a776d7c06dea561f12bdb17291dbaecc9e9a06f363d4599acb7fb7bfa71 - srcdir: /home/dataverse/dataverse + srcdir: /opt/dataverse/dataverse thumbnails: false unittests: enabled: false diff --git a/ec2/ec2-create-instance.sh b/ec2/ec2-create-instance.sh index c160fb73..3e0ac36a 100755 --- a/ec2/ec2-create-instance.sh +++ b/ec2/ec2-create-instance.sh @@ -251,16 +251,16 @@ if [ ! -z "$LOCAL_LOG_PATH" ]; then # 1 logdir should exist mkdir -p $LOCAL_LOG_PATH # 2 grab logs for local processing in jenkins - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/site $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/surefire-reports $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/site $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/surefire-reports $LOCAL_LOG_PATH/ # 3 grab mvn.out - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/mvn.out $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/mvn.out $LOCAL_LOG_PATH/ # 4 jacoco - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/coverage-it $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/coverage-reports/jacoco-unit.exec $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/jacoco_merged.exec $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/target/classes $LOCAL_LOG_PATH/ - rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/home/dataverse/dataverse/src $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/coverage-it $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/coverage-reports/jacoco-unit.exec $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/jacoco_merged.exec $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/classes $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/src $LOCAL_LOG_PATH/ # 5 server.logs rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/usr/local/payara*/glassfish/domains/domain1/logs/server.log* $LOCAL_LOG_PATH/ # 6 query_count.out diff --git a/tests/group_vars/jenkins.yml b/tests/group_vars/jenkins.yml index 7a7406a9..38b6d2af 100644 --- a/tests/group_vars/jenkins.yml +++ b/tests/group_vars/jenkins.yml @@ -244,7 +244,7 @@ dataverse: version: 9.3.0 listen: 127.0.0.1 checksum: sha256:2a924a776d7c06dea561f12bdb17291dbaecc9e9a06f363d4599acb7fb7bfa71 - srcdir: /home/dataverse/dataverse + srcdir: /opt/dataverse/dataverse thumbnails: false unittests: enabled: true diff --git a/tests/group_vars/vagrant.yml b/tests/group_vars/vagrant.yml index 98f8b18f..a7683afa 100644 --- a/tests/group_vars/vagrant.yml +++ b/tests/group_vars/vagrant.yml @@ -249,7 +249,7 @@ dataverse: version: 9.3.0 listen: 127.0.0.1 checksum: sha256:2a924a776d7c06dea561f12bdb17291dbaecc9e9a06f363d4599acb7fb7bfa71 - srcdir: /home/dataverse/dataverse + srcdir: /opt/dataverse/dataverse thumbnails: false unittests: enabled: true From a3fee840ffbb8d7c98f111e5580e34e5840749ca Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Thu, 26 Oct 2023 06:33:40 -0400 Subject: [PATCH 16/22] Payara service account must exist to create misc files dir --- tasks/dataverse-prereqs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tasks/dataverse-prereqs.yml b/tasks/dataverse-prereqs.yml index 020da44a..75e23647 100644 --- a/tasks/dataverse-prereqs.yml +++ b/tasks/dataverse-prereqs.yml @@ -86,6 +86,9 @@ when: - ansible_os_family == "Debian" +- name: Payara service account must exist + import_tasks: payara_service_account.yml + - name: create dataverse misc files directory for language and handle and other similar auxilliary files file: path: "{{ dataverse_misc_files_dir }}" From b50c2e97b39e7c5a0daa25889e88179e7c3bca41 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Tue, 31 Oct 2023 10:49:55 -0400 Subject: [PATCH 17/22] #333 can't template from another variable in same group_vars file. (#335) Co-authored-by: Don Sizemore --- defaults/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 3af2593e..51bc1de3 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -304,7 +304,8 @@ grafana: java: version: 17 - home: /usr/lib/jvm/java-{{ java.version}} + # this breaks templating. + #home: /usr/lib/jvm/java-{{ java.version}} localstack: enabled: false From 51467aaee25f3f8e9a0a55f7dc92393cfbcdfdab Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Tue, 31 Oct 2023 14:45:46 -0400 Subject: [PATCH 18/22] #332 bump Jenkins volume size to 20GB for Docker (#334) Co-authored-by: Don Sizemore --- ec2/ec2-create-instance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ec2/ec2-create-instance.sh b/ec2/ec2-create-instance.sh index 3e0ac36a..f9f40bd1 100755 --- a/ec2/ec2-create-instance.sh +++ b/ec2/ec2-create-instance.sh @@ -208,7 +208,7 @@ fi echo "Creating EC2 instance" # TODO: Add some error checking for "ec2 run-instances". -INSTANCE_ID=$(aws $PROFILE ec2 run-instances --image-id $AMI_ID --security-groups $AWS_SG $TAGARG --count 1 --instance-type $SIZE --key-name $KEY_NAME --query 'Instances[0].InstanceId' --block-device-mappings '[ { "DeviceName": "/dev/sda1", "Ebs": { "DeleteOnTermination": true } } ]' | tr -d \") +INSTANCE_ID=$(aws $PROFILE ec2 run-instances --image-id $AMI_ID --security-groups $AWS_SG $TAGARG --count 1 --instance-type $SIZE --key-name $KEY_NAME --query 'Instances[0].InstanceId' --block-device-mappings '[ { "DeviceName": "/dev/sda1", "Ebs": { "DeleteOnTermination": true, "VolumeSize": 20 } } ]' | tr -d \") echo "Instance ID: "$INSTANCE_ID DESTROY_CMD="aws $PROFILE ec2 terminate-instances --instance-ids $INSTANCE_ID" From e1bb5f2bacd906d3cc23a780c47b33fc0307a10f Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Mon, 20 Nov 2023 10:03:17 -0500 Subject: [PATCH 19/22] disablerepo=testing to deal with Rocky 8.9 churn --- ec2/ec2-create-instance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ec2/ec2-create-instance.sh b/ec2/ec2-create-instance.sh index f9f40bd1..2d079745 100755 --- a/ec2/ec2-create-instance.sh +++ b/ec2/ec2-create-instance.sh @@ -236,7 +236,7 @@ fi # TODO: Add some error checking for this ssh command. ssh -T -i $PEM_FILE -o 'StrictHostKeyChecking no' -o 'UserKnownHostsFile=/dev/null' -o 'ConnectTimeout=300' $USER_AT_HOST < Date: Thu, 7 Dec 2023 07:14:08 -0500 Subject: [PATCH 20/22] support Dockerized s3 tests (#337) * #327 initial jenkins.yml changes for phil's review * #327 only one storage_driver_id, remains file * #327 remove config deletion from dataverse-storage.yml * #327 stub out docker/localstack * #327 tweak lookup loop * #327 add localstack_create.yml * #327 add localstack_create_bucket.yml * #327 add CORS config * #327 add cors.json for S3 testing * #327 remove errant localstack variable name prefix * #327 correct docker repo installation * #327 remove second errant localstack variable name prefix * #327 ansible_user is now ansible_ssh_user * #327 infer docker user * #327 do we even need raw/endraw * #327 add necessary localstack group_vars * #327 add missing single ticks * #327 container should be container_name * #327 switch to docker compose v2 * #327 disable most of S3 configuration in favor of supporting multiple S3-based datastores * #327 uncomment S3 in favor of s3.enabled false in group_vars * #327 let's pass AWS keys to localstack * #327 too many items spoil the broth * #327 include_tasks with loop * #327 loop with block * #327 attempt include_tasks with_items * #327 now it IS sending item properly * #327 it sees item now * #327 remove errant item prefix * #327 add AWS binary where needed * #327 just use OS-included binary for now * #327 awscli package name may be specific to RedHat systems * #327 correct Red Hat check * #327 place AWS config in dataverse user homedir * #327 use dataverse homedir for localstack compose * #327 don't kill the container * #327 perform s3 commands as service account * #327 add dataverse user to docker group * #327 localstack is trying to nuke its own data_dir * #327 Phil's localstack config is much simpler * #327 correct container launch user * #327 localstack claims it doesn't need to be background-ed * #327 container stays up when not background-ed * #327 correct conditional evaluation * #327 custom_endpoint_region should just be region * #327 label should be item.label * #327 loop_control * #327 use loop_control in outer loop * #327 item should be outer * #327 correct key/value definitions * #327 replace key ticks, whoops * #327 let's define our own dict * #327 debug outer list * #327 quotes and ticks, quotes and ticks * #327 debug jvm-options * #327 let's set ingestsizelimit to 2000000000 * #327 correct key/value syntax * #327 jvm-options should be outer * #327 jvm-options must be jvm_options * #327 let's debug bucketlist again * #327 Ansible handles bucket and CORS * #327 swap inner and outer loop_vars * #327 do we need to escape custom_endpoint_url? * #327 correct unescaped custom endpoint url * #327 I was using ticks instead of quotes * #327 we bothered to escape the var, let's use it * #327 forgot to escape a dash * #327 don't explicitly read in bucket_options when populating jvm_options * #327 correct cors.json syntax * #327 whoops, cors settings requires custom_endpoint_escaped * #327 let's do MinIO as well, all in * #327 stub out MinIO settings * #327 replace original S3 group_vars, use localstack-specific * #327 untested stab at minio task * #327 correct spacing, add some DV-specific jvm-options * #327 correct group_vars syntax * #327 correct minio root setting tests * #327 docker-compose.yml should be minio_compose.yml * #327 correct minio template src filename * #327 correct variable typo in minio_compose.j2 * #327 correct variable typos in minio template * #327 one final typo in MinIO jinja template * #327 Ansible Docker module requires python-docker * #327 make that python3-docker * #327 Ansible needs docker-compose python package * #327 what a difference a colon makes * #327 now pip errors. sigh. * #327 specify minio_compose.yml * #327 project_src is required, files conflicts with definition instead * #327 don't use special docker network unless we have to * #327 launch minio as dataverse service account * #327 do we need minio docker user/group? * #327 variables must exist, length 0 * #327 keep variables, comment out in case * #327 adjust commented spacing * #327 adjust template indentation * #327 if blocks causing trouble * #327 set MinIO jvm-options * #327 quote MinIO CEU value * #327 escape custom-endpoint-url colons * #327 underscores only * #327 update minio variables * #327 localstack1, mybucket * #327 localstack label, caps count * #327 LocalStack: match Phil's options exactly * #327 touch up variables post-Zoom * #327 run tests and reports as Dataverse service account * #327 disable global s3 creds/config as a test * #327 correct localstack variables * #327 correct identifier group_var syntax * #327 use ibiblio mirror for now, minio wants to match AWS credentials * #327 correct minio secret-access-key * #327 add localstack/minio group_vars to defaults/vagrant yml * #327 duplicate Docker CIDR to default/vagrant * #327 hopefully final touch-ups * #327 keep aws_config.yml for now * #327 correct minio secret_key --------- Co-authored-by: Don Sizemore --- defaults/main.yml | 58 +++++++++++++++++-- files/cors.json | 10 ++++ tasks/aws_cli.yml | 14 +++++ tasks/aws_config.yml | 25 +++++++++ tasks/dataverse-api-testsuite.yml | 8 +++ tasks/dataverse-storage.yml | 66 ++-------------------- tasks/docker.yml | 71 ++++++++++++++++++++++++ tasks/localstack.yml | 62 ++++++--------------- tasks/localstack_create.yml | 38 ++++++------- tasks/localstack_create_bucket.yml | 59 ++++++++++++++++++++ tasks/main.yml | 5 ++ tasks/minio.yml | 86 +++++++++++++++++++++++++++++ tasks/minio_assert.yml | 16 ++++++ tasks/minio_jvm_options.yml | 32 +++++++++++ tasks/s3.yml | 19 ++----- templates/localstack-compose.yml.j2 | 16 +++--- templates/minio_compose.j2 | 42 ++++++++++++++ tests/group_vars/jenkins.yml | 64 ++++++++++++++++++--- tests/group_vars/vagrant.yml | 58 +++++++++++++++++++ 19 files changed, 585 insertions(+), 164 deletions(-) create mode 100644 files/cors.json create mode 100644 tasks/aws_cli.yml create mode 100644 tasks/aws_config.yml create mode 100644 tasks/docker.yml create mode 100644 tasks/localstack_create_bucket.yml create mode 100644 tasks/minio.yml create mode 100644 tasks/minio_assert.yml create mode 100644 tasks/minio_jvm_options.yml create mode 100644 templates/minio_compose.j2 diff --git a/defaults/main.yml b/defaults/main.yml index 51bc1de3..237a5037 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -298,6 +298,10 @@ db: plain: 'no' use_rds: false +docker: + cidr: 10.0.0.99/24 + repo: https://download.docker.com/linux/centos/docker-ce.repo + grafana: grafana_user: 'admin' grafana_password: 'adm1n' @@ -309,19 +313,61 @@ java: localstack: enabled: false - docker: - cidr: - repo: https://download.docker.com/linux/centos/docker-ce.repo - container: 's3-test' + container_name: dev_localstack # set to /tmp/localstack/data to enable persistence - data_dir: + data_dir: /tmp/localstack/data debug: true - hostname_external: + hostname_external: localstack + port: 4566 web_ui: 8888 + buckets: + - label: LocalStack + id: localstack1 + bucket_name: mybucket + enabled: false + access_key: 4cc355_k3y + secret_access_key: s3cr3t_4cc355_k3y + # for non-amazon services. + custom_endpoint_url: "http://localhost:4566" + # note: direct upload and download require CORS on the bucket + download_redirect: true + upload_redirect: true + files_type: s3 + ingestsizelimit: 2000000000 + # for localstack this must be true + path_style_access: true + region: us-east-2 + url_expiration_minutes: 60 + payload_signing: false + chunked_encoding: true maven: version: 3.8.8 +minio: + enabled: false + docker: + version: "latest" + service_name: "minio" + network: + name: "minio" + external: false + project_location: "/home/dataverse/minio" + timezone: "America/New_York" + user: dataverse + group: dataverse + server_port: 9000 + console_port: 9001 + root_user: "4cc355_k3y" + root_password: "s3cr3t_4cc355_k3y" + default_bucket: "mybucket" + label: MinIO + custom_endpoint_url: "http://localhost:9000" + custom_endpoint_region: us-east-1 + path-style-access: true + access_key: 4cc355_k3y + secret_key: s3cr3t_4cc355_k3y + munin: install: false admin: diff --git a/files/cors.json b/files/cors.json new file mode 100644 index 00000000..be6f35d5 --- /dev/null +++ b/files/cors.json @@ -0,0 +1,10 @@ +{ + "CORSRules": [ + { + "AllowedOrigins": ["*"], + "AllowedHeaders": ["*"], + "AllowedMethods": ["PUT", "GET"], + "ExposeHeaders": ["ETag", "Accept-Ranges", "Content-Encoding", "Content-Range"] + } + ] +} diff --git a/tasks/aws_cli.yml b/tasks/aws_cli.yml new file mode 100644 index 00000000..b5d840fe --- /dev/null +++ b/tasks/aws_cli.yml @@ -0,0 +1,14 @@ +--- + +#- name: we need awscli on 8 +# pip: +# name: awscli +# executable: pip3 +# when: (ansible_os_family == "RedHat") and +# (ansible_distribution_major_version) == "8" + +- name: just use OS-included binary for now + ansible.builtin.package: + name: awscli + state: latest + when: ansible_os_family == "RedHat" diff --git a/tasks/aws_config.yml b/tasks/aws_config.yml new file mode 100644 index 00000000..c26c976e --- /dev/null +++ b/tasks/aws_config.yml @@ -0,0 +1,25 @@ +--- + +- name: ensure dot dir exists + file: + path: '~{{ dataverse.payara.user }}/.aws' + state: directory + owner: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.group }}' + mode: '0700' + +- name: place aws credentials + template: + src: aws_credentials.j2 + dest: '~{{ dataverse.payara.user }}/.aws/credentials' + owner: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.group }}' + mode: '0600' + +- name: place aws config + template: + src: aws_config.j2 + dest: '~{{ dataverse.payara.user }}/.aws/config' + owner: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.group }}' + mode: '0600' diff --git a/tasks/dataverse-api-testsuite.yml b/tasks/dataverse-api-testsuite.yml index 5780e9b8..1b8a105d 100644 --- a/tasks/dataverse-api-testsuite.yml +++ b/tasks/dataverse-api-testsuite.yml @@ -89,6 +89,8 @@ DV_APIKEY: '{{ dataverse.usermgmtkey }}' DV_API_KEY: '{{ dataverse.usermgmtkey }}' when: maven.version == 'default' + become: true + become_user: '{{ dataverse.payara.user }}' - name: run test suite with custom maven shell: 'source /etc/profile.d/maven.sh && mvn test -Dtest={{ integration_test_list }} -Ddataverse.test.baseurl=http://localhost:8080' @@ -99,18 +101,24 @@ DV_APIKEY: '{{ dataverse.usermgmtkey }}' DV_API_KEY: '{{ dataverse.usermgmtkey }}' when: maven.version != 'default' + become: true + become_user: '{{ dataverse.payara.user }}' - name: generate HTML reports with default maven shell: 'mvn surefire-report:report' args: chdir: '{{ dataverse.srcdir }}' when: maven.version == 'default' + become: true + become_user: '{{ dataverse.payara.user }}' - name: generate HTML reports with custom maven shell: 'source /etc/profile.d/maven.sh && mvn surefire-report:report' args: chdir: '{{ dataverse.srcdir }}' when: maven.version != 'default' + become: true + become_user: '{{ dataverse.payara.user }}' - ansible.builtin.import_tasks: postgresql_perl_party.yml when: diff --git a/tasks/dataverse-storage.yml b/tasks/dataverse-storage.yml index 7a268c04..0dbe6835 100644 --- a/tasks/dataverse-storage.yml +++ b/tasks/dataverse-storage.yml @@ -12,17 +12,10 @@ register: dataverse_filestores_storage_driver_id changed_when: false -# default storage-driver-id is a group_var now -#- name: calculate default storage-driver-id -# set_fact: -# default_storage_driver_id: "{{ ((dataverse_filestores_storage_driver_id.stdout | trim) == '') | -# ternary(dataverse.filesdirs[0].label,dataverse_filestores_storage_driver_id.stdout) }}" -# to_delete_config_lines: [] - -# don't delete other storage drivers, as dataverse supports multiple now. -#- name: set to_delete_config_lines -# set_fact: -# to_delete_config_lines: [] +- name: calculate default storage-driver-id + set_fact: + default_storage_driver_id: "{{ ((dataverse_filestores_storage_driver_id.stdout | trim) == '') | + ternary(dataverse.filesdirs[0].label,dataverse_filestores_storage_driver_id.stdout) }}" - name: register default storage-driver-id set_fact: @@ -45,13 +38,6 @@ - default_dataverse_filestores_type.stdout != '' - default_dataverse_filestores_type.stdout != 'file' -# don't delete storage drivers, as dataverse supports multiple now. -#- name: calculate whether to delete storage-driver-id -# set_fact: -# to_delete_config_lines: "{{ (default_storage_driver_id != dataverse.filesdirs[0].label) | -# ternary(to_delete_config_lines+[ '-Ddataverse.files.storage-driver-id=' + default_storage_driver_id ], -# to_delete_config_lines ) }}" - - name: remove previous storage-driver-id if necessary command: "{{ payara_dir }}/bin/asadmin delete-jvm-options \"-Ddataverse.files.storage-driver-id={{ dataverse.filesdirs[0].label }}\"" when: default_storage_driver_id != dataverse_filestores_storage_driver_id.stdout @@ -59,21 +45,12 @@ - name: set default storage-driver-id if not set or changed command: "{{ payara_dir }}/bin/asadmin create-jvm-options \"-Ddataverse.files.storage-driver-id={{ dataverse.default.storage_id }}\"" when: default_storage_driver_id != dataverse_filestores_storage_driver_id.stdout - #when: dataverse_filestores_storage_driver_id.stdout != dataverse.filesdirs[0].label - name: get default dataverse storage directory shell: "{{ payara_dir }}/bin/asadmin list-jvm-options | grep dataverse.files.directory | sed 's/.*=//'" register: default_dataverse_filestores_directory changed_when: false -#- name: calculate whether to delete default dataverse storage directory -# set_fact: -# to_delete_config_lines: "{{ -# ( (default_dataverse_filestores_directory.stdout | trim) != '' and -# default_dataverse_filestores_directory.stdout != dataverse.filesdirs[0].path) | -# ternary(to_delete_config_lines+[ ('-Ddataverse.files.directory=' + (default_dataverse_filestores_directory.stdout)) ], -# to_delete_config_lines) }}" - - name: set default dataverse storage directory if not set or changed command: "{{ payara_dir }}/bin/asadmin create-jvm-options \"-Ddataverse.files.directory={{ dataverse.filesdirs[0].path }}\"" when: default_dataverse_filestores_directory.stdout != dataverse.filesdirs[0].path @@ -140,30 +117,6 @@ dataverse_filestores_labels_to_delete: "{{ dataverse_filestores_labels.stdout_lines | difference(dataverse_defined_filestores_labels_list) }}" dataverse_filestores_labels_to_create: "{{ dataverse_defined_filestores_labels_list | difference(dataverse_filestores_labels.stdout_lines) }}" -# don't delete other configs -#- name: calculate config lines to delete for filestores -# set_fact: -# to_delete_config_lines: "{{ to_delete_config_lines + -# [ '-Ddataverse.files.'+item.label+'.directory='+item.path ] }}" -# with_items: "{{ dataverse_filestores_to_delete }}" - -#- name: calculate config lines to delete for labels -# set_fact: -# to_delete_config_lines: "{{ to_delete_config_lines + -# [ '-Ddataverse.files.'+item+'.label='+item ] }}" -# with_items: "{{ dataverse_filestores_labels_to_delete }}" - -# don't delete filestores. dataverse supports multiple now. -#- name: calculate required changes to dataverse_filestore type entries -# set_fact: -# to_delete_config_lines: "{{ (dataverse_filestores_types_onserver[item]=='file' and -# dataverse_defined_filestores_labels_dict[item] is not defined) | -# ternary(to_delete_config_lines + -# [ '-Ddataverse.files.' + item + '.type=' + dataverse_filestores_types_onserver[item] ], -# to_delete_config_lines ) -# }}" -# with_items: "{{ dataverse_filestores_typelabels_onserver }}" - - name: calculate required changes to dataverse_filestore type entries set_fact: dataverse_filestores_type_to_create: "{{ (dataverse_filestores_types_onserver[item] is defined) | @@ -177,17 +130,6 @@ vars: dataverse_filestores_type_to_create: [] -#- name: delete storage config options -# command: '{{ payara_dir }}/bin/asadmin delete-jvm-options "{{ item }}"' -# with_items: "{{ to_delete_config_lines }}" - -#- name: delete all settings in domain.xml as may create double entries on restart -# lineinfile: -# path: '{{ payara_dir }}/payara/domains/{{ dataverse.payara.domain }}/config/domain.xml' -# regexp: '{{ item }}' -# state: absent -# with_items: "{{ to_delete_config_lines }}" - - name: payara should own dataverse.filesdirs file: path={{ item.path }} state=directory owner={{ dataverse.payara.user }} group={{ dataverse.payara.group }} diff --git a/tasks/docker.yml b/tasks/docker.yml new file mode 100644 index 00000000..cbe4c796 --- /dev/null +++ b/tasks/docker.yml @@ -0,0 +1,71 @@ +--- + +- name: install docker-ce repo + ansible.builtin.get_url: + url: '{{ docker.repo }}' + dest: /etc/yum.repos.d/docker-ce.repo + mode: '0644' + +- name: install docker-ce + ansible.builtin.dnf: + name: ['docker-ce', 'docker-compose-plugin'] + state: latest + +- name: Ansible docker module requires python-docker + ansible.builtin.package: + name: 'python3-docker' + state: latest + +- name: pip must update itself + ansible.builtin.pip: + name: pip + executable: pip3 + state: latest + +- name: and docker-compose python package + ansible.builtin.pip: + name: docker-compose + +- name: ensure /etc/docker exists + ansible.builtin.file: + path: /etc/docker + state: directory + owner: root + group: root + mode: 0755 + +- name: configure cidr range + ansible.builtin.template: + src: daemon.json.j2 + dest: /etc/docker/daemon.json + owner: root + group: root + mode: 0644 + when: docker.cidr is undefined + +- name: infer become_user + ansible.builtin.command: whoami + register: whoami_output + +- name: register docker_user + ansible.builtin.set_fact: + docker_user: "{{ whoami_output.stdout }}" + +- name: add ansible_ssh_user to docker group + ansible.builtin.user: + name: '{{ docker_user }}' + groups: docker + append: yes + +- name: add dataverse user to docker group + ansible.builtin.user: + name: '{{ dataverse.payara.user }}' + groups: docker + append: yes + +- name: reload systemd, enable docker, start + ansible.builtin.systemd: + name: docker + enabled: yes + daemon_reload: yes + state: started diff --git a/tasks/localstack.yml b/tasks/localstack.yml index 5306e105..01e81180 100644 --- a/tasks/localstack.yml +++ b/tasks/localstack.yml @@ -1,61 +1,35 @@ --- -- name: we need yum-utils - yum: - name: yum-utils - state: latest +- ansible.builtin.import_tasks: docker.yml -- name: to install the docker-ce repo - shell: 'yum-config-manager --add-repo {{ localstack.docker.repo }}' +- ansible.builtin.import_tasks: aws_cli.yml -- name: install docker-ce - yum: - name: ['docker-ce','docker-ce-cli','containerd.io','docker-compose','python-docker-py'] - -- name: ensure /etc/docker exists - file: - path: /etc/docker - state: directory - owner: root - group: root - mode: 0755 - -- name: configure cidr range - template: - src: daemon.json.j2 - dest: /etc/docker/daemon.json - owner: root - group: root - mode: 0644 - when: localstack.docker.cidr is undefined - -- name: add ansible_user to docker group - user: - name: '{{ ansible_user }}' - groups: docker - append: yes - -- name: reload systemd, enable docker, start - systemd: - name: docker - enabled: yes - daemon_reload: yes - state: started +- ansible.builtin.import_tasks: aws_config.yml # so we can run localstack task stand-alone - ansible.builtin.import_tasks: dataverse-fqdn.yml - name: is hostname_external set - set_fact: + ansible.builtin.set_fact: hostname_external: '{{ public_fqdn }}' when: localstack.hostname_external is undefined - name: place docker-compose.yml template - template: + ansible.builtin.template: src: localstack-compose.yml.j2 - dest: '{{ ansible_env.HOME }}/localstack-compose.yml' + dest: '~{{ dataverse.payara.user }}/localstack-compose.yml' + +- name: ensure DATA_DIR exists + ansible.builtin.file: + path: '{{ localstack.data_dir }}' + state: directory + owner: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.group }}' + mode: 0755 - name: start s3-test container - shell: 'docker-compose -f localstack-compose.yml up -d' + ansible.builtin.shell: 'docker compose -f localstack-compose.yml up -d' + become: true + become_user: '{{ dataverse.payara.user }}' args: - chdir: '{{ ansible_env.HOME }}' + chdir: '~{{ dataverse.payara.user }}' diff --git a/tasks/localstack_create.yml b/tasks/localstack_create.yml index a2421d18..a1dc600c 100644 --- a/tasks/localstack_create.yml +++ b/tasks/localstack_create.yml @@ -1,27 +1,21 @@ --- -- name: ensure dot dir exists - file: - path: '~/.aws' - state: directory - mode: '0700' - -- name: place aws credentials - template: - src: aws_credentials.j2 - dest: '~/.aws/credentials' - mode: '0600' +- name: flush handlers to restart Payara if needed + meta: flush_handlers -- name: place aws config - template: - src: aws_config.j2 - dest: '~/.aws/config' - mode: '0600' +- name: build list from json + ansible.builtin.set_fact: + s3_bucketlist: "{{ localstack.buckets }}" -- name: escape custom endpoint url colons - set_fact: "custom_endpoint_escaped={{ s3.custom_endpoint_url | regex_replace (':','\\:') }}" - when: s3.custom_endpoint_url | length > 0 +- name: debug list + ansible.builtin.debug: + msg: "{{ item }}" + loop: "{{ s3_bucketlist }}" -- name: create localstack bucket for testing - shell: 'aws --endpoint={{ custom_endpoint_escaped }} s3 mb s3://{{ s3.bucket_name }}' - when: s3.custom_endpoint_url | length > 0 +- name: create buckets and set options + block: + - name: create bucket and set options + ansible.builtin.include_tasks: localstack_create_bucket.yml + vars: + bucket_options: "{{ item }}" + with_items: "{{ s3_bucketlist }}" diff --git a/tasks/localstack_create_bucket.yml b/tasks/localstack_create_bucket.yml new file mode 100644 index 00000000..b2ed39ed --- /dev/null +++ b/tasks/localstack_create_bucket.yml @@ -0,0 +1,59 @@ +--- + +- name: escape custom endpoint url colons + ansible.builtin.set_fact: "custom_endpoint_escaped={{ bucket_options.custom_endpoint_url | regex_replace (':','\\:') }}" + when: bucket_options.custom_endpoint_url | length > 0 + +- name: create localstack bucket for testing + ansible.builtin.shell: "aws --endpoint={{ custom_endpoint_escaped }} s3 mb s3://{{ bucket_options.bucket_name }}" + become: yes + become_user: "{{ dataverse.payara.user }}" + when: bucket_options.custom_endpoint_url | length > 0 + +- name: place CORS.json + ansible.builtin.copy: + src: cors.json + dest: /tmp/cors.json + owner: root + group: root + mode: '0644' + +- name: set CORS on bucket when upload_redirect is true + ansible.builtin.shell: 'aws --endpoint={{ custom_endpoint_escaped }} s3api put-bucket-cors --bucket {{ bucket_options.bucket_name }} --cors-configuration file:///tmp/cors.json' + become: yes + become_user: "{{ dataverse.payara.user }}" + when: bucket_options.upload_redirect == true + +- name: debug + ansible.builtin.debug: + msg: '{{ bucket_options }}' + +- name: create jvm_options dictionary + set_fact: + jvm_options: "{{ jvm_options | default({}) | combine ({ inner.key : inner.value }) }}" + loop_control: + loop_var: inner + with_items: + - { 'key' : 'type' , 'value' : 's3' } + - { 'key' : 'id' , 'value' : '{{ bucket_options.id }}' } + - { 'key' : 'label' , 'value' : '{{ bucket_options.label }}' } + - { 'key' : 'bucket-name' , 'value' : '{{ bucket_options.bucket_name }}' } + - { 'key' : 'custom-endpoint-url' , 'value' : '{{ custom_endpoint_escaped }}' } + - { 'key' : 'custom-endpoint-region' , 'value' : '{{ bucket_options.region }}' } + - { 'key' : 'path-style-access' , 'value' : '{{ bucket_options.path_style_access }}' } + - { 'key' : 'upload-redirect' , 'value' : '{{ bucket_options.upload_redirect }}' } + - { 'key' : 'download-redirect' , 'value' : '{{ bucket_options.download_redirect }}' } + - { 'key' : 'access-key' , 'value' : '{{ bucket_options.access_key }}' } + - { 'key' : 'secret-access-key' , 'value' : '{{ bucket_options.secret_access_key }}' } + +- name: debug + ansible.builtin.debug: + msg: '{{ jvm_options }}' + +- name: set Payara JVM options + become: yes + become_user: '{{ dataverse.payara.user }}' + ansible.builtin.shell: '{{ payara_dir }}/bin/asadmin create-jvm-options "\-Ddataverse.files.{{ bucket_options.id }}.{{ inner.key }}={{ inner.value }}"' + loop: "{{ lookup('dict', jvm_options) }}" + loop_control: + loop_var: inner diff --git a/tasks/main.yml b/tasks/main.yml index f616e7be..b162e9b1 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -90,6 +90,11 @@ tags: - localstack +- ansible.builtin.import_tasks: minio.yml + when: minio.enabled == true + tags: + - minio + - ansible.builtin.import_tasks: dataverse-install.yml tags: - dataverse diff --git a/tasks/minio.yml b/tasks/minio.yml new file mode 100644 index 00000000..1c1b99a9 --- /dev/null +++ b/tasks/minio.yml @@ -0,0 +1,86 @@ +--- + +- ansible.builtin.import_tasks: docker.yml + +- name: STORAGE | Import assert.yml + ansible.builtin.import_tasks: minio_assert.yml + run_once: true + delegate_to: localhost + +- name: STORAGE | Initialized MINIO project directory. + block: + - name: STORAGE | Check MinIO initial directory + ansible.builtin.stat: + path: "{{ minio.docker.project_location }}" + register: minio_init_dir + + - name: STORAGE | Create minio project directory + ansible.builtin.file: + path: "{{ minio.docker.project_location }}" + state: directory + mode: "0755" + recurse: true + when: not minio_init_dir.stat.exists + +- name: STORAGE | Setup MINIO project. + block: + - name: STORAGE | Check MinIO initial directory + ansible.builtin.stat: + path: "{{ minio.docker.project_location }}" + register: minio_dir + + - name: STORAGE | Create minio persistent data directory. + ansible.builtin.file: + path: "{{ minio.docker.project_location }}/data" + state: directory + owner: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.group }}' + mode: "0755" + recurse: true + when: + - minio_dir.stat.exists + - minio_dir.stat.isdir + + - name: STORAGE | Copy MinIO docker compose variables + ansible.builtin.template: + src: minio_compose.j2 + dest: "{{ minio.docker.project_location }}/minio_compose.yml" + owner: '{{ dataverse.payara.user }}' + group: '{{ dataverse.payara.group }}' + mode: "0644" + lstrip_blocks: true + force: true + register: copy_compose + when: + - minio_dir.stat.exists + - minio_dir.stat.isdir + + - name: STORAGE | Get infos on container + community.docker.docker_container_info: + name: "{{ minio.docker.service_name }}" + register: minio_container + + - name: STORAGE | Check docker compose file + ansible.builtin.stat: + path: "{{ minio.docker.project_location }}/minio_compose.yml" + register: compose_file + + - name: STORAGE | Stop `docker-compose down` MinIO + community.docker.docker_compose: + project_src: "{{ minio.docker.project_location }}" + state: absent + remove_orphans: true + become: true + register: continer_stop + when: + - minio_container.exists + - copy_compose.changed + + - name: STORAGE | Run `docker-compose up` MinIO + community.docker.docker_compose: + project_src: "{{ minio.docker.project_location }}" + build: true + files: minio_compose.yml + when: (not minio_container.exists and minio_dir.stat.isdir and compose_file.stat.exists) or continer_stop.changed + +- ansible.builtin.import_tasks: minio_jvm_options.yml diff --git a/tasks/minio_assert.yml b/tasks/minio_assert.yml new file mode 100644 index 00000000..33b04e07 --- /dev/null +++ b/tasks/minio_assert.yml @@ -0,0 +1,16 @@ +--- +- name: STORAGE | Test if minio_docker_root_user is set correctly + ansible.builtin.assert: + that: + - minio.docker.root_user is defined + - minio.docker.root_user is string + - minio.docker.root_user | length > 0 + quiet: true + +- name: STORAGE | Test if minio.docker.root.password is set correctly + ansible.builtin.assert: + that: + - minio.docker.root_password is defined + - minio.docker.root_password is string + - minio.docker.root_password | length > 0 + quiet: true diff --git a/tasks/minio_jvm_options.yml b/tasks/minio_jvm_options.yml new file mode 100644 index 00000000..90f45ba8 --- /dev/null +++ b/tasks/minio_jvm_options.yml @@ -0,0 +1,32 @@ +--- + +- name: flush handlers to restart Payara if needed + meta: flush_handlers + +- name: escape custom endpoint url colons + ansible.builtin.set_fact: "custom_endpoint_escaped={{ minio.docker.custom_endpoint_url | regex_replace (':','\\:') }}" + +- name: build minio dict + ansible.builtin.set_fact: + minio_dict: "{{ minio_dict | default({}) | combine ({ item.key : item.value }) }}" + with_items: + - { 'key' : 'type' , 'value' : 's3' } + - { 'key' : 'label' , 'value' : '{{ minio.docker.label }}' } + - { 'key' : 'custom-endpoint-url' , 'value' : '{{ custom_endpoint_escaped }}' } + - { 'key' : 'custom-endpoint-region' , 'value' : '{{ minio.docker.custom_endpoint_region }}' } + - { 'key' : 'bucket-name' , 'value' : '{{ minio.docker.default_bucket }}' } + - { 'key' : 'path-style-access' , 'value' : 'true' } + - { 'key' : 'upload-redirect' , 'value' : 'false' } + - { 'key' : 'download-redirect' , 'value' : 'false' } + - { 'key' : 'access-key' , 'value' : '{{ minio.docker.access_key }}' } + - { 'key' : 'secret-access-key' , 'value' : '{{ minio.docker.secret_key }}' } + +- name: debug + ansible.builtin.debug: + msg: '{{ minio_dict }}' + +- name: set Payara MinIO JVM-options + become: yes + become_user: '{{ dataverse.payara.user }}' + ansible.builtin.shell: '{{ payara_dir }}/bin/asadmin create-jvm-options "\-Ddataverse.files.minio1.{{ item.key }}={{ item.value }}"' + loop: "{{ lookup('dict', minio_dict) }}" diff --git a/tasks/s3.yml b/tasks/s3.yml index d017e712..4d0094e2 100644 --- a/tasks/s3.yml +++ b/tasks/s3.yml @@ -1,18 +1,6 @@ --- -- name: we need awscli on 7 - yum: - name: awscli - state: latest - when: ansible_os_family == "RedHat" and - ansible_distribution_major_version == "7" - -- name: we need awscli on 8 - pip: - name: awscli - executable: pip3 - when: ansible_os_family == "RedHat" and - ansible_distribution_major_version == "8" +- ansible.builtin.import_tasks: aws_cli.yml - name: ensure dot dir exists file: @@ -38,8 +26,9 @@ group: '{{ dataverse.payara.group }}' mode: '0600' -- name: set storage-driver-id to s3 - shell: '{{ payara_dir}}/bin/asadmin create-jvm-options "-Ddataverse.files.storage-driver-id={{ s3.storage_driver_id }}"' +# this assumes only one S3 datastore. not completely viable any more. +#- name: set storage-driver-id to s3 +# shell: '{{ payara_dir}}/bin/asadmin create-jvm-options "-Ddataverse.files.storage-driver-id={{ s3.storage_driver_id }}"' - name: set s3 type to s3 shell: '{{ payara_dir}}/bin/asadmin create-jvm-options "-Ddataverse.files.s3.type={{ s3.files_type }}"' diff --git a/templates/localstack-compose.yml.j2 b/templates/localstack-compose.yml.j2 index 43189f0e..62f12659 100644 --- a/templates/localstack-compose.yml.j2 +++ b/templates/localstack-compose.yml.j2 @@ -2,19 +2,21 @@ version: '2.1' services: localstack: - container_name: "{{ localstack.container }}" + container_name: "{{ localstack.container_name }}" image: localstack/localstack ports: - - "4567-4599:4567-4599" - - "{{ localstack.web_ui }}-{{ localstack.web_ui }}:{{ localstack.web_ui }}-{{ localstack.web_ui }}" + - "127.0.0.1:4566:4566" + #- "4567-4599:4567-4599" + #- "{{ localstack.web_ui }}-{{ localstack.web_ui }}:{{ localstack.web_ui }}-{{ localstack.web_ui }}" environment: - SERVICES=s3 - DEBUG={{ localstack.debug }} - - DATA_DIR={{ localstack.data_dir }} - - PORT_WEB_UI={{ localstack.web_ui }} + #- DATA_DIR={{ localstack.data_dir }} + #- PORT_WEB_UI={{ localstack.web_ui }} - LOCALSTACK_HOSTNAME={{ localstack.hostname_external }} - - LAMBDA_EXECUTOR=local + #- LAMBDA_EXECUTOR=local - DOCKER_HOST=unix:///var/run/docker.sock volumes: - - "/tmp/localstack:/tmp/localstack" - "/var/run/docker.sock:/var/run/docker.sock" + tmpfs: + - /localstack:mode=770,size=128M,uid=1000,gid=1000 diff --git a/templates/minio_compose.j2 b/templates/minio_compose.j2 new file mode 100644 index 00000000..9178f7c6 --- /dev/null +++ b/templates/minio_compose.j2 @@ -0,0 +1,42 @@ +version: "3" + +#networks: +# {{ minio.docker.network.name }}: +# driver: bridge +# external: {{ minio.docker.network.external }} + +services: + {{ minio.docker.service_name }}: + container_name: {{ minio.docker.service_name }} + image: quay.io/minio/minio:{{ minio.docker.version }} + environment: + TZ: {{ minio.docker.timezone }} + MINIO_ROOT_USER: {{ minio.docker.root_user }} + MINIO_ROOT_PASSWORD: {{ minio.docker.root_password }} + MINIO_ACCESS_KEY: {{ minio.docker.access_key }} + MINIO_SECRET_KEY: {{ minio.docker.secret_key }} + ports: + - {{ minio.docker.server_port }}:{{ minio.docker.server_port }} + - {{ minio.docker.console_port }}:{{ minio.docker.console_port }} + #networks: + # - {{ minio.docker.network.name }} + volumes: + - {{ minio.docker.project_location }}/data:/data:rw + entrypoint: ["/bin/sh","-c"] + command: + - | + mkdir -p /data/{{ minio.docker.default_bucket }} + minio server --address ":{{ minio.docker.server_port }}" --console-address ":{{ minio.docker.console_port }}" /data + healthcheck: + #disable: true + test: ["CMD", "curl", "-f", "http://localhost:{{ minio.docker.server_port }}/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + restart: always + deploy: + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 120s diff --git a/tests/group_vars/jenkins.yml b/tests/group_vars/jenkins.yml index 38b6d2af..0cf930d5 100644 --- a/tests/group_vars/jenkins.yml +++ b/tests/group_vars/jenkins.yml @@ -119,6 +119,7 @@ dataverse: ## Also, changing the default storage takes effect immediately for temp files, but ## only after restart for publishing (i.e. without restart the temp files will be moved to the old default data store at publish time). filesdirs: + #storage_driver_id: file - label: file path: /usr/local/dvn/data # - label: label @@ -145,7 +146,8 @@ dataverse: siteurl: root: /usr/local dir: payara6 - zipurl: https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip + #zipurl: https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip + zipurl: https://mirrors.ibiblio.org/payara/payara-6.2023.8.zip zipchecksum: sha256:108ed0fa7f97bb3833ff00610a4951dd41389e2647dffe28132ad54231af694a google_analytics_key: jacoco: @@ -288,6 +290,10 @@ db: plain: 'no' use_rds: false +docker: + cidr: 10.0.0.99/24 + repo: https://download.docker.com/linux/centos/docker-ce.repo + grafana: grafana_user: 'admin' grafana_password: 'adm1n' @@ -296,20 +302,62 @@ java: version: 17 localstack: - enabled: false - docker: - cidr: - repo: https://download.docker.com/linux/centos/docker-ce.repo - container: 's3-test' + enabled: true + container_name: dev_localstack # set to /tmp/localstack/data to enable persistence - data_dir: + data_dir: /tmp/localstack/data debug: true - hostname_external: + hostname_external: localstack + port: 4566 web_ui: 8888 + buckets: + - label: LocalStack + id: localstack1 + bucket_name: mybucket + enabled: false + access_key: 4cc355_k3y + secret_access_key: s3cr3t_4cc355_k3y + # for non-amazon services. + custom_endpoint_url: "http://localhost:4566" + # note: direct upload and download require CORS on the bucket + download_redirect: true + upload_redirect: true + files_type: s3 + ingestsizelimit: 2000000000 + # for localstack this must be true + path_style_access: true + region: us-east-2 + url_expiration_minutes: 60 + payload_signing: false + chunked_encoding: true maven: version: 3.9.5 +minio: + enabled: true + docker: + version: "latest" + service_name: "minio" + network: + name: "minio" + external: false + project_location: "/home/dataverse/minio" + timezone: "America/New_York" + user: dataverse + group: dataverse + server_port: 9000 + console_port: 9001 + root_user: "4cc355_k3y" + root_password: "s3cr3t_4cc355_k3y" + default_bucket: "mybucket" + label: MinIO + custom_endpoint_url: "http://localhost:9000" + custom_endpoint_region: us-east-1 + path-style-access: true + access_key: 4cc355_k3y + secret_key: s3cr3t_4cc355_k3y + munin: install: false admin: diff --git a/tests/group_vars/vagrant.yml b/tests/group_vars/vagrant.yml index a7683afa..4fd90565 100644 --- a/tests/group_vars/vagrant.yml +++ b/tests/group_vars/vagrant.yml @@ -293,6 +293,10 @@ db: plain: 'no' use_rds: false +docker: + cidr: 10.0.0.99/24 + repo: https://download.docker.com/linux/centos/docker-ce.repo + grafana: grafana_user: 'admin' grafana_password: 'admin' @@ -300,9 +304,63 @@ grafana: java: version: 17 +localstack: + enabled: false + container_name: dev_localstack + # set to /tmp/localstack/data to enable persistence + data_dir: /tmp/localstack/data + debug: true + hostname_external: localstack + port: 4566 + web_ui: 8888 + buckets: + - label: LocalStack + id: localstack1 + bucket_name: mybucket + enabled: false + access_key: 4cc355_k3y + secret_access_key: s3cr3t_4cc355_k3y + # for non-amazon services. + custom_endpoint_url: "http://localhost:4566" + # note: direct upload and download require CORS on the bucket + download_redirect: true + upload_redirect: true + files_type: s3 + ingestsizelimit: 2000000000 + # for localstack this must be true + path_style_access: true + region: us-east-2 + url_expiration_minutes: 60 + payload_signing: false + chunked_encoding: true + maven: version: 3.8.8 +minio: + enabled: false + docker: + version: "latest" + service_name: "minio" + network: + name: "minio" + external: false + project_location: "/home/dataverse/minio" + timezone: "America/New_York" + user: dataverse + group: dataverse + server_port: 9000 + console_port: 9001 + root_user: "4cc355_k3y" + root_password: "s3cr3t_4cc355_k3y" + default_bucket: "mybucket" + label: MinIO + custom_endpoint_url: "http://localhost:9000" + custom_endpoint_region: us-east-1 + path-style-access: true + access_key: 4cc355_k3y + secret_key: s3cr3t_4cc355_k3y + munin: install: false admin: From e8a31de86e717e33f5eb032dfb2a61f58b7e9cd6 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Tue, 12 Dec 2023 14:14:47 -0500 Subject: [PATCH 21/22] bump Jenkins to 6.1 --- tests/group_vars/jenkins.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/group_vars/jenkins.yml b/tests/group_vars/jenkins.yml index 0cf930d5..22215e05 100644 --- a/tests/group_vars/jenkins.yml +++ b/tests/group_vars/jenkins.yml @@ -254,7 +254,7 @@ dataverse: usermgmtkey: burrito deployment: upgrade_only: false - version: '6.0' + version: '6.1' build_guides: false From 3cfe7e1c07b47e7e1514d1145bd64d52c08f4680 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 13 Dec 2023 11:17:06 -0500 Subject: [PATCH 22/22] bump to 6.1 --- defaults/main.yml | 2 +- tests/group_vars/vagrant.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 237a5037..83a32cfe 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -261,7 +261,7 @@ dataverse: usermgmtkey: burrito deployment: upgrade_only: false - version: '6.0' + version: '6.1' # robots_txt_url: https://guides.dataverse.org/en/latest/_downloads/3a5cd7a283eecd5e93289e30af713554/robots.txt build_guides: false diff --git a/tests/group_vars/vagrant.yml b/tests/group_vars/vagrant.yml index 4fd90565..5152cd78 100644 --- a/tests/group_vars/vagrant.yml +++ b/tests/group_vars/vagrant.yml @@ -257,7 +257,7 @@ dataverse: usermgmtkey: burrito deployment: upgrade_only: false - version: '6.0' + version: '6.1' build_guides: false