#243 fix merge conflicts in ec2-create.sh

gdcc · Dec 14, 2023 · 552f66d · 552f66d
2 parents 2ecac8f + 3cfe7e1
commit 552f66d
Show file tree

Hide file tree

Showing 33 changed files with 723 additions and 220 deletions.
diff --git a/defaults/main.yml b/defaults/main.yml
@@ -41,6 +41,8 @@ letsencrypt:
     email:
     user: certbot
 
+dataverse_misc_files_dir: '/opt/dv'
+
 dataverse:
   adminpass: admin1
   allow_signups: true
@@ -78,7 +80,7 @@ dataverse:
     language_packs:
       source: https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs.git
       version: develop
-    lang_directory: /opt/dv/lang
+    lang_directory: "{{ dataverse_misc_files_dir }}/lang"
   licenses:
     enabled: false
     user: dataverseAdmin
@@ -165,7 +167,7 @@ dataverse:
   handlenet:
     independenthandleservice: 'false'
 #   handleauthhandle: YOUR:HANDLE/USERNAME
-    admcredfile: /opt/dv/admpriv.bin
+    admcredfile: "{{ dataverse_misc_files_dir }}/admpriv.bin"
     admcredfile_source: files/handlenet/admcredfile
 #   admprivphrase:
 #   index: 300
@@ -244,20 +246,22 @@ dataverse:
   service_email: noreply@dataverse.yourinstitution.edu
   smtp: localhost # or the FQDN of your organization's SMTP relay
   solr:
-    download_url: https://dlcdn.apache.org/solr/solr/9.3.0/solr-9.3.0.tgz
+    download_url: https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz
     group: solr
     root: /usr/local/solr
     user: solr
     version: 9.3.0
     listen: 127.0.0.1
     checksum: sha256:2a924a776d7c06dea561f12bdb17291dbaecc9e9a06f363d4599acb7fb7bfa71
-  srcdir: /tmp/dataverse
+  srcdir: /opt/dataverse/dataverse
   thumbnails: false
   unittests:
     enabled: false
     argument: '-DcompilerArgument=-Xlint:unchecked test -P all-unit-tests'
   usermgmtkey: burrito
-  version: '6.0'
+  deployment:
+    upgrade_only: false
+  version: '6.1'
 #  robots_txt_url: https://guides.dataverse.org/en/latest/_downloads/3a5cd7a283eecd5e93289e30af713554/robots.txt
 
 build_guides: false
@@ -294,29 +298,76 @@ db:
     plain: 'no'
   use_rds: false
 
+docker:
+  cidr: 10.0.0.99/24
+  repo: https://download.docker.com/linux/centos/docker-ce.repo
+
 grafana:
   grafana_user: 'admin'
   grafana_password: 'adm1n'
 
 java:
   version: 17
-  home: /usr/lib/jvm/java-{{ java.version}}
+  # this breaks templating.
+  #home: /usr/lib/jvm/java-{{ java.version}}
 
 localstack:
   enabled: false
-  docker:
-    cidr: 
-    repo: https://download.docker.com/linux/centos/docker-ce.repo
-  container: 's3-test'
+  container_name: dev_localstack
   # set to /tmp/localstack/data to enable persistence
-  data_dir:
+  data_dir: /tmp/localstack/data
   debug: true
-  hostname_external: 
+  hostname_external: localstack
+  port: 4566
   web_ui: 8888
+  buckets:
+    - label: LocalStack
+      id: localstack1
+      bucket_name: mybucket
+      enabled: false
+      access_key: 4cc355_k3y
+      secret_access_key: s3cr3t_4cc355_k3y
+      # for non-amazon services.
+      custom_endpoint_url: "http://localhost:4566"
+      # note: direct upload and download require CORS on the bucket
+      download_redirect: true
+      upload_redirect: true
+      files_type: s3
+      ingestsizelimit: 2000000000
+      # for localstack this must be true
+      path_style_access: true
+      region: us-east-2
+      url_expiration_minutes: 60
+      payload_signing: false
+      chunked_encoding: true
 
 maven:
   version: 3.8.8
 
+minio:
+  enabled: false
+  docker:
+    version: "latest"
+    service_name: "minio"
+    network:
+      name: "minio"
+      external: false
+    project_location: "/home/dataverse/minio"
+    timezone: "America/New_York"
+    user: dataverse
+    group: dataverse
+    server_port: 9000
+    console_port: 9001
+    root_user: "4cc355_k3y"
+    root_password: "s3cr3t_4cc355_k3y"
+    default_bucket: "mybucket"
+    label: MinIO
+    custom_endpoint_url: "http://localhost:9000"
+    custom_endpoint_region: us-east-1
+    path-style-access: true
+    access_key: 4cc355_k3y
+    secret_key: s3cr3t_4cc355_k3y
+
 munin:
   install: false
   admin:

diff --git a/ec2/ec2-create-instance.sh b/ec2/ec2-create-instance.sh
@@ -208,7 +208,7 @@ fi
 
 echo "Creating EC2 instance"
 # TODO: Add some error checking for "ec2 run-instances".
-INSTANCE_ID=$(aws $PROFILE ec2 run-instances --image-id $AMI_ID --security-groups $AWS_SG $TAGARG --count 1 --instance-type $SIZE --key-name $KEY_NAME --query 'Instances[0].InstanceId' --block-device-mappings '[ { "DeviceName": "/dev/sda1", "Ebs": { "DeleteOnTermination": true } } ]' | tr -d \")
+INSTANCE_ID=$(aws $PROFILE ec2 run-instances --image-id $AMI_ID --security-groups $AWS_SG $TAGARG --count 1 --instance-type $SIZE --key-name $KEY_NAME --query 'Instances[0].InstanceId' --block-device-mappings '[ { "DeviceName": "/dev/sda1", "Ebs": { "DeleteOnTermination": true, "VolumeSize": 20 } } ]' | tr -d \")
 echo "Instance ID: "$INSTANCE_ID
 
 DESTROY_CMD="aws $PROFILE ec2 terminate-instances --instance-ids $INSTANCE_ID"
@@ -236,7 +236,7 @@ fi
 # TODO: Add some error checking for this ssh command.
 ssh -T -i $PEM_FILE -o 'StrictHostKeyChecking no' -o 'UserKnownHostsFile=/dev/null' -o 'ConnectTimeout=300' $USER_AT_HOST <<EOF
 sudo dnf -q -y install epel-release
-sudo dnf -q -y install ansible git nano
+sudo dnf -q -y install ansible git
 git clone -b $DA_BRANCH https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible.git dataverse
 export ANSIBLE_ROLES_PATH=.
 ansible-playbook $VERBOSE_ARG -i dataverse/inventory dataverse/dataverse.pb --connection=local $GVARG
@@ -251,15 +251,16 @@ if [ ! -z "$LOCAL_LOG_PATH" ]; then
    # 1 logdir should exist
    mkdir -p $LOCAL_LOG_PATH
    # 2 grab logs for local processing in jenkins
-   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/site $LOCAL_LOG_PATH/
-   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/surefire-reports $LOCAL_LOG_PATH/
+   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/site $LOCAL_LOG_PATH/
+   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/surefire-reports $LOCAL_LOG_PATH/
    # 3 grab mvn.out
-   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/mvn.out $LOCAL_LOG_PATH/
+   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/mvn.out $LOCAL_LOG_PATH/
    # 4 jacoco
-   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/coverage-it $LOCAL_LOG_PATH/
-   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/*.exec $LOCAL_LOG_PATH/
-   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/target/classes $LOCAL_LOG_PATH/
-   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/src $LOCAL_LOG_PATH/
+   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/coverage-it $LOCAL_LOG_PATH/
+   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/coverage-reports/jacoco-unit.exec $LOCAL_LOG_PATH/
+   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/jacoco_merged.exec $LOCAL_LOG_PATH/
+   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/target/classes $LOCAL_LOG_PATH/
+   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/opt/dataverse/dataverse/src $LOCAL_LOG_PATH/
    # 5 server.logs
    rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/usr/local/payara*/glassfish/domains/domain1/logs/server.log* $LOCAL_LOG_PATH/
    # 6 query_count.out

diff --git a/files/cors.json b/files/cors.json
@@ -0,0 +1,10 @@
+{
+  "CORSRules": [
+     {
+        "AllowedOrigins": ["*"],
+        "AllowedHeaders": ["*"],
+        "AllowedMethods": ["PUT", "GET"],
+        "ExposeHeaders": ["ETag", "Accept-Ranges", "Content-Encoding", "Content-Range"]
+     }
+  ]
+}
diff --git a/tasks/aws_cli.yml b/tasks/aws_cli.yml
@@ -0,0 +1,14 @@
+---
+
+#- name: we need awscli on 8
+#  pip:
+#    name: awscli
+#    executable: pip3
+#  when: (ansible_os_family == "RedHat") and
+#        (ansible_distribution_major_version) == "8"
+
+- name: just use OS-included binary for now
+  ansible.builtin.package:
+    name: awscli
+    state: latest
+  when: ansible_os_family == "RedHat"
diff --git a/tasks/aws_config.yml b/tasks/aws_config.yml
@@ -0,0 +1,25 @@
+---
+
+- name: ensure dot dir exists
+  file:
+    path: '~{{ dataverse.payara.user }}/.aws'
+    state: directory
+    owner: '{{ dataverse.payara.user }}'
+    group: '{{ dataverse.payara.group }}'
+    mode: '0700'
+
+- name: place aws credentials
+  template:
+    src: aws_credentials.j2
+    dest: '~{{ dataverse.payara.user }}/.aws/credentials'
+    owner: '{{ dataverse.payara.user }}'
+    group: '{{ dataverse.payara.group }}'
+    mode: '0600'
+
+- name: place aws config
+  template:
+    src: aws_config.j2
+    dest: '~{{ dataverse.payara.user }}/.aws/config'
+    owner: '{{ dataverse.payara.user }}'
+    group: '{{ dataverse.payara.group }}'
+    mode: '0600'
diff --git a/tasks/dataverse-api-testsuite.yml b/tasks/dataverse-api-testsuite.yml
@@ -89,6 +89,8 @@
     DV_APIKEY: '{{ dataverse.usermgmtkey }}'
     DV_API_KEY: '{{ dataverse.usermgmtkey }}'
   when: maven.version == 'default'
+  become: true
+  become_user: '{{ dataverse.payara.user }}'
 
 - name: run test suite with custom maven
   shell: 'source /etc/profile.d/maven.sh && mvn test -Dtest={{ integration_test_list }} -Ddataverse.test.baseurl=http://localhost:8080'
@@ -99,18 +101,24 @@
     DV_APIKEY: '{{ dataverse.usermgmtkey }}'
     DV_API_KEY: '{{ dataverse.usermgmtkey }}'
   when: maven.version != 'default'
+  become: true
+  become_user: '{{ dataverse.payara.user }}'
 
 - name: generate HTML reports with default maven
   shell: 'mvn surefire-report:report'
   args:
     chdir: '{{ dataverse.srcdir }}'
   when: maven.version == 'default'
+  become: true
+  become_user: '{{ dataverse.payara.user }}'
 
 - name: generate HTML reports with custom maven
   shell: 'source /etc/profile.d/maven.sh && mvn surefire-report:report'
   args:
     chdir: '{{ dataverse.srcdir }}'
   when: maven.version != 'default'
+  become: true
+  become_user: '{{ dataverse.payara.user }}'
 
 - ansible.builtin.import_tasks: postgresql_perl_party.yml
   when:

diff --git a/tasks/dataverse-branch.yml b/tasks/dataverse-branch.yml
@@ -4,9 +4,21 @@
   debug:
     msg: '##### CLONE BRANCH #####'
 
+- name: ensure service account and group exist
+  ansible.builtin.import_tasks: payara_service_account.yml
+
+- name: ensure target dir exists
+  ansible.builtin.file:
+    path: '{{ dataverse.srcdir }}'
+    state: directory
+    owner: '{{ dataverse.payara.user }}'
+    group: '{{ dataverse.payara.user }}'
+
 - name: clone specified branch
   git:
     repo: "{{ dataverse_repo }}"
     dest: "{{ dataverse.srcdir }}"
     version: "{{ dataverse_branch }}"
   when: dataverse_branch != "release"
+  become: true
+  become_user: '{{ dataverse.payara.user }}'
diff --git a/tasks/dataverse-build.yml b/tasks/dataverse-build.yml
@@ -32,27 +32,35 @@
     chdir: "{{ dataverse.srcdir }}"
   when: dataverse.unittests.enabled == false and
         maven.version == 'default'
+  become: true
+  become_user: '{{ dataverse.payara.user }}'
 
-- name: build warfile without tests and custom maven. tail /tmp/dataverse/mvn.out for gory details.
+- name: build warfile without tests and custom maven. tail /home/dataverse/dataverse/mvn.out for gory details.
   shell: "source /etc/profile.d/maven.sh && mvn -T 2C clean package -Dmaven.test.skip=true > mvn.out"
   args:
     chdir: "{{ dataverse.srcdir }}"
   when: dataverse.unittests.enabled == false and
         maven.version != 'default'
+  become: true
+  become_user: '{{ dataverse.payara.user }}'
 
 - name: build warfile with specified tests. tail /tmp/dataverse/mvn.out for gory details.
   shell: "mvn {{ dataverse.unittests.argument }} {{ jacoco_arg }} -T 2C clean package > mvn.out"
   args:
     chdir: "{{ dataverse.srcdir }}"
   when: dataverse.unittests.enabled == true and
         maven.version == 'default'
+  become: true
+  become_user: '{{ dataverse.payara.user }}'
 
-- name: build warfile with specified tests using custom maven. tail /tmp/dataverse/mvn.out for gory details.
+- name: build warfile with specified tests using custom maven. tail /home/dataverse/dataverse/mvn.out for gory details.
   shell: "source /etc/profile.d/maven.sh && mvn {{ dataverse.unittests.argument }} {{ jacoco_arg }} -T 2C clean package > mvn.out"
   args:
     chdir: "{{ dataverse.srcdir }}"
   when: dataverse.unittests.enabled == true and
         maven.version != 'default'
+  become: true
+  become_user: '{{ dataverse.payara.user }}'
 
 - name: copy warfile over release
   copy:

diff --git a/tasks/dataverse-install.yml b/tasks/dataverse-install.yml
@@ -10,6 +10,14 @@
   debug:
     msg: '##### DATAVERSE INSTALLER #####'
 
+# this is necessary for the installer. dataverse-storage.yml does the real work later.
+- name: filesdir must pre-exist
+  ansible.builtin.file:
+    path: '{{ dataverse.filesdirs[0].path }}'
+    state: directory
+    owner: '{{ dataverse.payara.user }}'
+    group: '{{ dataverse.payara.group }}'
+
 - ansible.builtin.import_tasks: dataverse-build.yml
   when: dataverse_branch != "release"
 
@@ -143,12 +151,20 @@
   become_user: '{{ dataverse.payara.user }}'
   args:
     chdir: /tmp/dvinstall
+  when: not dataverse.deployment.upgrade_only
+
+- name: upgrade only -- deploy dataverse war
+  shell: '{{ payara_dir }}/bin/asadmin redeploy --name dataverse /tmp/dvinstall/dataverse.war'
+  become: yes
+  become_user: '{{ dataverse.payara.user }}'
+  when: dataverse.deployment.upgrade_only
 
 - name: stop payara manually (systemd gums up the works)
   become: yes
   become_user: "{{ dataverse.payara.user }}"
   shell: '{{ payara_dir }}/bin/asadmin stop-domain {{ dataverse.payara.domain }}'
   ignore_errors: yes
+  when: not dataverse.deployment.upgrade_only
 
 - name: now start payara with systemd
   service: