Merge pull request #87 from IQSS/feature/dev-environment

65 - Containerized development environment

.gitignore

@@ -39,3 +39,8 @@ yarn-error.log*
 
 #deployment
 /deployment/payara/target
+
+#dev environment
+/dev-env/docker-dev-volumes
+/dev-env/dataverse
+/dev-env/dataverse-sample-data

README.md

@@ -62,6 +62,43 @@ Launches the prettier formatter. We recommend you to configure your IDE to run p
 Runs the Storybook in the development mode.  
 Open [http://localhost:6006](http://localhost:6006) to view it in your browser.
 
+## Local development environment
+
+A containerized environment, oriented to local development, is available to be run from the repository.
+
+This environment contains a dockerized instance of the Dataverse backend with its dependent services (database, mailserver, etc), as well as an npm development server running the SPA frontend (With code autoupdating).
+
+This environment is intended for locally testing any functionality that requires access to the Dataverse API from the SPA frontend.
+
+There is an Nginx reverse proxy container on top of the frontend and backend containers to avoid CORS issues while testing the application.
+
+### Run the environment
+
+Inside the `dev-env` folder, run the following command:
+
+```
+./run-env <dataverse_branch_name>
+```
+
+As the script argument, add the name of the Dataverse backend branch you want to deploy.
+
+Note that both the branch and the associated tag in the docker registry must to be pre pushed, otherwise the script will fail.
+
+If you are running the script for the first time, it may take a while, since `npm install` has to install all the dependencies. This can also happen if you added new dependencies to package.json.
+
+Once the script has finished, you will be able to access Dataverse via:
+
+- [http://localhost:8000/spa](http://localhost:8000/spa): SPA Frontend
+- [http://localhost:8000](http://localhost:8000): Dataverse Backend and JSF Frontend
+
+### Remove the environment
+
+To clean up your environment of any running environment containers, as well as any associated data volumes, run this script inside the `dev-env` folder:
+
+```
+./rm-env
+```
+
 ## Deployment
 
 Once the site is built through the `npm run build` command, it can be deployed in different ways to different types of infrastructure, depending on installation needs.

dev-env/.env

@@ -0,0 +1,4 @@
+POSTGRES_VERSION=13
+DATAVERSE_DB_USER=dataverse
+SOLR_VERSION=8.11.1
+REGISTRY=ghcr.io

dev-env/docker-compose-dev.yml

@@ -0,0 +1,137 @@
+version: '2.4'
+
+services:
+  dev_nginx:
+    container_name: 'dev_nginx_proxy'
+    image: nginx:stable
+    ports:
+      - '8000:80'
+    networks:
+      - dataverse
+    depends_on:
+      - dev_dataverse
+      - dev_frontend
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+      - ./docker-dev-volumes/nginx/logs:/var/log/nginx/
+
+  dev_frontend:
+    container_name: 'dev_frontend'
+    hostname: frontend
+    build:
+      context: ../
+      dockerfile: ./dev.Dockerfile
+      network: host
+    expose:
+      - '5173'
+    stdin_open: true
+    networks:
+      - dataverse
+    depends_on:
+      - dev_dataverse
+    environment:
+      - VITE_DATAVERSE_BACKEND_URL=http://localhost:8000
+    volumes:
+      - ../:/usr/src/app
+      - /usr/src/app/dev-env
+      - /usr/src/app/node_modules
+
+  dev_dataverse:
+    container_name: 'dev_dataverse'
+    hostname: dataverse
+    image: ${REGISTRY}/gdcc/dataverse:${DATAVERSE_BRANCH_NAME}
+    restart: on-failure
+    user: payara
+    environment:
+      - DATAVERSE_DB_HOST=postgres
+      - DATAVERSE_DB_PASSWORD=secret
+      - DATAVERSE_DB_USER=${DATAVERSE_DB_USER}
+      - DATAVERSE_FEATURE_API_SESSION_AUTH=1
+    # We open 8080, rather than just expose, so that the docker-final-setup.sh script works from the run-env.sh script
+    ports:
+      - '8080:8080'
+    networks:
+      - dataverse
+    depends_on:
+      - dev_postgres
+      - dev_solr
+    volumes:
+      - ./docker-dev-volumes/app/data:/dv
+      - ./docker-dev-volumes/app/secrets:/secrets
+    tmpfs:
+      - /dumps:mode=770,size=2052M,uid=1000,gid=1000
+      - /tmp:mode=770,size=2052M,uid=1000,gid=1000
+    mem_limit: 2147483648 # 2 GiB
+    mem_reservation: 1024m
+    privileged: false
+
+  dev_postgres:
+    container_name: 'dev_postgres'
+    hostname: postgres
+    image: postgres:${POSTGRES_VERSION}
+    restart: on-failure
+    environment:
+      - POSTGRES_USER=${DATAVERSE_DB_USER}
+      - POSTGRES_PASSWORD=secret
+    ports:
+      - '5432:5432'
+    networks:
+      - dataverse
+    volumes:
+      - ./docker-dev-volumes/postgresql/data:/var/lib/postgresql/data
+
+  dev_solr_initializer:
+    container_name: 'dev_solr_initializer'
+    image: alpine
+    restart: 'no'
+    command:
+      - sh
+      - -c
+      - 'chown 8983:8983 /conf /var/solr && cp *.xml /conf'
+    volumes:
+      - ./docker-dev-volumes/solr/data:/var/solr
+      - ./docker-dev-volumes/solr/conf:/conf
+      - ./dataverse/conf/solr/8.11.1/schema.xml:/schema.xml
+      - ./dataverse/conf/solr/8.11.1/solrconfig.xml:/solrconfig.xml
+
+  dev_solr:
+    container_name: 'dev_solr'
+    hostname: 'solr'
+    image: solr:${SOLR_VERSION}
+    depends_on:
+      - dev_solr_initializer
+    restart: on-failure
+    expose:
+      - '8983'
+    networks:
+      - dataverse
+    command:
+      - bash
+      - -c
+      - 'cd /opt/solr-${SOLR_VERSION}/server/solr/configsets/_default/conf && cp -R -n . /template && solr-precreate collection1 /template'
+    volumes:
+      - ./docker-dev-volumes/solr/data:/var/solr
+      - ./docker-dev-volumes/solr/conf:/template
+
+  dev_smtp:
+    container_name: 'dev_smtp'
+    hostname: 'smtp'
+    image: maildev/maildev:2.0.5
+    restart: on-failure
+    expose:
+      - '25' # smtp server
+    ports:
+      - '1080:1080' # web ui
+    environment:
+      - MAILDEV_SMTP_PORT=25
+      - MAILDEV_MAIL_DIRECTORY=/mail
+    networks:
+      - dataverse
+    #volumes:
+    #  - ./docker-dev-volumes/smtp/data:/mail
+    tmpfs:
+      - /mail:mode=770,size=128M,uid=1000,gid=1000
+
+networks:
+  dataverse:
+    driver: bridge

dev-env/dvconfig.py

@@ -0,0 +1,16 @@
+base_url = 'http://localhost:8000'
+api_token = '<DATAVERSE_API_TOKEN>'
+sample_data = [
+'data/dataverses/pums/pums.json',
+'data/dataverses/pums/datasets/2000pums5/2000pums5.json',
+'data/dataverses/dataverseno/dataverseno.json',
+'data/dataverses/open-source-at-harvard/open-source-at-harvard.json',
+'data/dataverses/open-source-at-harvard/dataverses/dataverse-project/dataverse-project.json',
+'data/dataverses/open-source-at-harvard/dataverses/dataverse-project/datasets/dataverse-irc-metrics/dataverse-irc-metrics.json',
+'data/dataverses/ubiquity-press/ubiquity-press.json',
+'data/dataverses/ubiquity-press/dataverses/jopd/jopd.json',
+'data/dataverses/ubiquity-press/dataverses/jopd/datasets/flynn-effect-in-estonia/flynn-effect-in-estonia.json',
+'data/dataverses/ubiquity-press/dataverses/jopd/datasets/bafacalo/bafacalo.json',
+'data/dataverses/king/king.json',
+'data/dataverses/king/datasets/cause-of-death/cause-of-death.json',
+]

dev-env/nginx.conf

@@ -0,0 +1,18 @@
+events {}
+http {
+  server {
+    listen 80;
+    server_name  localhost;
+
+    location / {
+      proxy_pass http://dataverse:8080;
+    }
+
+    location /spa {
+      proxy_pass http://frontend:5173;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "Upgrade";
+    }
+  }
+}

dev-env/rm-env.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+docker-compose -f "./docker-compose-dev.yml" down
+rm -rf ./docker-dev-volumes

dev-env/run-env.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+export DATAVERSE_BRANCH_NAME=$1
+
+# To avoid timeout issues on frontend container startup
+export COMPOSE_HTTP_TIMEOUT=200
+
+DATAVERSE_API_BASE_URL=http://localhost:8000/api
+
+echo "INFO - Setting up Dataverse on branch ${DATAVERSE_BRANCH_NAME}..."
+
+echo "INFO - Removing current environment if exists..."
+./rm-env.sh
+
+echo "INFO - Cloning Dataverse backend repository..."
+git clone -b ${DATAVERSE_BRANCH_NAME} git@github.com:IQSS/dataverse.git
+
+echo "INFO - Running docker containers..."
+docker-compose -f "./docker-compose-dev.yml" up -d --build
+
+echo "INFO - Waiting for containers to be ready..."
+# Up to ~5 minutes
+max_attempts=30
+n_attempts=0
+until $(curl --output /dev/null --silent --head --fail ${DATAVERSE_API_BASE_URL}/info/version); do
+   if [ ${n_attempts} -eq ${max_attempts} ];then
+     echo "ERROR - Timeout reached while waiting for containers to be ready"
+     ./rm-env.sh
+     rm -rf dataverse
+     exit 1
+   fi
+   n_attempts=$(($n_attempts+1))
+   sleep 10
+done
+
+echo "INFO - Bootstrapping dataverse..."
+cd dataverse
+./scripts/dev/docker-final-setup.sh
+
+echo "INFO - Cleaning up repository..."
+cd ..
+rm -rf dataverse
+
+echo "INFO - Cloning Dataverse sample data repository..."
+git clone git@github.com:IQSS/dataverse-sample-data.git
+
+echo "INFO - Configuring Dataverse sample data repository..."
+cd dataverse-sample-data
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+cp ../dvconfig.py ./dvconfig.py
+curl -X PUT -d 'true' ${DATAVERSE_API_BASE_URL}/admin/settings/:AllowApiTokenLookupViaApi
+dataverse_api_token=$(python3 get_api_token.py)
+sed -i '' "s/<DATAVERSE_API_TOKEN>/${dataverse_api_token}/g" dvconfig.py
+
+echo "INFO - Creating sample data..."
+python3 create_sample_data.py
+
+echo "INFO - Cleaning up repository..."
+cd ..
+rm -rf dataverse-sample-data

dev.Dockerfile

@@ -0,0 +1,6 @@
+FROM node:19.6.1
+WORKDIR /usr/src/app
+COPY package.json ./
+RUN npm install
+EXPOSE 5173
+CMD ["npm", "start"]

package.json

@@ -35,7 +35,7 @@
     "web-vitals": "^2.1.4"
   },
   "scripts": {
-    "start": "vite",
+    "start": "vite --base=/spa",
     "build": "tsc && vite build",
     "preview": "vite preview",
     "lint": "eslint --ignore-path .gitignore . && stylelint **/*.scss && prettier --check '**/*.(yml|json|md|css)' --ignore-path packages/design-system/.gitignore",

vite.config.ts

@@ -12,5 +12,13 @@ export default defineConfig({
   ],
   preview: {
     port: 5173
+  },
+  server: {
+    //https://github.com/vitejs/vite/discussions/3396
+    host: true,
+    port: 5173,
+    hmr: {
+      clientPort: 8000 // nginx reverse proxy port
+    }
   }
 })