Merge pull request #31 from CirclesUBI/improve-build-speed

Improve docker build time

Makefile

@@ -1,20 +1,66 @@
-ENV ?= frontend
+# Internal variables
+# ====================================
+pull = pull
+build = build
+skip = skip
+unset = 0
 
-# Common docker-compose method and arguments
-COMPOSE = docker-compose -f docker-compose.yml -f docker-compose.$(ENV).yml -p circles
+# Basic docker-compose variables
+# ====================================
+base_file = -f docker-compose.yml
+namespace = circles
+# ====================================
 
-# Tasks
-build: ## Build containers
-	$(COMPOSE) build
+# User variables
+# ====================================
+# PostgreSQL username for psql command
+POSTGRES_USER ?= postgres
+
+# Arguments to set if api or relayer are built from local folder ("build"),
+# pulled from registry ("pull") or skipped ("skip")
+RELAYER ?= $(pull)
+API ?= $(pull)
+
+# Flag to set if database and redis ports can be used outside of docker network
+EXPOSE_PORTS ?= $(unset)
+# ====================================
+
+# Use user arguments to build docker-compose base command
+ifneq ($(unset), $(EXPOSE_PORTS))
+	EXPOSE_PORTS_FILE = -f docker-compose.expose-ports.yml
+endif
+
+ifeq ($(pull), $(RELAYER))
+	RELAYER_FILE = -f docker-compose.relayer-pull.yml
+endif
+
+ifeq ($(build), $(RELAYER))
+	RELAYER_FILE = -f docker-compose.relayer-pull.yml -f docker-compose.relayer-build.yml
+endif
+
+ifeq ($(pull), $(API))
+	API_FILE = -f docker-compose.api-pull.yml
+endif
 
+ifeq ($(build), $(API))
+	API_FILE = -f docker-compose.api-pull.yml -f docker-compose.api-build.yml
+endif
+
+COMPOSE_UP = docker-compose ${base_file} ${RELAYER_FILE} ${API_FILE} ${EXPOSE_PORTS_FILE} -p ${namespace}
+
+# Address all containers even when they are not used. This is useful as a
+# independent "catch all" regardless of which containers were started
+COMPOSE_ALL = docker-compose ${base_file} -f docker-compose.relayer-${pull}.yml -f docker-compose.api-${pull}.yml -p ${namespace}
+
+# Tasks
 up: ## Start containers in background
-	$(COMPOSE) up -d
+	$(COMPOSE_UP) up --detach --remove-orphans --build
 
-down: ## Stop containers
-	$(COMPOSE) down
+down: ## Stop containers and remove attached volumes
+	$(COMPOSE_ALL) down --remove-orphans --volumes
 
 logs: ## Follow container logs
-	$(COMPOSE) logs -f --tail 100
+	$(COMPOSE_ALL) logs --follow --tail 100
 
 contracts: ## Download and migrate contracts
 	./scripts/migrate-contracts.sh
@@ -25,7 +71,13 @@ subgraph: ## Create and deploy subgraph
 clean: ## Remove temporary files
 	rm -rf .tmp
 
-.PHONY: build up down logs subgraph contracts clean
+sh: ## Run shell in a container
+	docker exec -it $(c) /bin/sh
+
+psql: ## Run psql in circles-db container
+	docker exec -it circles-db psql -U $(POSTGRES_USER)
+
+.PHONY: up down logs subgraph contracts clean sh psql
 
 .DEFAULT_GOAL := help
 

README.md

@@ -11,7 +11,7 @@ Infrastructure provisioning for Circles development.
 
 ## Setup
 
-* Make a copy of `.env.example` and rename it to `.env`, edit the file according to your needs. The default values should be sufficient if you only need a Circles backend during client development.
+* Make a copy of `.env.example` and rename it to `.env`, edit the file according to your needs. The default values should be sufficient for local development on your machine.
 
 * Edit your `/etc/hosts` file and add the following hostnames (or similar, depending on your `.env` configuration):
 
@@ -23,14 +23,36 @@ Infrastructure provisioning for Circles development.
 
 ## Usage
 
-```
-# Build all docker containers
-make build
+Please note that depending on how you installed Docker you might need to run these commands with `sudo` before them (except of `make contracts` and `make subgraph`).
 
-# Start containers
+```bash
+# Start all containers
 make up
 
-# Show container logs
+# If you need more control on how the `circles-api` and `safe-relay-service`
+# are handled during local development you can use the following values:
+#
+# pull = Loads the pre-built image from external registry (default)
+# build = Builds the container from your computer
+# skip = Skips starting this container
+#
+# Pass on these values for the `API` and `RELAYER` arguments depending on your
+# needs:
+make up API=skip|pull|build RELAYER=skip|pull|build
+
+# Example: Start all containers except api and safe-relayer-service
+make up API=skip RELAYER=skip
+
+# Example: Start all containers, but build relayer container from local
+# `safe-relay-service` repository (clone the project next to the
+# `circles-docker` repository)
+make up RELAYER=build
+
+# You can also expose the ports of the PostgreSQL and Redis database to use
+# them outside of the docker network
+make up EXPOSE_PORTS=1
+
+# Show all container logs
 make logs
 
 # Download and migrate contracts
@@ -39,38 +61,17 @@ make contracts
 # Build and upload subgraph
 make subgraph
 
-# Stop all containers
+# Stop all containers and remove attached volumes
 make down
 
 # Remove temporary files
 make clean
 
-# Convenient full-reset during development
-make down \
-    && docker container prune -f \
-    && docker volume prune -f \
-    && make build \
-    && make up \
-    && make contracts \
-    && make subgraph
-```
-
-## Development
+# Run interactive shell in container
+make sh c=circles-api
 
-Clone [circles-api](https://github.com/CirclesUBI/circles-api) and [safe-relay-service](https://github.com/CirclesUBI/safe-relay-service) into the parent folder of `circles-docker` if you're about to set up a development environment of the api or relayer.
-
-```
-├── circles-api
-├── circles-docker
-└── safe-relay-service
-```
-
-Start all `make` commands with `ENV=backend` to point the build steps against the local repositories. For example:
-
-```
-make ENV=backend build
-make ENV=backend up
-make ENV=backend down
+# Connect to PostgreSQL database via psql
+make psql
 ```
 
 ## License

docker-compose.api-build.yml

@@ -0,0 +1,14 @@
+version: '3'
+
+services:
+  nginx:
+    volumes:
+      - ./api/nginx.conf:/etc/nginx/vhost.d/${HOST_API}
+
+  api: &api
+    build: ../circles-api
+    volumes:
+      - "../circles-api/src:/usr/src/app/src"
+
+  api-worker:
+    <<: *api

docker-compose.api-pull.yml

@@ -0,0 +1,38 @@
+version: '3'
+
+volumes:
+  pathfinder:
+
+services:
+  nginx:
+    volumes:
+      - ./api/nginx.conf:/etc/nginx/vhost.d/${HOST_API}
+
+  api: &api
+    image: joincircles/circles-api:v1.3.2
+    container_name: circles-api
+    depends_on:
+      - db
+    env_file:
+      - .env
+    environment:
+      - VIRTUAL_HOST=${HOST_API}
+      - NODE_ENV=development
+      - POSTGRES_HOST=db
+      - POSTGRES_PORT=5432
+      - REDIS_URL=redis://redis:6379/0
+      - API_SERVICE_ENDPOINT=http://api:3000
+      - GRAPH_NODE_ENDPOINT=http://graph:8000
+      - RELAY_SERVICE_ENDPOINT=http://relayer-service:8888
+    volumes:
+      - pathfinder:/usr/src/app/edges-data
+    command: "npm run start"
+
+  api-worker:
+    <<: *api
+    container_name: circles-api-worker
+    depends_on:
+      - db
+      - api
+      - graph
+    command: "npm run worker:start"

docker-compose.expose-ports.yml

@@ -0,0 +1,10 @@
+version: '3'
+
+services:
+  db:
+    ports:
+      - "5432:5432"
+
+  redis:
+    ports:
+      - "6379:6379"

docker-compose.backend.yml → docker-compose.relayer-build.yml

@@ -1,14 +1,6 @@
 version: '3'
 
 services:
-  db:
-    ports:
-      - "5432:5432"
-
-  redis:
-    ports:
-      - "6379:6379"
-
   relayer-service: &relayer-service
     build: ../safe-relay-service
     volumes:
@@ -19,11 +11,3 @@ services:
 
   relayer-scheduler:
     <<: *relayer-worker
-
-  api: &api
-    build: ../circles-api
-    volumes:
-      - "../circles-api/src:/usr/src/app/src"
-
-  api-worker:
-    <<: *api

docker-compose.relayer-pull.yml

@@ -0,0 +1,51 @@
+version: '3'
+
+volumes:
+  relayer:
+  relayer_html:
+
+services:
+  nginx:
+    volumes:
+      - relayer_html:/usr/share/nginx/html/relayer
+      - ./relayer/nginx.conf:/etc/nginx/vhost.d/${HOST_RELAYER}
+
+  relayer-service: &relayer-service
+    image: joincircles/safe-relay-service:v4.1.7
+    container_name: circles-relayer-service
+    depends_on:
+      - db
+      - ganache
+      - redis
+    working_dir: /safe-relay-service
+    env_file:
+      - .env
+    environment:
+      - VIRTUAL_HOST=${HOST_RELAYER}
+      - GRAPH_NODE_ENDPOINT=http://graph:8000
+      - RELAYER_REDIS_URL=redis://redis:6379/0
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - POSTGRES_HOST=db
+      - POSTGRES_PORT=5432
+      - C_FORCE_ROOT=true
+      - DJANGO_SECRET_KEY=development
+      - DJANGO_SETTINGS_MODULE=config.settings.development
+    volumes:
+      - relayer:/nginx
+      - relayer_html:/usr/share/nginx/html/relayer
+    command: ./run.sh
+
+  relayer-worker: &relayer-worker
+    <<: *relayer-service
+    container_name: circles-relayer-worker
+    depends_on:
+      - db
+      - ganache
+      - redis
+      - relayer-service
+    command: ./run-worker.sh
+
+  relayer-scheduler:
+    <<: *relayer-worker
+    container_name: circles-relayer-scheduler
+    command: ./run-scheduler.sh