Skip to content

Commit

Permalink
Merge pull request #4804 from Joystream/docker-compose-tweaks
Browse files Browse the repository at this point in the history
Docker compose tweaks
  • Loading branch information
mnaamani authored Jul 22, 2023
2 parents 9a35c55 + a89d608 commit ac45ce8
Show file tree
Hide file tree
Showing 13 changed files with 106 additions and 56 deletions.
6 changes: 6 additions & 0 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,12 @@ APP_PRIVATE_KEY=this-is-not-so-secret-change-it
TRUSTED_REVERSE_PROXIES=1
# =====================================================================================

## Telemetry
# yes/no
TELEMETRY_ENABLED=no
# TELEMETRY_ENDPOINT=http://apm-server:8200
TELEMETRY_ENDPOINT=http://collector:4318

# joystream/node docker image tag
# We do not provide a default value - scripts that startup a joystream-node service
# Should be explicit about what version to use.
Expand Down
17 changes: 9 additions & 8 deletions .github/workflows/joystream-apps-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,14 @@ jobs:
echo "argus_version=$(cat distributor-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
echo "qn_version=$(cat query-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
- name: Make some space
shell: bash
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
Expand Down Expand Up @@ -74,14 +82,7 @@ jobs:
run: |
yarn build:packages
cp docker-compose-no-bind-volumes.yml docker-compose.yml
./tests/network-tests/run-test-node-docker.sh
sleep 15
docker compose up -d db indexer hydra-indexer-gateway processor graphql-server
sleep 30
./tests/network-tests/start-storage.sh
export REUSE_KEYS=true
export SKIP_STORAGE_AND_DISTRIBUTION=true
IGNORE_HIRED_LEADS=true ./tests/network-tests/run-test-scenario.sh content-directory
./tests/network-tests/run-tests.sh content-directory
- name: Push new versions
if: github.event_name == 'workflow_dispatch'
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ The following tools are required for building, testing and contributing to this
- [Rust](https://www.rust-lang.org/tools/install) toolchain - _required_
- [nodejs](https://nodejs.org/) v14.x - _required_
- [yarn classic](https://classic.yarnpkg.com/en/docs/install) package manager v1.22.x- _required_
- [docker](https://www.docker.com/get-started) and docker-compose v.1.29 or higher - _required_
- [docker](https://www.docker.com/get-started) and docker-compose v2.20.x or higher - _required_
- [ansible](https://www.ansible.com/) - _optional_

If you use VSCode as your code editor we recommend using the workspace [settings](devops/vscode/settings.json) for recommend eslint plugin to function properly.
Expand Down
4 changes: 2 additions & 2 deletions colossus.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM --platform=linux/x86-64 node:18 as builder
FROM node:18 as builder

WORKDIR /joystream

Expand Down Expand Up @@ -29,7 +29,7 @@ RUN yarn workspace @joystream/opentelemetry build
RUN yarn workspace storage-node build
RUN yarn cache clean

FROM --platform=linux/x86-64 node:18 as final
FROM node:18 as final
WORKDIR /joystream
COPY --from=builder /joystream /joystream
RUN yarn --frozen-lockfile --production
Expand Down
32 changes: 20 additions & 12 deletions devops/ansible/roles/common/tasks/install-tools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,20 @@
- lsb-release
state: latest

# ref: https://docs.docker.com/engine/install/ubuntu/#uninstall-old-versions
- name: Remove old docker version
become: true
apt:
name:
- docker-compose
- docker.io
- docker-doc
- docker-compose
- podman-docker
- containerd
- runc
state: absent

- name: Add Docker GPG Key and repo
become: true
shell:
Expand All @@ -47,7 +61,7 @@
"
creates: /etc/apt/sources.list.d/docker.list

- name: Update apt and install docker
- name: Install Docker
become: true
apt:
name:
Expand All @@ -58,30 +72,24 @@
state: latest
update_cache: true

- name: Remove old docker-compose
become: true
apt:
name:
- docker-compose
state: absent

- name: Install newer version of docker-compose
- name: Install docker-compose
become: true
shell:
cmd: |
bash -c "
curl -L 'https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)' -o /usr/local/bin/docker-compose
curl -L 'https://github.com/docker/compose/releases/download/v2.20.1/docker-compose-$(uname -s)-$(uname -m)' -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
"
creates: /usr/local/bin/docker-compose

- name: Install Docker Module for Python
- name: Install Docker module for Python
pip:
name: docker

# usermod -aG docker username
- name: Add {{ ansible_user }} to docker group
# To allow non root user to run docker
- name: Add user to docker group
become: true
ansible.builtin.user:
name: "{{ ansible_user }}"
Expand Down
4 changes: 2 additions & 2 deletions distributor-node.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM --platform=linux/x86-64 node:18 as builder
FROM node:18 as builder

WORKDIR /joystream

Expand Down Expand Up @@ -36,7 +36,7 @@ RUN \
yarn workspace @joystream/distributor-cli build &&\
yarn cache clean

FROM --platform=linux/x86-64 node:18 as final
FROM node:18 as final
WORKDIR /joystream
COPY --from=builder /joystream /joystream
RUN yarn --frozen-lockfile --production
Expand Down
34 changes: 23 additions & 11 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ services:
environment:
# ACCOUNT_URI overrides command line arg --accountUri
- ACCOUNT_URI=${COLOSSUS_1_TRANSACTOR_URI}
# - OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200
# - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production
entrypoint: ['yarn']
- OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
- OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production
entrypoint: ['/joystream/entrypoints/storage.sh']
command: [
'start', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data',
'server', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data',
'--sync', '--syncInterval=1',
'--queryNodeEndpoint=${COLOSSUS_QUERY_NODE_URL}',
'--apiUrl=${JOYSTREAM_NODE_WS}',
Expand Down Expand Up @@ -74,14 +74,14 @@ services:
JOYSTREAM_DISTRIBUTOR__DIRECTORIES__ASSETS: /data
JOYSTREAM_DISTRIBUTOR__DIRECTORIES__CACHE_STATE: /cache
JOYSTREAM_DISTRIBUTOR__LOGS__FILE__PATH: /logs
# OTEL_EXPORTER_OTLP_ENDPOINT: http://apm-server:8200
# OTEL_RESOURCE_ATTRIBUTES: service.name=distributor-1,deployment.environment=production
OTEL_EXPORTER_OTLP_ENDPOINT: ${TELEMETRY_ENDPOINT}
OTEL_RESOURCE_ATTRIBUTES: service.name=distributor-1,deployment.environment=production
# JOYSTREAM_DISTRIBUTOR__LOGS__CONSOLE: "off"
# JOYSTREAM_DISTRIBUTOR__LOGS__FILE: "off"
# JOYSTREAM_DISTRIBUTOR__LOGS__ELASTIC: "{\"level\":\"debug\",\"endpoint\":\"http://elasticsearch:9200\"}"
# JOYSTREAM_DISTRIBUTOR__LIMITS__STORAGE: 50G
# JOYSTREAM_DISTRIBUTOR__BUCKETS: "[\"1:0\",\"1:1\"]"
entrypoint: ['yarn']
entrypoint: ['/joystream/entrypoints/distributor.sh']
command: ['start']

colossus-2:
Expand Down Expand Up @@ -200,8 +200,8 @@ services:
- GRAPHQL_PLAYGROUND_ENDPOINT=${GRAPHQL_PLAYGROUND_ENDPOINT}
- GRAPHQL_PLAYGROUND_SUBSCRIPTION_ENDPOINT=${GRAPHQL_PLAYGROUND_SUBSCRIPTION_ENDPOINT}
- GRAPHQL_PLAYGROUND_CDN_URL=${GRAPHQL_PLAYGROUND_CDN_URL}
# - OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200
# - OTEL_RESOURCE_ATTRIBUTES=service.name=query-node,deployment.environment=production
- OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
- OTEL_RESOURCE_ATTRIBUTES=service.name=query-node,deployment.environment=production
ports:
- "${GRAPHQL_SERVER_PORT}:${GRAPHQL_SERVER_PORT}"
- "127.0.0.1:${PROCESSOR_STATE_APP_PORT}:${PROCESSOR_STATE_APP_PORT}"
Expand All @@ -212,7 +212,7 @@ services:
source: .
target: /joystream
working_dir: /joystream
command: ['yarn', 'workspace', 'query-node-root', 'query-node:start:prod']
entrypoint: ['/joystream/entrypoints/graphql-server.sh']

processor:
image: node:14
Expand All @@ -228,14 +228,26 @@ services:
- BATCH_SIZE=100
- QUEUE_FACTOR=1
- QUEUE_MAX_CAP_FACTOR=4
- WARTHOG_DB_HOST=db
- WARTHOG_DB_DATABASE=${DB_NAME}
- WARTHOG_DB_USERNAME=${DB_USER}
- WARTHOG_DB_PASSWORD=${DB_PASS}
- WARTHOG_APP_PORT=4002
depends_on:
- db
volumes:
- type: bind
source: .
target: /joystream
working_dir: /joystream
command: ['yarn', 'workspace', 'query-node-root', 'processor:start']
entrypoint: sh
command: >
-c "
yarn workspace query-node config:dev ;
yarn workspace query-node-root db:prepare ;
yarn workspace query-node-root db:migrate ;
yarn workspace query-node-root processor:start
"
indexer:
image: joystream/hydra-indexer:v5.0.0-alpha.1
Expand Down
8 changes: 8 additions & 0 deletions entrypoints/distributor.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

# docker entrypoint fot distributor node, to allow running with telemetry
if [[ "$TELEMETRY_ENABLED" = "yes" ]] && [[ $1 = "start" ]]; then
node --require @joystream/opentelemetry /joystream/distributor-node/bin/run $*
else
/joystream/distributor-node/bin/run $*
fi
8 changes: 8 additions & 0 deletions entrypoints/graphql-server.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

# docker entrypoint fot graphql-server, to allow running with telemetry
if [[ "$TELEMETRY_ENABLED" = "yes" ]]; then
yarn workspace query-node-root query-node:start:prod:with-instrumentation $*
else
yarn workspace query-node-root query-node:start:prod $*
fi
8 changes: 8 additions & 0 deletions entrypoints/storage.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

# docker entrypoint fot storage node, to allow running with telemetry
if [[ "$TELEMETRY_ENABLED" = "yes" ]] && [[ $1 = "server" ]]; then
node --require @joystream/opentelemetry /joystream/storage-node/bin/run $*
else
/joystream/storage-node/bin/run $*
fi
2 changes: 1 addition & 1 deletion query-node.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM --platform=linux/x86-64 node:18 as builder
FROM node:18 as builder

WORKDIR /joystream
COPY . /joystream
Expand Down
21 changes: 7 additions & 14 deletions query-node/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,17 @@ cd $SCRIPT_PATH

# Bring up db
docker-compose -f ../docker-compose.yml up -d db

# Wait for the db to be up
until docker-compose -f ../docker-compose.yml logs db | grep "server started"; do
echo "Waiting for the db to be ready..."
sleep 1
done

# Make sure we use dev config for db migrations (prevents "Cannot create database..." and some other errors)
docker-compose -f ../docker-compose.yml run --rm --entrypoint sh graphql-server -c "yarn workspace query-node config:dev"
# Migrate the databases
docker-compose -f ../docker-compose.yml run --rm --entrypoint sh graphql-server -c "yarn workspace query-node-root db:prepare"
docker-compose -f ../docker-compose.yml run --rm --entrypoint sh graphql-server -c "yarn workspace query-node-root db:migrate"
echo "Waiting for the db to be ready..."
sleep 5

# Start indexer and gateway
docker-compose -f ../docker-compose.yml up -d indexer
docker-compose -f ../docker-compose.yml up -d hydra-indexer-gateway

# Start processor and graphql server
# Start processor
docker-compose -f ../docker-compose.yml up -d processor
docker-compose -f ../docker-compose.yml up -d graphql-server
echo "Waiting for processor to be ready..." && sleep 30

# Start graphql-server
docker-compose -f ../docker-compose.yml up -d graphql-server
echo "Waiting for graphql-server to be ready..." && sleep 30
16 changes: 11 additions & 5 deletions setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,17 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
# code build tools
sudo apt-get update -y
sudo apt-get install -y coreutils clang llvm jq curl gcc xz-utils sudo pkg-config \
unzip libc6-dev make libssl-dev python3 cmake
# docker
sudo apt-get install -y docker.io containerd runc
# docker-compose
sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
unzip libc6-dev make libssl-dev python3 cmake protobuf-compiler libprotobuf-dev

# Docker: do not replace existing installation to avoid distrupting running containers
if ! command -v docker &> /dev/null
then
# Install Docker from linux distro maintaners
sudo apt-get install -y docker.io containerd runc
fi
# Install latest version of docker-compose
COMPOSE_VERSION=$(curl -sL https://api.github.com/repos/docker/compose/releases/latest | jq -r ".tag_name")
sudo curl -L "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
sudo ln -sf /usr/local/bin/docker-compose /usr/bin/docker-compose
elif [[ "$OSTYPE" == "darwin"* ]]; then
Expand Down

0 comments on commit ac45ce8

Please sign in to comment.