diff --git a/.env b/.env index 3c894975c1..13e8c403b8 100644 --- a/.env +++ b/.env @@ -123,6 +123,12 @@ APP_PRIVATE_KEY=this-is-not-so-secret-change-it TRUSTED_REVERSE_PROXIES=1 # ===================================================================================== +## Telemetry +# yes/no +TELEMETRY_ENABLED=no +# TELEMETRY_ENDPOINT=http://apm-server:8200 +TELEMETRY_ENDPOINT=http://collector:4318 + # joystream/node docker image tag # We do not provide a default value - scripts that startup a joystream-node service # Should be explicit about what version to use. diff --git a/.github/workflows/joystream-apps-docker.yml b/.github/workflows/joystream-apps-docker.yml index a8e1427efc..72e0a41d75 100644 --- a/.github/workflows/joystream-apps-docker.yml +++ b/.github/workflows/joystream-apps-docker.yml @@ -24,6 +24,14 @@ jobs: echo "argus_version=$(cat distributor-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT echo "qn_version=$(cat query-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT + - name: Make some space + shell: bash + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Login to DockerHub @@ -74,14 +82,7 @@ jobs: run: | yarn build:packages cp docker-compose-no-bind-volumes.yml docker-compose.yml - ./tests/network-tests/run-test-node-docker.sh - sleep 15 - docker compose up -d db indexer hydra-indexer-gateway processor graphql-server - sleep 30 - ./tests/network-tests/start-storage.sh - export REUSE_KEYS=true - export SKIP_STORAGE_AND_DISTRIBUTION=true - IGNORE_HIRED_LEADS=true ./tests/network-tests/run-test-scenario.sh content-directory + ./tests/network-tests/run-tests.sh content-directory - name: Push new versions if: github.event_name == 'workflow_dispatch' diff --git a/README.md b/README.md index 5866861328..fb4aae8efa 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The following tools are required for building, testing and contributing to this - [Rust](https://www.rust-lang.org/tools/install) toolchain - _required_ - [nodejs](https://nodejs.org/) v14.x - _required_ - [yarn classic](https://classic.yarnpkg.com/en/docs/install) package manager v1.22.x- _required_ -- [docker](https://www.docker.com/get-started) and docker-compose v.1.29 or higher - _required_ +- [docker](https://www.docker.com/get-started) and docker-compose v2.20.x or higher - _required_ - [ansible](https://www.ansible.com/) - _optional_ If you use VSCode as your code editor we recommend using the workspace [settings](devops/vscode/settings.json) for recommend eslint plugin to function properly. diff --git a/colossus.Dockerfile b/colossus.Dockerfile index c9643afcf9..36be93dc91 100644 --- a/colossus.Dockerfile +++ b/colossus.Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/x86-64 node:18 as builder +FROM node:18 as builder WORKDIR /joystream @@ -29,7 +29,7 @@ RUN yarn workspace @joystream/opentelemetry build RUN yarn workspace storage-node build RUN yarn cache clean -FROM --platform=linux/x86-64 node:18 as final +FROM node:18 as final WORKDIR /joystream COPY --from=builder /joystream /joystream RUN yarn --frozen-lockfile --production diff --git a/devops/ansible/roles/common/tasks/install-tools.yml b/devops/ansible/roles/common/tasks/install-tools.yml index dc3d9b2e86..082cdf8036 100644 --- a/devops/ansible/roles/common/tasks/install-tools.yml +++ b/devops/ansible/roles/common/tasks/install-tools.yml @@ -33,6 +33,20 @@ - lsb-release state: latest +# ref: https://docs.docker.com/engine/install/ubuntu/#uninstall-old-versions +- name: Remove old docker version + become: true + apt: + name: + - docker-compose + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + state: absent + - name: Add Docker GPG Key and repo become: true shell: @@ -47,7 +61,7 @@ " creates: /etc/apt/sources.list.d/docker.list -- name: Update apt and install docker +- name: Install Docker become: true apt: name: @@ -58,30 +72,24 @@ state: latest update_cache: true -- name: Remove old docker-compose - become: true - apt: - name: - - docker-compose - state: absent - -- name: Install newer version of docker-compose +- name: Install docker-compose become: true shell: cmd: | bash -c " - curl -L 'https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)' -o /usr/local/bin/docker-compose + curl -L 'https://github.com/docker/compose/releases/download/v2.20.1/docker-compose-$(uname -s)-$(uname -m)' -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose " creates: /usr/local/bin/docker-compose -- name: Install Docker Module for Python +- name: Install Docker module for Python pip: name: docker # usermod -aG docker username -- name: Add {{ ansible_user }} to docker group +# To allow non root user to run docker +- name: Add user to docker group become: true ansible.builtin.user: name: "{{ ansible_user }}" diff --git a/distributor-node.Dockerfile b/distributor-node.Dockerfile index f32ed255b7..90045df25d 100644 --- a/distributor-node.Dockerfile +++ b/distributor-node.Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/x86-64 node:18 as builder +FROM node:18 as builder WORKDIR /joystream @@ -36,7 +36,7 @@ RUN \ yarn workspace @joystream/distributor-cli build &&\ yarn cache clean -FROM --platform=linux/x86-64 node:18 as final +FROM node:18 as final WORKDIR /joystream COPY --from=builder /joystream /joystream RUN yarn --frozen-lockfile --production diff --git a/docker-compose.yml b/docker-compose.yml index 0a166d2c23..9774b2674d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,11 +32,11 @@ services: environment: # ACCOUNT_URI overrides command line arg --accountUri - ACCOUNT_URI=${COLOSSUS_1_TRANSACTOR_URI} - # - OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200 - # - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production - entrypoint: ['yarn'] + - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT} + - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production + entrypoint: ['/joystream/entrypoints/storage.sh'] command: [ - 'start', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data', + 'server', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data', '--sync', '--syncInterval=1', '--queryNodeEndpoint=${COLOSSUS_QUERY_NODE_URL}', '--apiUrl=${JOYSTREAM_NODE_WS}', @@ -74,14 +74,14 @@ services: JOYSTREAM_DISTRIBUTOR__DIRECTORIES__ASSETS: /data JOYSTREAM_DISTRIBUTOR__DIRECTORIES__CACHE_STATE: /cache JOYSTREAM_DISTRIBUTOR__LOGS__FILE__PATH: /logs - # OTEL_EXPORTER_OTLP_ENDPOINT: http://apm-server:8200 - # OTEL_RESOURCE_ATTRIBUTES: service.name=distributor-1,deployment.environment=production + OTEL_EXPORTER_OTLP_ENDPOINT: ${TELEMETRY_ENDPOINT} + OTEL_RESOURCE_ATTRIBUTES: service.name=distributor-1,deployment.environment=production # JOYSTREAM_DISTRIBUTOR__LOGS__CONSOLE: "off" # JOYSTREAM_DISTRIBUTOR__LOGS__FILE: "off" # JOYSTREAM_DISTRIBUTOR__LOGS__ELASTIC: "{\"level\":\"debug\",\"endpoint\":\"http://elasticsearch:9200\"}" # JOYSTREAM_DISTRIBUTOR__LIMITS__STORAGE: 50G # JOYSTREAM_DISTRIBUTOR__BUCKETS: "[\"1:0\",\"1:1\"]" - entrypoint: ['yarn'] + entrypoint: ['/joystream/entrypoints/distributor.sh'] command: ['start'] colossus-2: @@ -200,8 +200,8 @@ services: - GRAPHQL_PLAYGROUND_ENDPOINT=${GRAPHQL_PLAYGROUND_ENDPOINT} - GRAPHQL_PLAYGROUND_SUBSCRIPTION_ENDPOINT=${GRAPHQL_PLAYGROUND_SUBSCRIPTION_ENDPOINT} - GRAPHQL_PLAYGROUND_CDN_URL=${GRAPHQL_PLAYGROUND_CDN_URL} - # - OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200 - # - OTEL_RESOURCE_ATTRIBUTES=service.name=query-node,deployment.environment=production + - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT} + - OTEL_RESOURCE_ATTRIBUTES=service.name=query-node,deployment.environment=production ports: - "${GRAPHQL_SERVER_PORT}:${GRAPHQL_SERVER_PORT}" - "127.0.0.1:${PROCESSOR_STATE_APP_PORT}:${PROCESSOR_STATE_APP_PORT}" @@ -212,7 +212,7 @@ services: source: . target: /joystream working_dir: /joystream - command: ['yarn', 'workspace', 'query-node-root', 'query-node:start:prod'] + entrypoint: ['/joystream/entrypoints/graphql-server.sh'] processor: image: node:14 @@ -228,6 +228,11 @@ services: - BATCH_SIZE=100 - QUEUE_FACTOR=1 - QUEUE_MAX_CAP_FACTOR=4 + - WARTHOG_DB_HOST=db + - WARTHOG_DB_DATABASE=${DB_NAME} + - WARTHOG_DB_USERNAME=${DB_USER} + - WARTHOG_DB_PASSWORD=${DB_PASS} + - WARTHOG_APP_PORT=4002 depends_on: - db volumes: @@ -235,7 +240,14 @@ services: source: . target: /joystream working_dir: /joystream - command: ['yarn', 'workspace', 'query-node-root', 'processor:start'] + entrypoint: sh + command: > + -c " + yarn workspace query-node config:dev ; + yarn workspace query-node-root db:prepare ; + yarn workspace query-node-root db:migrate ; + yarn workspace query-node-root processor:start + " indexer: image: joystream/hydra-indexer:v5.0.0-alpha.1 diff --git a/entrypoints/distributor.sh b/entrypoints/distributor.sh new file mode 100755 index 0000000000..6f3be713f6 --- /dev/null +++ b/entrypoints/distributor.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# docker entrypoint fot distributor node, to allow running with telemetry +if [[ "$TELEMETRY_ENABLED" = "yes" ]] && [[ $1 = "start" ]]; then + node --require @joystream/opentelemetry /joystream/distributor-node/bin/run $* +else + /joystream/distributor-node/bin/run $* +fi diff --git a/entrypoints/graphql-server.sh b/entrypoints/graphql-server.sh new file mode 100755 index 0000000000..4faddd6f80 --- /dev/null +++ b/entrypoints/graphql-server.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# docker entrypoint fot graphql-server, to allow running with telemetry +if [[ "$TELEMETRY_ENABLED" = "yes" ]]; then + yarn workspace query-node-root query-node:start:prod:with-instrumentation $* +else + yarn workspace query-node-root query-node:start:prod $* +fi diff --git a/entrypoints/storage.sh b/entrypoints/storage.sh new file mode 100755 index 0000000000..96dc9660f9 --- /dev/null +++ b/entrypoints/storage.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# docker entrypoint fot storage node, to allow running with telemetry +if [[ "$TELEMETRY_ENABLED" = "yes" ]] && [[ $1 = "server" ]]; then + node --require @joystream/opentelemetry /joystream/storage-node/bin/run $* +else + /joystream/storage-node/bin/run $* +fi diff --git a/query-node.Dockerfile b/query-node.Dockerfile index e4a8f418ed..0228c189aa 100644 --- a/query-node.Dockerfile +++ b/query-node.Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/x86-64 node:18 as builder +FROM node:18 as builder WORKDIR /joystream COPY . /joystream diff --git a/query-node/start.sh b/query-node/start.sh index 4a71ad4d62..63a356938d 100755 --- a/query-node/start.sh +++ b/query-node/start.sh @@ -9,24 +9,17 @@ cd $SCRIPT_PATH # Bring up db docker-compose -f ../docker-compose.yml up -d db - -# Wait for the db to be up -until docker-compose -f ../docker-compose.yml logs db | grep "server started"; do - echo "Waiting for the db to be ready..." - sleep 1 -done - -# Make sure we use dev config for db migrations (prevents "Cannot create database..." and some other errors) -docker-compose -f ../docker-compose.yml run --rm --entrypoint sh graphql-server -c "yarn workspace query-node config:dev" -# Migrate the databases -docker-compose -f ../docker-compose.yml run --rm --entrypoint sh graphql-server -c "yarn workspace query-node-root db:prepare" -docker-compose -f ../docker-compose.yml run --rm --entrypoint sh graphql-server -c "yarn workspace query-node-root db:migrate" +echo "Waiting for the db to be ready..." +sleep 5 # Start indexer and gateway docker-compose -f ../docker-compose.yml up -d indexer docker-compose -f ../docker-compose.yml up -d hydra-indexer-gateway -# Start processor and graphql server +# Start processor docker-compose -f ../docker-compose.yml up -d processor -docker-compose -f ../docker-compose.yml up -d graphql-server +echo "Waiting for processor to be ready..." && sleep 30 +# Start graphql-server +docker-compose -f ../docker-compose.yml up -d graphql-server +echo "Waiting for graphql-server to be ready..." && sleep 30 diff --git a/setup.sh b/setup.sh index 71be02d3b7..88540d6409 100755 --- a/setup.sh +++ b/setup.sh @@ -8,11 +8,17 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then # code build tools sudo apt-get update -y sudo apt-get install -y coreutils clang llvm jq curl gcc xz-utils sudo pkg-config \ - unzip libc6-dev make libssl-dev python3 cmake - # docker - sudo apt-get install -y docker.io containerd runc - # docker-compose - sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + unzip libc6-dev make libssl-dev python3 cmake protobuf-compiler libprotobuf-dev + + # Docker: do not replace existing installation to avoid distrupting running containers + if ! command -v docker &> /dev/null + then + # Install Docker from linux distro maintaners + sudo apt-get install -y docker.io containerd runc + fi + # Install latest version of docker-compose + COMPOSE_VERSION=$(curl -sL https://api.github.com/repos/docker/compose/releases/latest | jq -r ".tag_name") + sudo curl -L "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose sudo ln -sf /usr/local/bin/docker-compose /usr/bin/docker-compose elif [[ "$OSTYPE" == "darwin"* ]]; then