we put more docker in your docker so...you can test your network

docker/Dockerfile.synthetic_network

@@ -0,0 +1,14 @@
+FROM syntheticnet:latest as synthnet
+
+FROM aleph-node:latest
+
+# Linux networking tools and node.js
+RUN apt update && \
+    apt install nodejs curl iproute2 iputils-ping net-tools netwox tcpdump gdb gdbserver stress -y
+
+COPY --from=synthnet /opt/lib/ /opt/lib/
+WORKDIR /opt/lib
+ENTRYPOINT []
+ENV ENTRY="/node/docker_entrypoint.sh"
+CMD ["/opt/lib/setup.sh"]
+

docker/docker-compose.synthetic-network.yml

@@ -0,0 +1,96 @@
+services:
+  Node0:
+    extends:
+      file: docker-compose.base.yml
+      service: Node0
+    image: aleph-node:syntheticnet
+    networks:
+      - synthetic-network
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+      - SYS_PTRACE
+    environment:
+      - SYNTHETIC_NETWORK=10.77.0.0/16
+      - PUBLIC_VALIDATOR_ADDRESS=Node0:30343
+    ports:
+      - 3000:80
+
+  Node1:
+    extends:
+      file: docker-compose.base.yml
+      service: Node1
+    image: aleph-node:syntheticnet
+    networks:
+      - synthetic-network
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+      - SYS_PTRACE
+    environment:
+      - SYNTHETIC_NETWORK=10.77.0.0/16
+      - PUBLIC_VALIDATOR_ADDRESS=Node1:30344
+      - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID
+    ports:
+      - 3001:80
+
+  Node2:
+    extends:
+      file: docker-compose.base.yml
+      service: Node2
+    image: aleph-node:syntheticnet
+    networks:
+      - synthetic-network
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+      - SYS_PTRACE
+    environment:
+      - SYNTHETIC_NETWORK=10.77.0.0/16
+      - PUBLIC_VALIDATOR_ADDRESS=Node2:30345
+      - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID
+    ports:
+      - 3002:80
+
+  Node3:
+    extends:
+      file: docker-compose.base.yml
+      service: Node3
+    image: aleph-node:syntheticnet
+    networks:
+      - synthetic-network
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+      - SYS_PTRACE
+    environment:
+      - SYNTHETIC_NETWORK=10.77.0.0/16
+      - PUBLIC_VALIDATOR_ADDRESS=Node3:30346
+      - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID
+    ports:
+      - 3003:80
+
+  Node4:
+    extends:
+      file: docker-compose.base.yml
+      service: Node4
+    image: aleph-node:syntheticnet
+    networks:
+      - synthetic-network
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+      - SYS_PTRACE
+    environment:
+      - SYNTHETIC_NETWORK=10.77.0.0/16
+      - PUBLIC_VALIDATOR_ADDRESS=Node4:30347
+      - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID
+    ports:
+      - 3004:80
+
+networks:
+  synthetic-network:
+    ipam:
+      config:
+        - subnet: 10.77.0.0/16
+

scripts/common.sh

@@ -103,7 +103,7 @@ function get_last_block() {
     local validator=$1
     local rpc_port=$2
 
-    local last_block=0
+    local last_block=""
     while [[ -z "$last_block" ]]; do
         last_block=$(retrieve_last_block $validator $rpc_port)
         sleep 1

scripts/synthetic-network/README.md

@@ -0,0 +1,21 @@
+# synthetic-network
+
+This folder contains various scripts that allows to spawn and interact with `aleph-node` executed within a so called
+synthetic-network. synthetic-network is a tool for docker that allows you to simulate different network conditions, like
+variable latency, rate limit, etc. Easiest way to manage parameters of a synthetic-network is to use its web-ui - after
+executing `run_consensus_synthetic-network.sh` it should be available at http://localhost:3000 (each node has separate settings
+page, i.e. :3001 is Node1, ...).
+
+# Content of this folder
+
+Main file in this folder is `run_consensus_synthetic-network.sh`. It builds a docker-image containing `aleph-node` and some
+arbitrary set of networking and debugging tools. It also consist of files required to spawn an instance of the
+synthetic-network. Its requirements are: docker, docker-compose, git, `aleph-node:latest` docker-image.
+
+`set_defaults_synthetic-network.sh` allows you to reset settings of the synthetic-network to some sane defaults. You might need
+to use it when you set too restrictive values for some of its parameters, i.e. rate limit that make you unable to further
+interact with its web-ui.
+
+Additionally, this folder contains an example .js script introducing API of the synthetic-network. You can use it by executing
+`run_script_for_synthetic-network.sh --script-path ./latency.js`.
+

scripts/synthetic-network/build_synthetic-network.sh

@@ -0,0 +1,35 @@
+#!/bin/env bash
+
+set -euo pipefail
+
+source ./scripts/common.sh
+
+GIT_COMMIT=${GIT_COMMIT:-72bbb4fde915e4132c19cd7ce3605364abac58a5}
+
+TMPDIR="$(dirname $0)/vendor"
+mkdir -p $TMPDIR
+log "created a temporary folder at $TMPDIR"
+
+pushd .
+
+log "cloning synthetic-network's git repo"
+cd $TMPDIR
+if [[ ! -d ./synthetic-network ]]; then
+    git clone https://github.com/daily-co/synthetic-network.git
+fi
+cd synthetic-network
+git fetch origin
+git checkout $GIT_COMMIT
+
+log "building base docker image for synthetic-network with support for synthetic-network"
+log "patching synthetic network"
+# aleph-node crashes since it uses newer glibc than this image
+sed -i 's/FROM node:12.20.2/FROM node:19.2/' Dockerfile
+docker build -t syntheticnet .
+
+popd
+
+log "building docker image for aleph-node that supports synthetic-network"
+docker build -t aleph-node:syntheticnet -f docker/Dockerfile.synthetic_network .
+
+exit 0

scripts/synthetic-network/latency.js

@@ -0,0 +1,22 @@
+const argv = require('node:process').argv;
+const inLatency = argv.length <= 2 ? 0 : argv.at(2);
+const outLatency = argv.length <= 3 ? 0 : argv.at(3);
+console.log("setting in-latency to", inLatency);
+console.log("setting out-latency to", outLatency);
+
+const SyntheticNetwork = require('../vendor/synthetic-network/frontend');
+
+async function setLatency(host, port, inLatency, outLatency) {
+    const synthnet = new SyntheticNetwork({ hostname: host, port: port });
+    synthnet.default_link.egress.latency(outLatency);
+    synthnet.default_link.ingress.latency(inLatency);
+    await synthnet.commit();
+}
+
+async function run(inLatency, outLatency) {
+    for (let it = 0; it < 5; it++) {
+        await setLatency('localhost', 3000 + it, inLatency, outLatency);
+    }
+}
+
+run(inLatency, outLatency);

scripts/synthetic-network/run_consensus_synthetic-network.sh

@@ -0,0 +1,65 @@
+#!/bin/env bash
+
+set -euo pipefail
+
+source ./scripts/common.sh
+
+function usage(){
+    cat << EOF
+Usage:
+  $0
+  This script allows you to run aleph-node within docker and simulate some custom network conditions, e.g. delays, rate limit,
+  package loss. Additionally, each node is preinstalled with the 'stress' tool, that allows to simulate high occupancy of nodes
+  cpu and io. It should allow us test more realistic high volume network conditions without the need to spawn hundreds of
+  aws instances. For more details on networking part of this solution, visit https://github.com/daily-co/synthetic-network .
+  IMPORTANT: this script requires aleph-node:latest docker image.
+    --no-build-image
+        skip docker image build
+    --commit 72bbb4fde915e4132c19cd7ce3605364abac58a5
+        commit hash used to build synthetic-network, default is 72bbb4fde915e4132c19cd7ce3605364abac58a5
+EOF
+    exit 0
+}
+
+function build_test_image() {
+    local commit=$1
+    local path=$2
+
+    GIT_COMMIT=$commit ${path}/build_synthetic-network.sh
+}
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --no-build-image)
+            BUILD_IMAGE=false
+            shift
+            ;;
+        --commit)
+            GIT_COMMIT="$2"
+            shift;shift
+            ;;
+        --help)
+            usage
+            shift
+            ;;
+        *)
+            error "Unrecognized argument $1!"
+            ;;
+    esac
+done
+
+BUILD_IMAGE=${BUILD_IMAGE:-true}
+GIT_COMMIT=${GIT_COMMIT:-72bbb4fde915e4132c19cd7ce3605364abac58a5}
+
+if [[ "$BUILD_IMAGE" = true ]]; then
+    log "building custom docker image for synthetic-network tests"
+    path=$(dirname $0)
+    build_test_image $GIT_COMMIT $path
+fi
+
+log "running synthetic-network"
+DOCKER_COMPOSE=./docker/docker-compose.synthetic-network.yml ./.github/scripts/run_consensus.sh
+log "open a web browser at http://localhost:3000 (port 3000 is Node0, 3001 is Node1, ...)"
+xdg-open http://localhost:3000
+
+exit 0

scripts/synthetic-network/run_script_for_synthetic-network.sh

@@ -0,0 +1,65 @@
+#!/bin/env bash
+
+set -euo pipefail
+
+source ./scripts/common.sh
+
+function usage(){
+    cat << EOF
+Usage:
+  $0
+  This script allows you to run a custom .js script using the synthetic-network network simulation tool.
+  IMPORTANT: first you need to call 'scripts/run_consensus_synthetic-network.sh' and let it run in background.
+             It spawns docker-compose configured with synthetic-network.
+             It requires node.js to run.
+    --commit 72bbb4fde915e4132c19cd7ce3605364abac58a5
+        commit hash used to build synthetic-network, default is 72bbb4fde915e4132c19cd7ce3605364abac58a5
+    --script-path scripts/vendor/synthetic-network/frontend/udp_rate_sine_demo.js
+        path to a synthetic-network scrypt. Default is a demo scripts/vendor/synthetic-network/frontend/udp_rate_sine_demo.js
+        from the synthetic-network repo. Please consult synthetic-network repo for details: https://github.com/daily-co/synthetic-network
+EOF
+    exit 0
+}
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --commit)
+            GIT_COMMIT="$2"
+            shift;shift
+            ;;
+        --script-path)
+            SCRIPT_PATH="$2"
+            shift;shift
+            ;;
+        --help)
+            usage
+            shift
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+GIT_COMMIT=${GIT_COMMIT:-72bbb4fde915e4132c19cd7ce3605364abac58a5}
+SCRIPT_PATH=${SCRIPT_PATH:-scripts/vendor/synthetic-network/frontend/udp_rate_sine_demo.js}
+SCRIPT_PATH=$(realpath $SCRIPT_PATH)
+
+TMPDIR="$(dirname $0)/vendor"
+mkdir -p $TMPDIR
+log "created a temporary folder at $TMPDIR"
+
+log "cloning synthetic-network's git repo"
+cd $TMPDIR
+if [[ ! -d ./synthetic-network ]]; then
+    git clone https://github.com/daily-co/synthetic-network.git
+fi
+cd synthetic-network
+git fetch origin
+git checkout $GIT_COMMIT
+cd frontend
+
+log "running .js script"
+node $SCRIPT_PATH ${@:1}
+
+exit 0