diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0650d36442..ada7f07ae8 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -43,7 +43,7 @@ echo "~~~ Starting bazel remote cache proxy" # Start bazel remote cache proxy for S3 # Note that S3 keys are injected by buildkite, see # https://buildkite.com/docs/pipelines/secrets#storing-secrets-with-the-elastic-ci-stack-for-aws -docker-compose -f .buildkite/hooks/bazel-remote.yml -p bazel_remote up -d +docker compose --compatibility -f .buildkite/hooks/bazel-remote.yml -p bazel_remote up -d echo "~~~ Starting go module proxy" -docker-compose -f .buildkite/hooks/go-module-proxy.yml -p athens up -d +docker compose --compatibility -f .buildkite/hooks/go-module-proxy.yml -p athens up -d diff --git a/BUILD.bazel b/BUILD.bazel index be76b0e6c1..267914fd4b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -245,6 +245,7 @@ updatesrc_update_all( write_source_files( name = "write_all_source_files", additional_update_targets = [ + "//doc/_build/_static/command:write_files", "//doc/command:write_files", ], ) diff --git a/Makefile b/Makefile index 43fe60d304..2cb2419407 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,10 @@ clean: bazel clean rm -f bin/* +scrub: + bazel clean --expunge + rm -f bin/* + bazel: rm -f bin/* bazel build //:scion //:scion-ci diff --git a/acceptance/common/README.md b/acceptance/common/README.md index 0ca1fc3080..c2759d81b7 100644 --- a/acceptance/common/README.md +++ b/acceptance/common/README.md @@ -56,7 +56,7 @@ Tests can use: - `self.artifacts`: the specified directory for test outputs, created during setup. - `self.get_executable()`: returns an executable specified using the `--executable` switch. -- `self.dc`: a wrapper for `docker-compose`, instantiated during `TestTopogen.setup`. +- `self.dc`: a wrapper for `docker compose`, instantiated during `TestTopogen.setup`. The `base.main` function is the main entry point to run the tests and must be invoked in `__main__`. diff --git a/acceptance/common/base.py b/acceptance/common/base.py index dda57be94a..204c35f9ce 100644 --- a/acceptance/common/base.py +++ b/acceptance/common/base.py @@ -203,7 +203,7 @@ def setup_start(self): raise Exception("Failed services.\n" + ps) def teardown(self): - # Avoid running docker-compose teardown if setup_prepare failed + # Avoid running docker compose teardown if setup_prepare failed if self._setup_prepare_failed: return out_dir = self.artifacts / "logs" diff --git a/acceptance/common/docker.py b/acceptance/common/docker.py index b77a7f8b4b..e4a1b7014b 100644 --- a/acceptance/common/docker.py +++ b/acceptance/common/docker.py @@ -51,7 +51,8 @@ def __call__(self, *args, **kwargs) -> str: # Note: not using plumbum here due to complications with encodings in the captured output try: res = subprocess.run( - ["docker-compose", "-f", self.compose_file, "-p", self.project, *args], + ["docker", "compose", "--compatibility", + "-f", self.compose_file, "-p", self.project, *args], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") except subprocess.CalledProcessError as e: raise _CalledProcessErrorWithOutput(e) from None diff --git a/acceptance/common/raw.bzl b/acceptance/common/raw.bzl index 5d8734d7ec..2f024ca8d9 100644 --- a/acceptance/common/raw.bzl +++ b/acceptance/common/raw.bzl @@ -1,6 +1,11 @@ load("//tools/lint:py.bzl", "py_binary", "py_library", "py_test") load("@com_github_scionproto_scion_python_deps//:requirements.bzl", "requirement") +# Bug in bazel: HOME isn't set to TEST_TMPDIR. +# Bug in docker-compose v2.21 a writable HOME is required (eventhough not used). +# Poor design in Bazel, there's no sane way to obtain the path to some +# location that's not a litteral dependency. +# So, HOME must be provided by the invoker. def raw_test( name, src, @@ -8,6 +13,7 @@ def raw_test( deps = [], data = [], tags = [], + homedir = "", local = False): py_library( name = "%s_lib" % name, @@ -63,5 +69,6 @@ def raw_test( "PYTHONUNBUFFERED": "1", # Ensure that unicode output can be printed to the log/console "PYTHONIOENCODING": "utf-8", + "HOME": homedir, }, ) diff --git a/acceptance/common/topogen.bzl b/acceptance/common/topogen.bzl index cc18515f01..08d90df42f 100644 --- a/acceptance/common/topogen.bzl +++ b/acceptance/common/topogen.bzl @@ -1,6 +1,11 @@ load("//tools/lint:py.bzl", "py_binary", "py_library", "py_test") load("@com_github_scionproto_scion_python_deps//:requirements.bzl", "requirement") +# Bug in bazel: HOME isn't set to TEST_TMPDIR. +# Bug in docker-compose v2.21 a writable HOME is required (eventhough not used). +# Poor design in Bazel, there's no sane way to obtain the path to some +# location that's not a litteral dependency. +# So, HOME must be provided by the invoker. def topogen_test( name, src, @@ -10,6 +15,7 @@ def topogen_test( args = [], deps = [], data = [], + homedir = "", tester = "//docker:tester"): """Creates a test based on a topology file. @@ -105,6 +111,7 @@ def topogen_test( "PYTHONUNBUFFERED": "1", # Ensure that unicode output can be printed to the log/console "PYTHONIOENCODING": "utf-8", + "HOME": homedir, }, ) diff --git a/acceptance/router_multi/BUILD.bazel b/acceptance/router_multi/BUILD.bazel index 1809014062..762c404da2 100644 --- a/acceptance/router_multi/BUILD.bazel +++ b/acceptance/router_multi/BUILD.bazel @@ -30,6 +30,7 @@ raw_test( "--bfd", ], data = data, + homedir = "$(rootpath :router)", # This test uses sudo and accesses /var/run/netns. local = True, ) @@ -39,6 +40,7 @@ raw_test( src = "test.py", args = args, data = data, + homedir = "$(rootpath :router)", # This test uses sudo and accesses /var/run/netns. local = True, ) diff --git a/acceptance/sig_short_exp_time/test b/acceptance/sig_short_exp_time/test index 1908cde8dc..10bc25eb2a 100755 --- a/acceptance/sig_short_exp_time/test +++ b/acceptance/sig_short_exp_time/test @@ -55,24 +55,24 @@ run_test() {(set -e docker image load -i acceptance/sig_short_exp_time/sig1.tar docker image load -i acceptance/sig_short_exp_time/sig2.tar - docker-compose -f acceptance/sig_short_exp_time/docker-compose.yml up -d dispatcher1 dispatcher2 sig1 sig2 patha pathb + docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml up -d dispatcher1 dispatcher2 sig1 sig2 patha pathb # Set up forward route on network stack 1 and 2 through the sig tunnel # device. The route is a property of the network stack, and persists after # the container that added it has exited. # # If the route configuration fails, the test is not stopped. - docker-compose -f acceptance/sig_short_exp_time/docker-compose.yml run --name route1 --rm tester1 ip route add 242.254.200.2/32 dev sig || true - docker-compose -f acceptance/sig_short_exp_time/docker-compose.yml run --name route2 --rm tester2 ip route add 242.254.100.2/32 dev sig || true + docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml run --name route1 --rm tester1 ip route add 242.254.200.2/32 dev sig || true + docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml run --name route2 --rm tester2 ip route add 242.254.100.2/32 dev sig || true echo "Start background ping, ping every 0.2 seconds" - docker-compose -f acceptance/sig_short_exp_time/docker-compose.yml run --name tester1 -d tester1 ping -i 0.2 242.254.200.2 + docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml run --name tester1 -d tester1 ping -i 0.2 242.254.200.2 echo "Waiting 10 seconds for path A to expire..." sleep 10 echo "Path A expired, simulating it by shutting down path A proxy" # Traffic should have switched beforehand to path b, and no pings should be lost - docker-compose -f acceptance/sig_short_exp_time/docker-compose.yml stop patha + docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml stop patha sleep 1 docker kill -s SIGINT tester1 @@ -104,9 +104,9 @@ OUTPUT_DIR=$TEST_UNDECLARED_OUTPUTS_DIR mkdir -p $OUTPUT_DIR/logs for CNTR in sig1 sig2 dispatcher1 dispatcher2; do - docker-compose -f acceptance/sig_short_exp_time/docker-compose.yml logs "$CNTR" > "$OUTPUT_DIR/logs/$CNTR.log" + docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml logs "$CNTR" > "$OUTPUT_DIR/logs/$CNTR.log" done -docker-compose -f acceptance/sig_short_exp_time/docker-compose.yml down -v +docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml down -v exit $RC diff --git a/acceptance/topo_cs_reload/reload_test.go b/acceptance/topo_cs_reload/reload_test.go index 790f4a0120..53186c6428 100644 --- a/acceptance/topo_cs_reload/reload_test.go +++ b/acceptance/topo_cs_reload/reload_test.go @@ -108,16 +108,19 @@ func setupTest(t *testing.T) testState { s.mustExec(t, "docker", "image", "load", "-i", "dispatcher.tar") s.mustExec(t, "docker", "image", "load", "-i", "control.tar") // now start the docker containers - s.mustExec(t, "docker-compose", "-f", "docker-compose.yml", "up", "-d") + s.mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml", + "up", "-d") // wait a bit to make sure the containers are ready. time.Sleep(time.Second / 2) t.Log("Test setup done") - s.mustExec(t, "docker-compose", "-f", "docker-compose.yml", "ps") + s.mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml", + "ps") return s } func (s testState) teardownTest(t *testing.T) { - defer s.mustExec(t, "docker-compose", "-f", "docker-compose.yml", "down", "-v") + defer s.mustExec(t, "docker", "compose", "--compatibility", + "-f", "docker-compose.yml", "down", "-v") outdir, exists := os.LookupEnv("TEST_UNDECLARED_OUTPUTS_DIR") require.True(t, exists, "TEST_UNDECLARED_OUTPUTS_DIR must be defined") @@ -127,8 +130,8 @@ func (s testState) teardownTest(t *testing.T) { "topo_cs_reload_dispatcher": "disp.log", "topo_cs_reload_control_srv": "control.log", } { - cmd := exec.Command("docker-compose", "-f", "docker-compose.yml", "logs", "--no-color", - service) + cmd := exec.Command("docker", "compose", "--compatibility", + "-f", "docker-compose.yml", "logs", "--no-color", service) logFileName := fmt.Sprintf("%s/logs/%s", outdir, file) logFile, err := os.Create(logFileName) if err != nil { @@ -146,10 +149,10 @@ func (s testState) teardownTest(t *testing.T) { func (s testState) loadTopo(t *testing.T, name string) { t.Helper() - s.mustExec(t, "docker-compose", "-f", "docker-compose.yml", "exec", "-T", - "topo_cs_reload_control_srv", "mv", name, "/topology.json") - s.mustExec(t, "docker-compose", "-f", "docker-compose.yml", "kill", "-s", "SIGHUP", - "topo_cs_reload_control_srv") + s.mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml", + "exec", "-T", "topo_cs_reload_control_srv", "mv", name, "/topology.json") + s.mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml", + "kill", "-s", "SIGHUP", "topo_cs_reload_control_srv") } func (s testState) mustExec(t *testing.T, name string, arg ...string) { diff --git a/acceptance/topo_daemon_reload/reload_test.go b/acceptance/topo_daemon_reload/reload_test.go index 243cb133aa..ee9b9f44bf 100644 --- a/acceptance/topo_daemon_reload/reload_test.go +++ b/acceptance/topo_daemon_reload/reload_test.go @@ -71,16 +71,18 @@ func setupTest(t *testing.T) { mustExec(t, "docker", "image", "load", "-i", "dispatcher.tar") mustExec(t, "docker", "image", "load", "-i", "daemon.tar") // now start the docker containers - mustExec(t, "docker-compose", "-f", "docker-compose.yml", "up", - "-d", "topo_daemon_reload_dispatcher", "topo_daemon_reload_daemon") + mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml", + "up", "-d", "topo_daemon_reload_dispatcher", "topo_daemon_reload_daemon") // wait a bit to make sure the containers are ready. time.Sleep(time.Second / 2) t.Log("Test setup done") - mustExec(t, "docker-compose", "-f", "docker-compose.yml", "ps") + mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml", + "ps") } func teardownTest(t *testing.T) { - defer mustExec(t, "docker-compose", "-f", "docker-compose.yml", "down", "-v") + defer mustExec(t, "docker", "compose", "--compatibility", + "-f", "docker-compose.yml", "down", "-v") outdir, exists := os.LookupEnv("TEST_UNDECLARED_OUTPUTS_DIR") require.True(t, exists, "TEST_UNDECLARED_OUTPUTS_DIR must be defined") @@ -90,7 +92,8 @@ func teardownTest(t *testing.T) { "topo_daemon_reload_dispatcher": "disp.log", "topo_daemon_reload_daemon": "daemon.log", } { - cmd := exec.Command("docker-compose", "-f", "docker-compose.yml", "logs", "--no-color", + cmd := exec.Command("docker", "compose", "--compatibility", + "-f", "docker-compose.yml", "logs", "--no-color", service) logFileName := fmt.Sprintf("%s/logs/%s", outdir, file) logFile, err := os.Create(logFileName) @@ -108,10 +111,10 @@ func teardownTest(t *testing.T) { func loadTopo(t *testing.T, name string) { t.Helper() - mustExec(t, "docker-compose", "-f", "docker-compose.yml", "exec", "-T", - "topo_daemon_reload_daemon", "mv", name, "/topology.json") - mustExec(t, "docker-compose", "-f", "docker-compose.yml", "kill", "-s", "SIGHUP", - "topo_daemon_reload_daemon") + mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml", + "exec", "-T", "topo_daemon_reload_daemon", "mv", name, "/topology.json") + mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml", + "kill", "-s", "SIGHUP", "topo_daemon_reload_daemon") } func mustExec(t *testing.T, name string, arg ...string) { diff --git a/acceptance/trc_update/BUILD.bazel b/acceptance/trc_update/BUILD.bazel index 4f91062faf..6e8125faf7 100644 --- a/acceptance/trc_update/BUILD.bazel +++ b/acceptance/trc_update/BUILD.bazel @@ -7,5 +7,6 @@ topogen_test( "--executable=end2end_integration:$(location //tools/end2end_integration)", ], data = ["//tools/end2end_integration"], + homedir = "$(rootpath //tools/end2end_integration)", topo = "//topology:tiny4.topo", ) diff --git a/doc/dev/run.rst b/doc/dev/run.rst index 77563c6fc7..0de92eb3cc 100644 --- a/doc/dev/run.rst +++ b/doc/dev/run.rst @@ -14,8 +14,9 @@ Running SCION in this developement setup, is also called running a **local topol The scripts support two different process orchestrators as "backends": - `supervisor `_. This is the default and a bit more light-weight. Packets are sent over the loopback interface. -- `docker-compose `_. Runs individual processes in separate containers connected with docker network bridges. Only this mode supports running a "SCION-IP gateway". +- `docker compose `_. Runs individual processes in separate containers connected with docker network bridges. Only this mode supports running a "SCION-IP gateway". + .. hint:: Before attempting to use the docker compose mode, be sure to build the necessary docker images with ``make docker-images`` .. TODO - Describe configuration directory (referencing manuals) @@ -31,7 +32,7 @@ Quickstart * Build, using ``make`` #. Generate the control-plane PKI keys and certificates, configuration files, and process - orchestrator (supervisor or docker-compose) configuration. + orchestrator (supervisor or docker compose) configuration. .. code-block:: bash @@ -84,6 +85,13 @@ Various helper files are also generated for the benefit of scripts and tooling o for example, ``gen/sciond_addresses.json`` is a simple mapping from AS number to the address of the corresponding :doc:`scion daemon ` instance. +If :option:`scion.sh topology -d` command is used, configuration files are created to +enable running the SCION services in docker containers (see :ref:`docker-section`). Otherwise, +a configuration file is created to enable running the SCION services as plain processes +(see :ref:`supervisor-section`) + +.. _supervisor-section: + supervisor ---------- The ``gen/supervisord.conf`` configuration defines the programs that make up the local topology. @@ -109,16 +117,18 @@ For example:: # and now ping this host from inside AS 1-ff00:0:110, with interactive path prompt bin/scion ping --sciond $(./scion.sh sciond-addr 110) 1-ff00:0:111,127.0.0.1 --interactive +.. _docker-section: + docker ------ -The main docker-compose file is ``gen/scion-dc.yml``. +The main docker compose file is ``gen/scion-dc.yml``. Each SCION service or router runs in a separate container, and the network access of the individual containers is configured to mimick real-world connectivity. There are "tester" containers configured in each AS to mimick end hosts in a SCION AS. These tester containers can be used to run commands accessing the SCION network. -As a shorthand for the somewhat unwieldy ``docker-compose`` invocation, the :file-ref:`tools/dc` +As a shorthand for the somewhat unwieldy ``docker compose`` invocation, the :file-ref:`tools/dc` script can be used. For example:: # show paths from 1-ff00:0:112 to 1-ff00:0:110 @@ -140,7 +150,7 @@ scion.sh .. Note:: The SCION tools and services need to be built **before** running these commands, using - ``make`` or ``make docker-images`` (when using the docker-compose configuration). + ``make`` or ``make docker-images`` (when using the docker compose configuration). The basic usage is ``./scion.sh ``. The main subcommands are: @@ -149,7 +159,7 @@ The basic usage is ``./scion.sh ``. The main subcommands a .. option:: topology Generate the control-plane PKI keys and certificates, configuration files, - and process orchestrator (supervisor or docker-compose) configuration + and process orchestrator (supervisor or docker compose) configuration for a given network topopology defined in a :file-ref:`*.topo configuration file `. @@ -161,7 +171,7 @@ The basic usage is ``./scion.sh ``. The main subcommands a .. option:: -d, --docker - Create a docker-compose configuration (instead of default supervisord). + Create a docker compose configuration (instead of default supervisord). .. option:: --sig @@ -180,6 +190,14 @@ The basic usage is ``./scion.sh ``. The main subcommands a Terminate this run of the local SCION topology. +.. option:: start-monitoring + + Start the monitoring services (jaeger and prometheus). + +.. option:: stop-monitoring + + Stop the monitoring services. + .. option:: sciond-addr Return the address for the scion daemon for the matching ISD-AS by consulting @@ -190,3 +208,15 @@ The basic usage is ``./scion.sh ``. The main subcommands a .. option:: help Describe all available subcommands + +end2end_integration +=================== + +:program:bin/end2end_integration is a basic functional test. + +The basic usage is ``./end2end_integration ``. + +.. option:: -d + + Assume the SCION services are dockerized. Must be consistent with the last + invocation of ``scion.sh topology``. diff --git a/doc/dev/setup.rst b/doc/dev/setup.rst index 53040cec4b..dadeb2d690 100644 --- a/doc/dev/setup.rst +++ b/doc/dev/setup.rst @@ -23,9 +23,10 @@ Prerequisites ``sudo usermod -a -G docker ${LOGNAME}``, where ``${LOGNAME}`` is replaced with your user name. Log out and log back in so that your membership of the ``docker`` group is seen by the shell session. - Optionally install ``docker-compose``. This is needed if you want to run the - ``docker-compose`` based test topology setup instead of the default setup based on ``supervisord``. - Please follow the instructions for `docker-compose `_. + Optionally install the ``Docker Compose Plugin``. This is needed if you want to run the + ``docker compose`` based test topology setup instead of the default setup based on ``supervisord``. + Please follow the instructions for + `Install Compose Plugin `_. Bazel ----- @@ -63,7 +64,7 @@ Bazel build artifacts from bazel. Bazel-remote can manage the disk space and does not infinitely grow like the Bazel built-in disk-cache. To start bazel-remote run:: - ./scion.sh bazel_remote + ./scion.sh bazel-remote #. Build SCION services and tools. diff --git a/scion.sh b/scion.sh index c437e32125..cf851d4f64 100755 --- a/scion.sh +++ b/scion.sh @@ -2,16 +2,17 @@ # BEGIN subcommand functions -cmd_bazel_remote() { +cmd_bazel-remote() { mkdir -p "$HOME/.cache/bazel/remote" uid=$(id -u) gid=$(id -g) - USER_ID="$uid" GROUP_ID="$gid" docker-compose -f bazel-remote.yml -p bazel_remote up -d + USER_ID="$uid" GROUP_ID="$gid" docker compose --compatibility -f bazel-remote.yml -p bazel_remote up -d } -cmd_topo_clean() { +cmd_topo-clean() { set -e - cmd_stop || true + stop_scion || true + cmd_stop-monitoring || true rm -rf traces/* mkdir -p logs traces gen gen-cache gen-certs find gen gen-cache gen-certs -mindepth 1 -maxdepth 1 -exec rm -r {} + @@ -19,7 +20,7 @@ cmd_topo_clean() { cmd_topology() { set -e - cmd_topo_clean + cmd_topo-clean echo "Create topology, configuration, and execution files." tools/topogen.py "$@" @@ -32,11 +33,10 @@ cmd_topodot() { ./tools/topodot.py "$@" } -cmd_run() { - run_jaeger +start_scion() { echo "Running the network..." if is_docker_be; then - docker-compose -f gen/scion-dc.yml -p scion up -d + docker compose --compatibility -f gen/scion-dc.yml -p scion up -d return 0 else run_setup @@ -44,6 +44,13 @@ cmd_run() { fi } +cmd_start() { + start_scion + echo "Note that jaeger is no longer started automatically." + echo "To run jaeger and prometheus, use $PROGRAM start-monitoring." + +} + cmd_sciond-addr() { jq -r 'to_entries | map(select(.key | match("'"$1"'";"i"))) | @@ -51,24 +58,26 @@ cmd_sciond-addr() { "[\(.value)]:30255"' gen/sciond_addresses.json } -run_jaeger() { - if [ ! -f "gen/jaeger-dc.yml" ]; then +cmd_start-monitoring() { + if [ ! -f "gen/monitoring-dc.yml" ]; then return fi - echo "Running jaeger..." - ./tools/quiet ./tools/dc jaeger up -d + echo "Running monitoring..." + echo "Jaeger UI: http://localhost:16686" + echo "Prometheus UI: http://localhost:9090" + ./tools/quiet ./tools/dc monitoring up -d } -stop_jaeger() { - if [ ! -f "gen/jaeger-dc.yml" ]; then +cmd_stop-monitoring() { + if [ ! -f "gen/monitoring-dc.yml" ]; then return fi - echo "Stopping jaeger..." - ./tools/quiet ./tools/dc jaeger down -v + echo "Stopping monitoring..." + ./tools/quiet ./tools/dc monitoring down -v } cmd_mstart() { - # Run with docker-compose or supervisor + # Run with docker compose or supervisor if is_docker_be; then services="$(glob_docker "$@")" [ -z "$services" ] && { echo "ERROR: No process matched for $@!"; exit 255; } @@ -99,7 +108,7 @@ run_teardown() { fi } -cmd_stop() { +stop_scion() { echo "Terminating this run of the SCION infrastructure" if is_docker_be; then ./tools/quiet ./tools/dc down @@ -107,7 +116,12 @@ cmd_stop() { ./tools/quiet tools/supervisor.sh shutdown run_teardown fi - stop_jaeger +} + +cmd_stop() { + stop_scion + echo "Note that jaeger is no longer stopped automatically." + echo "To stop jaeger and prometheus, use $PROGRAM stop-monitoring." } cmd_mstop() { @@ -125,25 +139,23 @@ cmd_status() { } cmd_mstatus() { + rscount=0 + tscount=0 if is_docker_be; then services="$(glob_docker "$@")" [ -z "$services" ] && { echo "ERROR: No process matched for $@!"; exit 255; } - out=$(./tools/dc scion ps $services | tail -n +3) - rscount=$(echo "$out" | grep '\' | wc -l) # Number of running services + rscount=$(./tools/dc scion ps --status=running --format "{{.Name}}" $services | wc -l) tscount=$(echo "$services" | wc -w) # Number of all globed services - echo "$out" | grep -v '\' - [ $rscount -eq $tscount ] + ./tools/dc scion ps -a --format "table {{.Name}}\t{{upper .State}}\tuptime {{.RunningFor}}" $services | sed "s/ ago//" | tail -n+2 else - if [ $# -ne 0 ]; then - services="$(glob_supervisor "$@")" - [ -z "$services" ] && { echo "ERROR: No process matched for $@!"; exit 255; } - tools/supervisor.sh status "$services" | grep -v RUNNING - else - tools/supervisor.sh status | grep -v RUNNING - fi - [ $? -eq 1 ] + services="$(glob_supervisor "$@")" + [ -z "$services" ] && { echo "ERROR: No process matched for $@!"; exit 255; } + rscount=$(./tools/supervisor.sh status "$services" | grep RUNNIN | wc -l) + tscount=$(echo "$services" | wc -w) # Number of all globed services + tools/supervisor.sh status "$services" fi # If all tasks are running, then return 0. Else return 1. + [ $rscount -eq $tscount ] return } @@ -187,45 +199,15 @@ is_docker_be() { [ -f gen/scion-dc.yml ] } -traces_name() { - local name=jaeger_read_badger_traces - echo "$name" -} - -cmd_traces() { - set -e - local trace_dir=${1:-"$(readlink -e .)/traces"} - local port=16687 - local name=$(traces_name) - cmd_stop_traces - docker run -d --name "$name" \ - -u "$(id -u):$(id -g)" \ - -e SPAN_STORAGE_TYPE=badger \ - -e BADGER_EPHEMERAL=false \ - -e BADGER_DIRECTORY_VALUE=/badger/data \ - -e BADGER_DIRECTORY_KEY=/badger/key \ - -v "$trace_dir:/badger" \ - -p "$port":16686 \ - jaegertracing/all-in-one:1.22.0 - sleep 3 - x-www-browser "http://localhost:$port" -} - -cmd_stop_traces() { - local name=$(traces_name) - docker stop "$name" || true - docker rm "$name" || true -} - cmd_help() { - cat <<-_EOF + cat <<-_EOF SCION $PROGRAM runs a SCION network locally for development and testing purposes. Two options for process control systems are supported to run the SCION services. - supervisord (default) - - docker-compose + - docker compose This can be selected when initially creating the configuration with the topology subcommand. @@ -236,15 +218,19 @@ cmd_help() { $PROGRAM run Run network. $PROGRAM mstart PROCESS - Start multiple processes + Start multiple processes. $PROGRAM stop Terminate this run of the SCION infrastructure. $PROGRAM mstop PROCESS - Stop multiple processes + Stop multiple processes. + $PROGRAM start-monitoring + Run the monitoring infrastructure. + $PROGRAM stop-monitoring + Terminate this run of the monitoring infrastructure. $PROGRAM status Show all non-running tasks. $PROGRAM mstatus PROCESS - Show status of provided processes + Show status of provided processes. $PROGRAM sciond-addr ISD-AS Return the address for the scion daemon for the matching ISD-AS by consulting gen/sciond_addresses.json. @@ -254,11 +240,7 @@ cmd_help() { Draw a graphviz graph of a *.topo topology configuration file. $PROGRAM help Show this text. - $PROGRAM traces [folder] - Serve jaeger traces from the specified folder (default: traces/) - $PROGRAM stop_traces - Stop the jaeger container started during the traces command - $PROGRAM bazel_remote + $PROGRAM bazel-remote Starts the bazel remote. _EOF } @@ -269,8 +251,8 @@ COMMAND="$1" shift case "$COMMAND" in - help|run|mstart|mstatus|mstop|stop|status|topology|sciond-addr|traces|stop_traces|topo_clean|topodot|bazel_remote) + help|start|start-monitoring|mstart|mstatus|mstop|stop|stop-monitoring|status|topology|sciond-addr|topo-clean|topodot|bazel-remote) "cmd_$COMMAND" "$@" ;; - start) cmd_run "$@" ;; + run) cmd_start "$@" ;; *) cmd_help; exit 1 ;; esac diff --git a/tools/await-connectivity b/tools/await-connectivity index 63ee037e86..4f0d1f2aee 100755 --- a/tools/await-connectivity +++ b/tools/await-connectivity @@ -1,7 +1,7 @@ #!/bin/bash # This script waits for the beaconing process in a local topology (running -# either with supervisor or docker-compose) to establish full connectivity. +# either with supervisor or docker compose) to establish full connectivity. # Uses the control service's segments/ API to determine whether segments have # been registered. # diff --git a/tools/dc b/tools/dc index 7901082a31..a2912882aa 100755 --- a/tools/dc +++ b/tools/dc @@ -37,8 +37,8 @@ cmd_help() { - [SERVICE]: As scion service glob, e.g. cs1*. - [GROUP]: - scion: For all scion services. - - prom: For the prometheus service. - - [COMMAND]: A docker-compose command like 'up -d' or 'down' + - monitoring: For the monitoring service (i.e. prometheus and yaeger). + - [COMMAND]: A docker compose command like 'up -d' or 'down' - [IA]: An IA number like 1-ff00:0:110 - [LOG_DIR]: A folder. _EOF @@ -70,12 +70,8 @@ cmd_scion() { dc "scion" "$@" } -cmd_jaeger() { - dc "jaeger" "$@" -} - -cmd_prom() { - dc "prom" "$@" +cmd_monitoring() { + dc "monitoring" "$@" } # Runs docker compose for the given project @@ -83,7 +79,7 @@ dc() { local project="$1" local dc_file="gen/$1-dc.yml" shift - COMPOSE_FILE="$dc_file" docker-compose -p "$project" --ansi never "$@" + COMPOSE_FILE="$dc_file" docker compose --compatibility -p "$project" --ansi never "$@" } cmd_collect_logs() { @@ -135,7 +131,7 @@ COMMAND="$1" shift case "$COMMAND" in - start|stop|down|run|scion|jaeger|prom|collect_logs) + start|stop|down|run|scion|monitoring|collect_logs) "cmd_$COMMAND" "$@" ;; exec_tester) "exec_tester" "$@" ;; diff --git a/tools/install_docker b/tools/install_docker deleted file mode 100755 index 384219a5d8..0000000000 --- a/tools/install_docker +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -set -e - -[ $(id -u) -eq 0 ] || { echo "Error: this script should be run as root" && exit 1; } - -# Install prereqs -DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https ca-certificates curl lsb-release software-properties-common - -chksum() { - echo "${1:?} ${2:?}" | sha256sum --status -c - -} - -if [ "$(lsb_release -is)" == "LinuxMint" ]; then - release=$(grep UBUNTU_CODENAME /etc/os-release | cut -f2 -d=) -fi -release=${release:-$(lsb_release -cs)} - -# Add docker apt repo. -if ! grep -Rq "https://download.docker.com/linux/ubuntu" /etc/apt/sources.list.d/; then - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $release stable" - apt-get update -fi - -# Install docker-ce -DEBIAN_FRONTEND=noninteractive apt-get install -y docker-ce docker-ce-cli containerd.io - -# Install docker-compose v2.5.1 -case "$(uname -m)" in - x86_64) - src=https://github.com/docker/compose/releases/download/v2.5.1/docker-compose-linux-x86_64 - sum=d99de1ea7616f2a4c7914d37674f0650660a5e832be555805a71c0fc377233c9 - file=/usr/libexec/docker/cli-plugins/docker-compose - ;; - *) - echo "ERROR: unsupported architecture '$(uname -m)'" - exit 1 -esac -# If the file doesn't exist, or the checksum fails, (re)download it. -if [ ! -e "$file" ] || ! chksum $sum $file; then - curl -sSL "$src" -o "$file" - chksum $sum $file || { echo "Error: $file doesn't match the expected checksum. ($sum)"; exit 1; } -fi -chmod +x "$file" - -# Install docker-compose switch. -curl -fL https://raw.githubusercontent.com/docker/compose-switch/master/install_on_linux.sh | sh diff --git a/tools/integration/cmd.go b/tools/integration/cmd.go index 5b666d781f..e93b920991 100644 --- a/tools/integration/cmd.go +++ b/tools/integration/cmd.go @@ -75,7 +75,7 @@ func Run(ctx context.Context, cfg RunConfig) error { if cfg.Tester != "" { args := append([]string{}, dockerArgs...) args = append(args, cfg.Tester, "sh", "-c", joinCmds(cfg.Commands)) - cmd = exec.CommandContext(ctx, "docker-compose", args...) + cmd = exec.CommandContext(ctx, "docker", args...) log.Debug("Running docker command", "cmd", cmd) } else { cmd = exec.CommandContext(ctx, "sh", "-c", joinCmds(cfg.Commands)) diff --git a/tools/integration/docker.go b/tools/integration/docker.go index e154ab264b..055c391830 100644 --- a/tools/integration/docker.go +++ b/tools/integration/docker.go @@ -27,7 +27,7 @@ import ( ) const ( - dockerCmd = "docker-compose" + dockerCmd = "docker" ) var ( @@ -38,7 +38,8 @@ var ( var dockerArgs []string func initDockerArgs() { - dockerArgs = []string{"-f", GenFile("scion-dc.yml"), "-p", "scion", "exec", "-T", "-e", + dockerArgs = []string{"compose", "--compatibility", + "-f", GenFile("scion-dc.yml"), "-p", "scion", "exec", "-T", "-e", fmt.Sprintf("%s=1", GoIntegrationEnv)} } diff --git a/tools/integration/revocation_test.sh b/tools/integration/revocation_test.sh index 9215c9ed6a..ac78309274 100755 --- a/tools/integration/revocation_test.sh +++ b/tools/integration/revocation_test.sh @@ -27,14 +27,19 @@ done # Bring down routers. echo "Revocation test" -echo "Stopping routers and waiting for ${SLEEP}s." +echo "Stopping routers and waiting for 4s." ./scion.sh mstop $REV_BRS if [ $? -ne 0 ]; then echo "Failed stopping routers." exit 1 fi sleep 4 + +if [ -f gen/scion-dc.yml ]; then + DOCKER_ARGS="-d" +fi + # Do another round of e2e test with retries echo "Testing connectivity between all the hosts (with retries)." -bin/end2end_integration -log.console info -attempts 15 -subset 1-ff00:0:131#2-ff00:0:222 $DOCKER_ARGS +bin/end2end_integration $DOCKER_ARGS -log.console info -attempts 15 -subset 1-ff00:0:131#2-ff00:0:222 exit $? diff --git a/tools/jaeger_ui.sh b/tools/jaeger_ui.sh new file mode 100755 index 0000000000..071d4c2463 --- /dev/null +++ b/tools/jaeger_ui.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# BEGIN subcommand functions + +traces_name() { + echo "jaeger_read_badger_traces" +} + +cmd_start() { + set -e + local trace_dir=${1:-"$(readlink -e .)/traces"} + local port=16687 + local name=$(traces_name) + cmd_stop + docker run -d --name "$name" \ + -u "$(id -u):$(id -g)" \ + -e SPAN_STORAGE_TYPE=badger \ + -e BADGER_EPHEMERAL=false \ + -e BADGER_DIRECTORY_VALUE=/badger/data \ + -e BADGER_DIRECTORY_KEY=/badger/key \ + -v "$trace_dir:/badger" \ + -p "$port":16686 \ + jaegertracing/all-in-one:1.22.0 + sleep 3 + echo "Jaeger UI: http://localhost:$port" +} + +cmd_stop() { + local name=$(traces_name) + docker stop "$name" || true + docker rm "$name" || true +} + +cmd_help() { + cat <<-_EOF + $PROGRAM is a helper script to start and stop a stand-alone jaeger UI. + It does not initiate any jaeger trace collection and shows whatever + traces happen to be in the given directory; not necessarily scion traces. + + If you mean to initiate SCION jaeger traces (and look at them), use + scion.sh start-traces instead. + + Usage: + $PROGRAM start + Serve jaeger traces from the default folder: traces/. + $PROGRAM start [folder] + Serve jaeger traces from the specified folder. + $PROGRAM stop + Stop the jaeger container started with start command. + _EOF +} +# END subcommand functions + +PROGRAM="${0##*/}" +COMMAND="$1" +shift + +case "$COMMAND" in + help|start|stop) + "cmd_$COMMAND" "$@" ;; + *) cmd_help; exit 1 ;; +esac diff --git a/tools/topogen.py b/tools/topogen.py index 209dc75e08..eb57e9e8ab 100755 --- a/tools/topogen.py +++ b/tools/topogen.py @@ -35,7 +35,7 @@ def add_arguments(parser): parser.add_argument('-c', '--topo-config', default=DEFAULT_TOPOLOGY_FILE, help='Path policy file') parser.add_argument('-d', '--docker', action='store_true', - help='Create a docker-compose configuration') + help='Create a docker compose configuration') parser.add_argument('-n', '--network', help='Network to create subnets in (E.g. "127.0.0.0/8"') parser.add_argument('-o', '--output-dir', default=GEN_PATH, diff --git a/tools/topology/config.py b/tools/topology/config.py index 5fcabb91c9..aa70bfbb2e 100644 --- a/tools/topology/config.py +++ b/tools/topology/config.py @@ -38,14 +38,13 @@ from topology.common import ArgsBase from topology.docker import DockerGenArgs, DockerGenerator from topology.go import GoGenArgs, GoGenerator -from topology.jaeger import JaegerGenArgs, JaegerGenerator from topology.net import ( NetworkDescription, IPNetwork, SubnetGenerator, DEFAULT_NETWORK, ) -from topology.prometheus import PrometheusGenArgs, PrometheusGenerator +from topology.monitoring import MonitoringGenArgs, MonitoringGenerator from topology.supervisor import SupervisorGenArgs, SupervisorGenerator from topology.topo import TopoGenArgs, TopoGenerator @@ -113,8 +112,7 @@ def _generate_with_topo(self, topo_dicts): self._generate_docker(topo_dicts) else: self._generate_supervisor(topo_dicts) - self._generate_jaeger(topo_dicts) - self._generate_prom_conf(topo_dicts) + self._generate_monitoring_conf(topo_dicts) self._generate_certs_trcs(topo_dicts) def _generate_certs_trcs(self, topo_dicts): @@ -135,11 +133,6 @@ def _generate_go(self, topo_dicts): def _go_args(self, topo_dicts): return GoGenArgs(self.args, self.topo_config, topo_dicts, self.networks) - def _generate_jaeger(self, topo_dicts): - args = JaegerGenArgs(self.args, topo_dicts) - jaeger_gen = JaegerGenerator(args) - jaeger_gen.generate() - def _generate_topology(self): topo_gen = TopoGenerator(self._topo_args()) return topo_gen.generate() @@ -164,13 +157,13 @@ def _generate_docker(self, topo_dicts): def _docker_args(self, topo_dicts): return DockerGenArgs(self.args, topo_dicts, self.all_networks) - def _generate_prom_conf(self, topo_dicts): - args = self._prometheus_args(topo_dicts) - prom_gen = PrometheusGenerator(args) - prom_gen.generate() + def _generate_monitoring_conf(self, topo_dicts): + args = self._monitoring_args(topo_dicts) + mon_gen = MonitoringGenerator(args) + mon_gen.generate() - def _prometheus_args(self, topo_dicts): - return PrometheusGenArgs(self.args, topo_dicts, self.networks) + def _monitoring_args(self, topo_dicts): + return MonitoringGenArgs(self.args, topo_dicts, self.networks) def _write_ca_files(self, topo_dicts, ca_files): isds = set() diff --git a/tools/topology/go.py b/tools/topology/go.py index 36c82537e2..d8f06f6058 100644 --- a/tools/topology/go.py +++ b/tools/topology/go.py @@ -38,7 +38,7 @@ from topology.net import socket_address_str, NetworkDescription, IPNetwork -from topology.prometheus import ( +from topology.monitoring import ( CS_PROM_PORT, DEFAULT_BR_PROM_PORT, SCIOND_PROM_PORT, diff --git a/tools/topology/jaeger.py b/tools/topology/jaeger.py deleted file mode 100644 index 7355694c55..0000000000 --- a/tools/topology/jaeger.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2019 Anapaya Systems -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import yaml - -from topology.util import write_file -from topology.common import ( - ArgsTopoDicts, -) - -JAEGER_DC = 'jaeger-dc.yml' - - -class JaegerGenArgs(ArgsTopoDicts): - pass - - -class JaegerGenerator(object): - - def __init__(self, args): - self.args = args - output_base = os.environ.get('SCION_OUTPUT_BASE', os.getcwd()) - self.local_jaeger_dir = os.path.join('traces') - self.docker_jaeger_dir = os.path.join(output_base, self.local_jaeger_dir) - - def generate(self): - dc_conf = self._generate_dc() - os.makedirs(os.path.join(self.local_jaeger_dir, 'data'), exist_ok=True) - os.makedirs(os.path.join(self.local_jaeger_dir, 'key'), exist_ok=True) - write_file(os.path.join(self.args.output_dir, JAEGER_DC), - yaml.dump(dc_conf, default_flow_style=False)) - - def _generate_dc(self): - name = 'jaeger' - entry = { - 'version': '2', - 'services': { - 'jaeger': { - 'image': 'jaegertracing/all-in-one:1.22.0', - 'container_name': name, - 'user': '%s:%s' % (str(os.getuid()), str(os.getgid())), - 'ports': [ - '6831:6831/udp', - '16686:16686' - ], - 'environment': [ - 'SPAN_STORAGE_TYPE=badger', - 'BADGER_EPHEMERAL=false', - 'BADGER_DIRECTORY_VALUE=/badger/data', - 'BADGER_DIRECTORY_KEY=/badger/key', - ], - 'volumes': [ - '%s:/badger:rw' % self.docker_jaeger_dir, - ], - } - } - } - return entry diff --git a/tools/topology/prometheus.py b/tools/topology/monitoring.py similarity index 76% rename from tools/topology/prometheus.py rename to tools/topology/monitoring.py index 9a896a64fc..60ac56007e 100644 --- a/tools/topology/prometheus.py +++ b/tools/topology/monitoring.py @@ -1,5 +1,7 @@ # Copyright 2014 ETH Zurich # Copyright 2018 ETH Zurich, Anapaya Systems +# Copyright 2019 Anapaya Systems +# Copyright 2023 SCION Association # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,9 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -:mod:`prometheus` --- SCION topology prometheus generator -============================================= +:mod:`monitoring` --- SCION topology monitoring generator +========================================================= """ + # Stdlib import os from collections import defaultdict @@ -43,17 +46,16 @@ SIG_PROM_PORT = 30456 DISP_PROM_PORT = 30441 DEFAULT_BR_PROM_PORT = 30442 - -PROM_DC_FILE = "prom-dc.yml" +MONITORING_DC_FILE = "monitoring-dc.yml" -class PrometheusGenArgs(ArgsTopoDicts): +class MonitoringGenArgs(ArgsTopoDicts): def __init__(self, args, topo_dicts, networks: Mapping[IPNetwork, NetworkDescription]): super().__init__(args, topo_dicts) self.networks = networks -class PrometheusGenerator(object): +class MonitoringGenerator(object): PROM_DIR = "prometheus" TARGET_FILES = { "BorderRouters": "br.yml", @@ -70,10 +72,12 @@ class PrometheusGenerator(object): def __init__(self, args): """ - :param PrometheusGenArgs args: Contains the passed command line arguments and topo dicts. + :param MonitoringGenArgs args: Contains the passed command line arguments and topo dicts. """ self.args = args self.output_base = os.environ.get('SCION_OUTPUT_BASE', os.getcwd()) + self.local_jaeger_dir = os.path.join('traces') + self.docker_jaeger_dir = os.path.join(self.output_base, self.local_jaeger_dir) def generate(self): config_dict = {} @@ -99,7 +103,12 @@ def generate(self): self._write_dc_file() self._write_disp_file() + # For yeager + os.makedirs(os.path.join(self.local_jaeger_dir, 'data'), exist_ok=True) + os.makedirs(os.path.join(self.local_jaeger_dir, 'key'), exist_ok=True) + def _write_config_files(self, config_dict): + # For Prometheus targets_paths = defaultdict(list) for topo_id, ele_dict in config_dict.items(): base = topo_id.base_dir(self.args.output_dir) @@ -143,26 +152,45 @@ def _write_disp_file(self): if self.args.docker: return targets_path = os.path.join(self.args.output_dir, "dispatcher", - PrometheusGenerator.PROM_DIR, "disp.yml") + self.PROM_DIR, "disp.yml") target_config = [{'targets': [prom_addr_dispatcher(False, None, None, DISP_PROM_PORT, None)]}] write_file(targets_path, yaml.dump(target_config, default_flow_style=False)) def _write_dc_file(self): - name = 'prometheus' - prom_dc = { + # Merged yeager and prometheus files. + name = 'monitoring' + monitoring_dc = { 'version': DOCKER_COMPOSE_CONFIG_VERSION, 'services': { - name: { + 'prometheus': { 'image': 'prom/prometheus:v2.6.0', - 'container_name': name, + 'container_name': name+'prometheus', 'network_mode': 'host', 'volumes': [ self.output_base + '/gen:/prom-config:ro' ], 'command': ['--config.file', '/prom-config/prometheus.yml'], + }, + 'jaeger': { + 'image': 'jaegertracing/all-in-one:1.22.0', + 'container_name': name+'yeager', + 'user': '%s:%s' % (str(os.getuid()), str(os.getgid())), + 'ports': [ + '6831:6831/udp', + '16686:16686' + ], + 'environment': [ + 'SPAN_STORAGE_TYPE=badger', + 'BADGER_EPHEMERAL=false', + 'BADGER_DIRECTORY_VALUE=/badger/data', + 'BADGER_DIRECTORY_KEY=/badger/key', + ], + 'volumes': [ + '%s:/badger:rw' % self.docker_jaeger_dir, + ], } } } - write_file(os.path.join(self.args.output_dir, PROM_DC_FILE), - yaml.dump(prom_dc, default_flow_style=False)) + write_file(os.path.join(self.args.output_dir, MONITORING_DC_FILE), + yaml.dump(monitoring_dc, default_flow_style=False)) diff --git a/tools/topology/sig.py b/tools/topology/sig.py index eb7855c0ae..9ddd2121ef 100644 --- a/tools/topology/sig.py +++ b/tools/topology/sig.py @@ -29,7 +29,7 @@ translate_features, ) from topology.net import socket_address_str -from topology.prometheus import SIG_PROM_PORT +from topology.monitoring import SIG_PROM_PORT class SIGGenArgs(ArgsBase):