From 8968fa8cd9f34633da3708ee8d0a1bead6ba5f5d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 8 Jul 2021 17:28:03 +0200 Subject: [PATCH 01/13] migration service starts an http server --- services/migration/Dockerfile | 6 ++++-- services/migration/README.md | 6 +++++- services/migration/docker/entrypoint.sh | 4 ++-- services/migration/src/server.py | 21 +++++++++++++++++++++ 4 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 services/migration/src/server.py diff --git a/services/migration/Dockerfile b/services/migration/Dockerfile index 68b9749625a..180a0384990 100644 --- a/services/migration/Dockerfile +++ b/services/migration/Dockerfile @@ -33,8 +33,8 @@ FROM base as build RUN apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - git \ + build-essential \ + git \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -52,6 +52,7 @@ WORKDIR /build # install only base 3rd party dependencies COPY --chown=scu:scu packages/postgres-database/ . RUN pip --no-cache-dir --quiet install .[migration] +RUN pip --no-cache-dir install fastapi uvicorn @@ -74,5 +75,6 @@ USER ${SC_USER_NAME} # bring installed package without build tools COPY --from=build ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY --chown=scu:scu services/migration/docker services/migration/docker +COPY --chown=scu:scu services/migration/src services/migration/src ENTRYPOINT [ "/bin/sh", "services/migration/docker/entrypoint.sh" ] diff --git a/services/migration/README.md b/services/migration/README.md index c0bb902936a..21762edb7df 100644 --- a/services/migration/README.md +++ b/services/migration/README.md @@ -1,4 +1,8 @@ # migration service This service starts in the swarm, discovers postgres service and upgrades the database -to the latest version of of the schema +to the latest version of of the schema. + +While postgres service is not available, the process is continuously restarted. +Once a connection to postgress is the migration is applied. If the process succeeds +it will expose on port 8000 an http replying in plain text to `GET migration:8000/`. diff --git a/services/migration/docker/entrypoint.sh b/services/migration/docker/entrypoint.sh index ac3ec14240f..7e3b0fdb417 100755 --- a/services/migration/docker/entrypoint.sh +++ b/services/migration/docker/entrypoint.sh @@ -19,5 +19,5 @@ echo "$INFO local dir : $(ls -al)" echo "$INFO Starting migration ..." sc-pg upgrade-and-close -echo "$INFO Migration Done. Wait forever ..." -exec tail -f /dev/null +echo "$INFO Migration Done. Expose http server to allow other services in the stack to start ..." +uvicorn services.migration.src.server:app diff --git a/services/migration/src/server.py b/services/migration/src/server.py new file mode 100644 index 00000000000..308385e62e6 --- /dev/null +++ b/services/migration/src/server.py @@ -0,0 +1,21 @@ +from fastapi import FastAPI +from fastapi.responses import PlainTextResponse + +MIGRATION_OK: str = "42" + +app = FastAPI() + + +@app.get( + "/", + include_in_schema=False, + response_class=PlainTextResponse, + description=( + "Empty route used by entrypoint.sh scripts in " + "docker images to check if service is available." + "when this service is ready it means Postgres is " + "available and all migrations have finished." + ), +) +async def can_start_other_services_in_stack() -> str: + return MIGRATION_OK From 3d58640ddccc6fc1dbbc5b5de3051e612a2c25d3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 9 Jul 2021 11:46:34 +0200 Subject: [PATCH 02/13] added wait-for script --- scripts/common-docker-boot/README.md | 3 + scripts/common-docker-boot/wait-for | 184 ++++++++++++++++++ tests/environment-setup/test_used_wait_for.py | 36 ++++ 3 files changed, 223 insertions(+) create mode 100644 scripts/common-docker-boot/README.md create mode 100644 scripts/common-docker-boot/wait-for create mode 100644 tests/environment-setup/test_used_wait_for.py diff --git a/scripts/common-docker-boot/README.md b/scripts/common-docker-boot/README.md new file mode 100644 index 00000000000..395a61e3e92 --- /dev/null +++ b/scripts/common-docker-boot/README.md @@ -0,0 +1,3 @@ +# wait-for script + +Used by almost all the services. Update it from the repo's official [release page](https://github.com/eficode/wait-for/releases). diff --git a/scripts/common-docker-boot/wait-for b/scripts/common-docker-boot/wait-for new file mode 100644 index 00000000000..b0eb09b1d84 --- /dev/null +++ b/scripts/common-docker-boot/wait-for @@ -0,0 +1,184 @@ +#!/bin/sh + +# The MIT License (MIT) +# +# Copyright (c) 2017 Eficode Oy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -- "$@" -- "$TIMEOUT" "$QUIET" "$PROTOCOL" "$HOST" "$PORT" "$result" +TIMEOUT=15 +QUIET=0 +# The protocol to make the request with, either "tcp" or "http" +PROTOCOL="tcp" + +echoerr() { + if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi +} + +usage() { + exitcode="$1" + cat << USAGE >&2 +Usage: + $0 host:port|url [-t timeout] [-- command args] + -q | --quiet Do not output any status messages + -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit "$exitcode" +} + +wait_for() { + case "$PROTOCOL" in + tcp) + if ! command -v nc >/dev/null; then + echoerr 'nc command is missing!' + exit 1 + fi + ;; + wget) + if ! command -v wget >/dev/null; then + echoerr 'nc command is missing!' + exit 1 + fi + ;; + esac + + while :; do + case "$PROTOCOL" in + tcp) + nc -w 1 -z "$HOST" "$PORT" > /dev/null 2>&1 + ;; + http) + wget --timeout=1 -q "$HOST" -O /dev/null > /dev/null 2>&1 + ;; + *) + echoerr "Unknown protocol '$PROTOCOL'" + exit 1 + ;; + esac + + result=$? + + if [ $result -eq 0 ] ; then + if [ $# -gt 7 ] ; then + for result in $(seq $(($# - 7))); do + result=$1 + shift + set -- "$@" "$result" + done + + TIMEOUT=$2 QUIET=$3 PROTOCOL=$4 HOST=$5 PORT=$6 result=$7 + shift 7 + exec "$@" + fi + exit 0 + fi + + if [ "$TIMEOUT" -le 0 ]; then + break + fi + TIMEOUT=$((TIMEOUT - 1)) + + sleep 1 + done + echo "Operation timed out" >&2 + exit 1 +} + +while :; do + case "$1" in + http://*|https://*) + HOST="$1" + PROTOCOL="http" + shift 1 + ;; + *:* ) + HOST=$(printf "%s\n" "$1"| cut -d : -f 1) + PORT=$(printf "%s\n" "$1"| cut -d : -f 2) + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -q-*) + QUIET=0 + echoerr "Unknown option: $1" + usage 1 + ;; + -q*) + QUIET=1 + result=$1 + shift 1 + set -- -"${result#-q}" "$@" + ;; + -t | --timeout) + TIMEOUT="$2" + shift 2 + ;; + -t*) + TIMEOUT="${1#-t}" + shift 1 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + break + ;; + --help) + usage 0 + ;; + -*) + QUIET=0 + echoerr "Unknown option: $1" + usage 1 + ;; + *) + QUIET=0 + echoerr "Unknown argument: $1" + usage 1 + ;; + esac +done + +if ! [ "$TIMEOUT" -ge 0 ] 2>/dev/null; then + echoerr "Error: invalid timeout '$TIMEOUT'" + usage 3 +fi + +case "$PROTOCOL" in + tcp) + if [ "$HOST" = "" ] || [ "$PORT" = "" ]; then + echoerr "Error: you need to provide a host and port to test." + usage 2 + fi + ;; + http) + if [ "$HOST" = "" ]; then + echoerr "Error: you need to provide a host to test." + usage 2 + fi + ;; +esac + +wait_for "$@" diff --git a/tests/environment-setup/test_used_wait_for.py b/tests/environment-setup/test_used_wait_for.py new file mode 100644 index 00000000000..96369e68b33 --- /dev/null +++ b/tests/environment-setup/test_used_wait_for.py @@ -0,0 +1,36 @@ +# pylint: disable=redefined-outer-name + +import hashlib +import sys +from pathlib import Path + +import pytest + +CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).parent.resolve() + +ASSERT_ERROR_MESSAGE = ( + "Looks like thewait-for script changed. Ensure the code is taken from" + "https://github.com/eficode/wait-for/releases" +) + + +@pytest.fixture +def wait_for_path() -> Path: + return CURRENT_DIR / ".." / ".." / "scripts" / "common-docker-boot" / "wait-for" + + +@pytest.fixture +def sha256sum() -> str: + # NOTICE: only change this value if you have updated the script + # compute this on a linux box with (macos will produce a different value) + # from this directory run: + # sha256sum ../../scripts/common-docker-boot/wait-for + # current version v2.1.2 + return "a444f069a25a333b375cc835895e39f44606040701a622f3c0abb2fc62d39ebf" + + +def test_wait_for_did_not_change(sha256sum: str, wait_for_path: Path) -> None: + with open(wait_for_path, "rb") as f: + file_content = f.read() + readable_hash = hashlib.sha256(file_content).hexdigest() + assert readable_hash == sha256sum, ASSERT_ERROR_MESSAGE From 6b3f0912f370d3cbe49c9b89e36eac234854162f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 9 Jul 2021 11:46:48 +0200 Subject: [PATCH 03/13] exposed service on all ports --- services/migration/docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/migration/docker/entrypoint.sh b/services/migration/docker/entrypoint.sh index 7e3b0fdb417..67050d50945 100755 --- a/services/migration/docker/entrypoint.sh +++ b/services/migration/docker/entrypoint.sh @@ -20,4 +20,4 @@ echo "$INFO Starting migration ..." sc-pg upgrade-and-close echo "$INFO Migration Done. Expose http server to allow other services in the stack to start ..." -uvicorn services.migration.src.server:app +uvicorn services.migration.src.server:app --host 0.0.0.0 From 0c80caddf7c9b36458733aace942f7f2b0d7c99d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 9 Jul 2021 11:47:33 +0200 Subject: [PATCH 04/13] added wait-for in docker files --- services/api-server/Dockerfile | 73 +++++++++++++++++++------------- services/catalog/Dockerfile | 74 +++++++++++++++++++-------------- services/director-v2/Dockerfile | 17 +++++++- services/director/Dockerfile | 18 +++++++- services/sidecar/Dockerfile | 17 +++++++- services/storage/Dockerfile | 17 +++++++- services/web/Dockerfile | 12 +++++- 7 files changed, 157 insertions(+), 71 deletions(-) diff --git a/services/api-server/Dockerfile b/services/api-server/Dockerfile index 2df1eb9e487..0cf90931f75 100644 --- a/services/api-server/Dockerfile +++ b/services/api-server/Dockerfile @@ -11,25 +11,25 @@ FROM python:${PYTHON_VERSION}-slim-buster as base LABEL maintainer=pcrespov RUN set -eux; \ - apt-get update; \ - apt-get install -y gosu; \ - rm -rf /var/lib/apt/lists/*; \ - # verify that the binary works - gosu nobody true + apt-get update; \ + apt-get install -y gosu; \ + rm -rf /var/lib/apt/lists/*; \ + # verify that the binary works + gosu nobody true # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) ENV SC_USER_ID=8004 \ - SC_USER_NAME=scu \ - SC_BUILD_TARGET=base \ - SC_BOOT_MODE=default + SC_USER_NAME=scu \ + SC_BUILD_TARGET=base \ + SC_BOOT_MODE=default RUN adduser \ - --uid ${SC_USER_ID} \ - --disabled-password \ - --gecos "" \ - --shell /bin/sh \ - --home /home/${SC_USER_NAME} \ - ${SC_USER_NAME} + --uid ${SC_USER_ID} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} # Sets utf-8 encoding for Python et al @@ -37,7 +37,7 @@ ENV LANG=C.UTF-8 # Turns off writing .pyc files; superfluous on an ephemeral container. ENV PYTHONDONTWRITEBYTECODE=1 \ - VIRTUAL_ENV=/home/scu/.venv + VIRTUAL_ENV=/home/scu/.venv # Ensures that the python and pip executables used in the image will be # those from our virtualenv. @@ -56,10 +56,11 @@ FROM base as build ENV SC_BUILD_TARGET=build RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && apt-get install -y --no-install-recommends \ + build-essential \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* # NOTE: python virtualenv is used here such that installed @@ -67,9 +68,9 @@ RUN apt-get update \ RUN python -m venv "${VIRTUAL_ENV}" RUN pip install --no-cache-dir --upgrade \ - pip~=21.0.1 \ - wheel \ - setuptools + pip~=21.0.1 \ + wheel \ + setuptools WORKDIR /build @@ -96,7 +97,7 @@ COPY --chown=scu:scu services/storage/client-sdk /build/services/storage/client- WORKDIR /build/services/api-server RUN pip --no-cache-dir install -r requirements/prod.txt &&\ - pip --no-cache-dir list -v + pip --no-cache-dir list -v # --------------------------Production stage ------------------- @@ -109,7 +110,7 @@ RUN pip --no-cache-dir install -r requirements/prod.txt &&\ FROM base as production ENV SC_BUILD_TARGET=production \ - SC_BOOT_MODE=production + SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE @@ -118,18 +119,27 @@ WORKDIR /home/scu # Starting from clean base image, copies pre-installed virtualenv from prod-only-deps COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} +# wget required by wait-for +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + # Copies booting scripts COPY --chown=scu:scu services/api-server/docker services/api-server/docker RUN chmod +x services/api-server/docker/*.sh +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for HEALTHCHECK --interval=30s \ - --timeout=20s \ - --start-period=30s \ - --retries=3 \ - CMD ["python3", "services/api-server/docker/healthcheck.py", "http://localhost:8000/"] + --timeout=20s \ + --start-period=30s \ + --retries=3 \ + CMD ["python3", "services/api-server/docker/healthcheck.py", "http://localhost:8000/"] ENTRYPOINT [ "/bin/sh", "services/api-server/docker/entrypoint.sh" ] -CMD ["/bin/sh", "services/api-server/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/api-server/docker/boot.sh"] # --------------------------Development stage ------------------- @@ -148,5 +158,8 @@ WORKDIR /devel RUN chown -R scu:scu "${VIRTUAL_ENV}" +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for + ENTRYPOINT ["/bin/sh", "services/api-server/docker/entrypoint.sh"] -CMD ["/bin/sh", "services/api-server/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/api-server/docker/boot.sh"] diff --git a/services/catalog/Dockerfile b/services/catalog/Dockerfile index 98ae84cb26a..38ab5090a04 100644 --- a/services/catalog/Dockerfile +++ b/services/catalog/Dockerfile @@ -12,25 +12,25 @@ FROM python:${PYTHON_VERSION}-slim-buster as base LABEL maintainer=pcrespov RUN set -eux; \ - apt-get update; \ - apt-get install -y gosu; \ - rm -rf /var/lib/apt/lists/*; \ - # verify that the binary works - gosu nobody true + apt-get update; \ + apt-get install -y gosu; \ + rm -rf /var/lib/apt/lists/*; \ + # verify that the binary works + gosu nobody true # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) ENV SC_USER_ID=8004 \ - SC_USER_NAME=scu \ - SC_BUILD_TARGET=base \ - SC_BOOT_MODE=default + SC_USER_NAME=scu \ + SC_BUILD_TARGET=base \ + SC_BOOT_MODE=default RUN adduser \ - --uid ${SC_USER_ID} \ - --disabled-password \ - --gecos "" \ - --shell /bin/sh \ - --home /home/${SC_USER_NAME} \ - ${SC_USER_NAME} + --uid ${SC_USER_ID} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} # Sets utf-8 encoding for Python et al @@ -38,7 +38,7 @@ ENV LANG=C.UTF-8 # Turns off writing .pyc files; superfluous on an ephemeral container. ENV PYTHONDONTWRITEBYTECODE=1 \ - VIRTUAL_ENV=/home/scu/.venv + VIRTUAL_ENV=/home/scu/.venv # Ensures that the python and pip executables used in the image will be # those from our virtualenv. @@ -57,19 +57,20 @@ FROM base as build ENV SC_BUILD_TARGET=build RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && apt-get install -y --no-install-recommends \ + build-essential \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" RUN pip install --no-cache-dir --upgrade \ - pip~=21.0.1 \ - wheel \ - setuptools + pip~=21.0.1 \ + wheel \ + setuptools WORKDIR /build @@ -95,7 +96,7 @@ COPY --chown=scu:scu services/catalog /build/services/catalog WORKDIR /build/services/catalog RUN pip3 --no-cache-dir install -r requirements/prod.txt &&\ - pip3 --no-cache-dir list -v + pip3 --no-cache-dir list -v # --------------------------Production stage ------------------- @@ -108,7 +109,7 @@ RUN pip3 --no-cache-dir install -r requirements/prod.txt &&\ FROM base as production ENV SC_BUILD_TARGET=production \ - SC_BOOT_MODE=production + SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE @@ -117,19 +118,27 @@ WORKDIR /home/scu # Starting from clean base image, copies pre-installed virtualenv from prod-only-deps COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} +# wget required by wait-for +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + # Copies booting scripts COPY --chown=scu:scu services/catalog/docker services/catalog/docker RUN chmod +x services/catalog/docker/*.sh - +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for HEALTHCHECK --interval=30s \ - --timeout=20s \ - --start-period=30s \ - --retries=3 \ - CMD ["python3", "services/catalog/docker/healthcheck.py", "http://localhost:8000/"] + --timeout=20s \ + --start-period=30s \ + --retries=3 \ + CMD ["python3", "services/catalog/docker/healthcheck.py", "http://localhost:8000/"] ENTRYPOINT [ "/bin/sh", "services/catalog/docker/entrypoint.sh" ] -CMD ["/bin/sh", "services/catalog/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/catalog/docker/boot.sh"] # --------------------------Development stage ------------------- @@ -148,5 +157,8 @@ WORKDIR /devel RUN chown -R scu:scu "${VIRTUAL_ENV}" +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for + ENTRYPOINT ["/bin/sh", "services/catalog/docker/entrypoint.sh"] -CMD ["/bin/sh", "services/catalog/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/catalog/docker/boot.sh"] diff --git a/services/director-v2/Dockerfile b/services/director-v2/Dockerfile index c5f463f0c29..bfd98178dc6 100644 --- a/services/director-v2/Dockerfile +++ b/services/director-v2/Dockerfile @@ -59,6 +59,7 @@ ENV SC_BUILD_TARGET=build RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ + wget \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -118,9 +119,18 @@ WORKDIR /home/scu # Starting from clean base image, copies pre-installed virtualenv from prod-only-deps COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} +# wget required by wait-for +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + # Copies booting scripts COPY --chown=scu:scu services/director-v2/docker services/director-v2/docker RUN chmod +x services/director-v2/docker/*.sh +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for HEALTHCHECK --interval=30s \ @@ -130,7 +140,7 @@ HEALTHCHECK --interval=30s \ CMD ["python3", "services/director-v2/docker/healthcheck.py", "http://localhost:8000/"] ENTRYPOINT [ "/bin/sh", "services/director-v2/docker/entrypoint.sh" ] -CMD ["/bin/sh", "services/director-v2/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/director-v2/docker/boot.sh"] # --------------------------Development stage ------------------- @@ -150,5 +160,8 @@ WORKDIR /devel RUN chown -R scu:scu "${VIRTUAL_ENV}" +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for + ENTRYPOINT ["/bin/sh", "services/director-v2/docker/entrypoint.sh"] -CMD ["/bin/sh", "services/director-v2/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/director-v2/docker/boot.sh"] diff --git a/services/director/Dockerfile b/services/director/Dockerfile index ad28410fa8a..f33f456ec25 100644 --- a/services/director/Dockerfile +++ b/services/director/Dockerfile @@ -66,6 +66,7 @@ ENV SC_BUILD_TARGET=build RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ + wget \ git \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -122,9 +123,18 @@ WORKDIR /home/scu # bring installed package without build tools COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} +# wget required by wait-for +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + # copy docker entrypoint and boot scripts COPY --chown=scu:scu services/director/docker services/director/docker RUN chmod +x services/director/docker/*.sh +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for HEALTHCHECK --interval=30s \ --timeout=120s \ @@ -133,7 +143,7 @@ HEALTHCHECK --interval=30s \ CMD ["python3", "/home/scu/services/director/docker/healthcheck.py", "http://localhost:8080/v0/"] ENTRYPOINT [ "services/director/docker/entrypoint.sh" ] -CMD ["services/director/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/director/docker/boot.sh"] # --------------------------Development stage ------------------- @@ -150,5 +160,9 @@ ENV SC_BUILD_TARGET=development ENV NODE_SCHEMA_LOCATION=../../../api/specs/common/schemas/node-meta-v0.0.1.json WORKDIR /devel RUN chown -R scu:scu "${VIRTUAL_ENV}" + +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for + ENTRYPOINT [ "/bin/sh", "services/director/docker/entrypoint.sh" ] -CMD ["/bin/sh", "services/director/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/director/docker/boot.sh"] diff --git a/services/sidecar/Dockerfile b/services/sidecar/Dockerfile index b14e17c9cbf..3eacb14a937 100644 --- a/services/sidecar/Dockerfile +++ b/services/sidecar/Dockerfile @@ -60,6 +60,7 @@ ENV SC_BUILD_TARGET=build RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ + wget \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -111,8 +112,17 @@ WORKDIR /home/scu # bring installed package without build tools COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +# wget required by wait-for +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* # copy docker entrypoint and boot scripts COPY --chown=scu:scu services/sidecar/docker services/sidecar/docker +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for # we use the celery API HEALTHCHECK --interval=60s \ @@ -122,7 +132,7 @@ HEALTHCHECK --interval=60s \ CMD celery inspect ping -b amqp://${RABBIT_USER}:${RABBIT_PASSWORD}@${RABBIT_HOST}:${RABBIT_PORT} -d sidecar."$(cat ${SIDECAR_HOST_HOSTNAME_PATH})"@${HOSTNAME} ENTRYPOINT [ "/bin/sh", "services/sidecar/docker/entrypoint.sh" ] -CMD ["/bin/sh", "services/sidecar/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/sidecar/docker/boot.sh"] # --------------------------Development stage ------------------- @@ -142,5 +152,8 @@ RUN chown -R scu:scu "${VIRTUAL_ENV}" # NOTE: devel mode does NOT have HEALTHCHECK +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for + ENTRYPOINT [ "/bin/sh", "services/sidecar/docker/entrypoint.sh" ] -CMD ["/bin/sh", "services/sidecar/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/sidecar/docker/boot.sh"] diff --git a/services/storage/Dockerfile b/services/storage/Dockerfile index 946b76f4b4b..c32106e7f27 100644 --- a/services/storage/Dockerfile +++ b/services/storage/Dockerfile @@ -66,6 +66,7 @@ ENV SC_BUILD_TARGET build RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ + wget \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -127,9 +128,18 @@ WORKDIR /home/scu # bring installed package without build tools COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} +# wget required by wait-for +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + # copy docker entrypoint and boot scripts COPY --chown=scu:scu services/storage/docker services/storage/docker RUN chmod +x services/storage/docker/*.sh +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for HEALTHCHECK --interval=30s \ --timeout=120s \ @@ -139,7 +149,7 @@ HEALTHCHECK --interval=30s \ ENTRYPOINT [ "/bin/sh", "services/storage/docker/entrypoint.sh" ] -CMD ["/bin/sh", "services/storage/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/storage/docker/boot.sh"] # -------------------------- ------------------------------- @@ -166,6 +176,9 @@ RUN chown -R scu:scu "${VIRTUAL_ENV}" # the client's output folder to /devel/services/web/client. # The latter ls no files +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for + ENTRYPOINT [ "/bin/sh", "services/storage/docker/entrypoint.sh" ] -CMD ["/bin/sh", "services/storage/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/storage/docker/boot.sh"] # -------------------------- ------------------------------- diff --git a/services/web/Dockerfile b/services/web/Dockerfile index a6116d0b97a..d6e0628d3df 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -69,6 +69,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ libmagic1 \ + wget \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -135,15 +136,19 @@ WORKDIR /home/scu COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} # libmagic is requried by the exporter +# wget required by wait-for RUN apt-get update \ && apt-get install -y --no-install-recommends \ libmagic1 \ + wget \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* # copy docker entrypoint and boot scripts COPY --chown=scu:scu services/web/server/docker services/web/server/docker RUN chmod +x services/web/server/docker/*.sh +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for HEALTHCHECK --interval=30s \ --timeout=120s \ @@ -152,7 +157,7 @@ HEALTHCHECK --interval=30s \ CMD ["python3", "/home/scu/services/web/server/docker/healthcheck.py", "http://localhost:8080/v0/health"] ENTRYPOINT [ "services/web/server/docker/entrypoint.sh" ] -CMD ["services/web/server/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/web/server/docker/boot.sh"] @@ -176,5 +181,8 @@ RUN chown -R scu:scu "${VIRTUAL_ENV}" # the client's output folder to /devel/services/web/client. # The latter ls no files +COPY --chown=scu:scu scripts/common-docker-boot/wait-for /usr/local/bin/wait-for +RUN chmod +x /usr/local/bin/wait-for + ENTRYPOINT [ "/bin/sh", "services/web/server/docker/entrypoint.sh" ] -CMD ["/bin/sh", "services/web/server/docker/boot.sh"] +CMD ["/bin/sh", "-c", "wait-for http://migration:8000 -- services/web/server/docker/boot.sh"] From 18c6b0e751a8951c7fa7322ed7522484848480d6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 9 Jul 2021 11:54:41 +0200 Subject: [PATCH 05/13] sidecars were not connected to the network --- services/docker-compose.devel.yml | 2 ++ services/docker-compose.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index c0f2417d47b..366a99ab17e 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -158,6 +158,7 @@ services: - SWARM_STACK_NAME=${SWARM_STACK_NAME:-simcore} - SIDECAR_HOST_HOSTNAME_PATH=${SIDECAR_HOST_HOSTNAME_PATH:-/home/scu/hostname} networks: + - default - computational_services_subnet # adding a separate worker to handling MPI mode for development @@ -215,6 +216,7 @@ services: - SIDECAR_HOST_HOSTNAME_PATH=${SIDECAR_HOST_HOSTNAME_PATH:-/home/scu/hostname} - TARGET_MPI_NODE_CPU_COUNT=${DEV_PC_CPU_COUNT:-0} # development computer CPU count, if env var is missing put to 0 to disable networks: + - default - computational_services_subnet storage: diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 34edbd1e35f..2f67df1681e 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -226,6 +226,7 @@ services: START_AS_MODE_CPU: ${SIDECAR_FORCE_CPU_NODE:-0} networks: + - default - computational_services_subnet dask-sidecar: From dd515e60645935d38c6d601578cd442183296804 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 9 Jul 2021 13:02:52 +0200 Subject: [PATCH 06/13] storage requires migration --- services/sidecar/tests/integration/test_sidecar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/sidecar/tests/integration/test_sidecar.py b/services/sidecar/tests/integration/test_sidecar.py index e31c77e8f36..3cf9f448993 100644 --- a/services/sidecar/tests/integration/test_sidecar.py +++ b/services/sidecar/tests/integration/test_sidecar.py @@ -27,7 +27,7 @@ # # SEE packages/pytest-simcore/src/pytest_simcore/docker_compose.py # -pytest_simcore_core_services_selection = ["postgres", "rabbit", "storage"] +pytest_simcore_core_services_selection = ["postgres", "migration", "rabbit", "storage"] pytest_simcore_ops_services_selection = ["minio", "adminer"] From f9143578d24aa332e00a1dfe800fb93784556064 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 20 Jul 2021 16:43:45 +0200 Subject: [PATCH 07/13] adding more debugging info for test_core_service_running when fialing --- tests/swarm-deploy/test_swarm_runs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/swarm-deploy/test_swarm_runs.py b/tests/swarm-deploy/test_swarm_runs.py index 7a7eb6d07af..b814feb594a 100644 --- a/tests/swarm-deploy/test_swarm_runs.py +++ b/tests/swarm-deploy/test_swarm_runs.py @@ -148,6 +148,7 @@ def test_core_service_running( "Expected running, got instead" f"\n{pformat(task)}" f"\n{get_failed_tasks_logs(service, docker_client)}" + f"\n{service.logs()}" ) From 165d83298a8a02d9f26126d282ca6ea3fa54474f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 21 Jul 2021 08:07:48 +0200 Subject: [PATCH 08/13] adding more debug logging --- tests/swarm-deploy/test_swarm_runs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/swarm-deploy/test_swarm_runs.py b/tests/swarm-deploy/test_swarm_runs.py index b814feb594a..de289a20292 100644 --- a/tests/swarm-deploy/test_swarm_runs.py +++ b/tests/swarm-deploy/test_swarm_runs.py @@ -125,6 +125,7 @@ def test_core_service_running( f"Expected a {num_tasks} task(s) for '{service_name}', got instead" f"\n{ get_tasks_summary(tasks) }" f"\n{ get_failed_tasks_logs(service, docker_client) }" + f"\n{service.logs()}" ) for i in range(num_tasks): From f8719f6fae547c8026bc60ec6226ace57468d0c6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 21 Jul 2021 08:42:50 +0200 Subject: [PATCH 09/13] removing failing command --- tests/swarm-deploy/test_swarm_runs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/swarm-deploy/test_swarm_runs.py b/tests/swarm-deploy/test_swarm_runs.py index de289a20292..7a7eb6d07af 100644 --- a/tests/swarm-deploy/test_swarm_runs.py +++ b/tests/swarm-deploy/test_swarm_runs.py @@ -125,7 +125,6 @@ def test_core_service_running( f"Expected a {num_tasks} task(s) for '{service_name}', got instead" f"\n{ get_tasks_summary(tasks) }" f"\n{ get_failed_tasks_logs(service, docker_client) }" - f"\n{service.logs()}" ) for i in range(num_tasks): @@ -149,7 +148,6 @@ def test_core_service_running( "Expected running, got instead" f"\n{pformat(task)}" f"\n{get_failed_tasks_logs(service, docker_client)}" - f"\n{service.logs()}" ) From aae25347d34a91fd5f5615b25b452195de38e6f8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 21 Jul 2021 10:08:51 +0200 Subject: [PATCH 10/13] fix test causing CI to fail --- tests/swarm-deploy/test_swarm_runs.py | 127 +++++++++++++++++--------- 1 file changed, 83 insertions(+), 44 deletions(-) diff --git a/tests/swarm-deploy/test_swarm_runs.py b/tests/swarm-deploy/test_swarm_runs.py index 7a7eb6d07af..4ace6c3f85a 100644 --- a/tests/swarm-deploy/test_swarm_runs.py +++ b/tests/swarm-deploy/test_swarm_runs.py @@ -5,9 +5,10 @@ import logging import sys import time +from collections import deque from pathlib import Path from pprint import pformat -from typing import Any, Dict, List +from typing import Any, Deque, Dict, List, Set import pytest import requests @@ -20,16 +21,13 @@ current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent +SEPARATOR_LINE = "-" * 50 WAIT_TIME_SECS = 40 RETRY_COUNT = 7 MAX_WAIT_TIME = 240 -# wait if running pre-state -# https://docs.docker.com/engine/swarm/how-swarm-mode-works/swarm-task-states/ -SWARM_TASK_PRE_STATES = ["NEW", "PENDING", "ASSIGNED", "PREPARING", "STARTING"] - SWARM_TASK_FAILED_STATES = [ "COMPLETE", "FAILED", @@ -118,37 +116,73 @@ def test_core_service_running( # ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS # puiaevvmtbs1 simcore_storage.1 simcore_storage:latest crespo-wkstn Running Running 18 minutes ago # j5xtlrnn684y \_ simcore_storage.1 simcore_storage:latest crespo-wkstn Shutdown Failed 18 minutes ago "task: non-zero exit (1)" - tasks = service.tasks() - num_tasks = get_replicas(service_config) - - assert len(tasks) == num_tasks, ( - f"Expected a {num_tasks} task(s) for '{service_name}', got instead" - f"\n{ get_tasks_summary(tasks) }" - f"\n{ get_failed_tasks_logs(service, docker_client) }" - ) - for i in range(num_tasks): - task: Dict[str, Any] = service.tasks()[i] + expected_replicas = get_replicas(service_config) - for n in range(RETRY_COUNT): - task = service.tasks()[i] + # assert number of slots equals number of expected services + tasks = service.tasks() + slots: Set[int] = {t["Slot"] for t in tasks} + slot_count = len(slots) + + assert slot_count == expected_replicas, ( + f"Expected to have {expected_replicas} slot(s), " + f"instead {slot_count} slot(s) were found." + f"\n{get_tasks_summary(tasks)}" + f"\n{get_failed_tasks_logs(service, docker_client)}" + ) - if task["Status"]["State"].upper() in SWARM_TASK_PRE_STATES: - print( - "Waiting [{}/{}] ...\n{}".format( - n, RETRY_COUNT, get_tasks_summary(tasks) - ) + def _get_tasks_by_slots() -> Dict[int, Deque[Dict[str, Any]]]: + results: Dict[int, Deque[Dict[str, Any]]] = {} + + for task in service.tasks(): + slot = task["Slot"] + if slot not in results: + results[slot] = deque() + results[slot].append(task) + + return results + + def _are_any_tasks_running(slot_tasks: Deque[Dict[str, Any]]) -> bool: + running_tasks = [ + t for t in slot_tasks if t["Status"]["State"].upper() == "RUNNING" + ] + return len(running_tasks) > 0 + + # check at least one service in per slot is in running mode else raise error + tasks_by_slot: Dict[int, Deque[Dict[str, Any]]] = {} + running_service_by_slot: Dict[int, bool] = {s: False for s in slots} + for _ in range(RETRY_COUNT): + tasks_by_slot = _get_tasks_by_slots() + assert len(tasks_by_slot) == expected_replicas + + for slot, slot_tasks in tasks_by_slot.items(): + if _are_any_tasks_running(slot_tasks): + running_service_by_slot[slot] = True + + # if all services for all states are running then + if all(running_service_by_slot.values()): + break + else: + time.sleep(WAIT_TIME_SECS) + + # expecting no error, otherwise a nice error message is welcomed + error_message = "" + for slot, slot_tasks in tasks_by_slot.items(): + if not _are_any_tasks_running(slot_tasks): + message = ( + f"Expected running service for slot {slot}, " + f"but got instead\n{SEPARATOR_LINE}" + ) + for task in slot_tasks: + message += ( + f"\n{get_task_logs(task, service.name, docker_client)}" + f"\n{pformat(task)}" + f"\n{get_task_logs(task, service.name, docker_client)}" + f"\n{SEPARATOR_LINE}" ) - time.sleep(WAIT_TIME_SECS) - else: - break - - # should be running - assert task["Status"]["State"].upper() == "RUNNING", ( - "Expected running, got instead" - f"\n{pformat(task)}" - f"\n{get_failed_tasks_logs(service, docker_client)}" - ) + error_message += f"{message}\n" + + assert error_message == "" @pytest.mark.parametrize( @@ -210,21 +244,26 @@ def get_tasks_summary(tasks): return msg +def get_task_logs(task, service_name, docker_client): + task_logs = "" + + cid = task["Status"]["ContainerStatus"]["ContainerID"] + task_logs += "{2} {0} - {1} BEGIN {2}\n".format(service_name, task["ID"], "=" * 10) + if cid: + container = docker_client.containers.get(cid) + task_logs += container.logs().decode("utf-8") + else: + task_logs += " log unavailable. container does not exists\n" + task_logs += "{2} {0} - {1} END {2}\n".format(service_name, task["ID"], "=" * 10) + + return task_logs + + def get_failed_tasks_logs(service, docker_client): failed_logs = "" + for t in service.tasks(): if t["Status"]["State"].upper() in SWARM_TASK_FAILED_STATES: - cid = t["Status"]["ContainerStatus"]["ContainerID"] - failed_logs += "{2} {0} - {1} BEGIN {2}\n".format( - service.name, t["ID"], "=" * 10 - ) - if cid: - container = docker_client.containers.get(cid) - failed_logs += container.logs().decode("utf-8") - else: - failed_logs += " log unavailable. container does not exists\n" - failed_logs += "{2} {0} - {1} END {2}\n".format( - service.name, t["ID"], "=" * 10 - ) + failed_logs += get_task_logs(t, service.name, docker_client) return failed_logs From 53227031070fabf07eb3c88a3fada7cc79ad83a1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 21 Jul 2021 13:00:43 +0200 Subject: [PATCH 11/13] fixes broken integration tests --- .../pytest-simcore/src/pytest_simcore/simcore_services.py | 2 +- packages/simcore-sdk/tests/integration/test_filemanager.py | 2 +- packages/simcore-sdk/tests/integration/test_nodeports.py | 2 +- packages/simcore-sdk/tests/integration/test_nodeports2.py | 2 +- .../director-v2/tests/integration/test_computation_api.py | 1 + .../tests/unit/with_dbs/test_modules_scheduler.py | 2 +- services/docker-compose.local.yml | 4 ++++ services/docker-compose.yml | 2 ++ services/web/server/tests/integration/01/test_exporter.py | 1 + .../server/tests/integration/01/test_garbage_collection.py | 5 +++-- .../web/server/tests/integration/01/test_project_workflow.py | 1 + services/web/server/tests/integration/02/test_computation.py | 1 + 12 files changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index 9b500ebb08d..ca3ef6498c1 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -16,7 +16,7 @@ log = logging.getLogger(__name__) SERVICES_TO_SKIP = ["sidecar", "postgres", "redis", "rabbit"] -SERVICE_HEALTHCHECK_ENTRYPOINT = {"director-v2": "/"} +SERVICE_HEALTHCHECK_ENTRYPOINT = {"director-v2": "/", "migration": "/"} @pytest.fixture(scope="module") diff --git a/packages/simcore-sdk/tests/integration/test_filemanager.py b/packages/simcore-sdk/tests/integration/test_filemanager.py index d916c307b3c..e37874f4d1b 100644 --- a/packages/simcore-sdk/tests/integration/test_filemanager.py +++ b/packages/simcore-sdk/tests/integration/test_filemanager.py @@ -12,7 +12,7 @@ import pytest from simcore_sdk.node_ports import exceptions, filemanager -pytest_simcore_core_services_selection = ["postgres", "storage"] +pytest_simcore_core_services_selection = ["migration", "postgres", "storage"] pytest_simcore_ops_services_selection = [ "minio", diff --git a/packages/simcore-sdk/tests/integration/test_nodeports.py b/packages/simcore-sdk/tests/integration/test_nodeports.py index 42848ea5d06..6f12f375800 100644 --- a/packages/simcore-sdk/tests/integration/test_nodeports.py +++ b/packages/simcore-sdk/tests/integration/test_nodeports.py @@ -17,7 +17,7 @@ from simcore_sdk.node_ports._item import ItemConcreteValue from simcore_sdk.node_ports.nodeports import Nodeports -pytest_simcore_core_services_selection = ["postgres", "storage"] +pytest_simcore_core_services_selection = ["migration", "postgres", "storage"] pytest_simcore_ops_services_selection = [ "minio", diff --git a/packages/simcore-sdk/tests/integration/test_nodeports2.py b/packages/simcore-sdk/tests/integration/test_nodeports2.py index 29eca4881a6..933f2e57c98 100644 --- a/packages/simcore-sdk/tests/integration/test_nodeports2.py +++ b/packages/simcore-sdk/tests/integration/test_nodeports2.py @@ -17,7 +17,7 @@ from simcore_sdk.node_ports_v2.links import ItemConcreteValue from simcore_sdk.node_ports_v2.nodeports_v2 import Nodeports -pytest_simcore_core_services_selection = ["postgres", "storage"] +pytest_simcore_core_services_selection = ["migration", "postgres", "storage"] pytest_simcore_ops_services_selection = ["minio"] diff --git a/services/director-v2/tests/integration/test_computation_api.py b/services/director-v2/tests/integration/test_computation_api.py index 45e47b77a5e..bf3d833b150 100644 --- a/services/director-v2/tests/integration/test_computation_api.py +++ b/services/director-v2/tests/integration/test_computation_api.py @@ -38,6 +38,7 @@ from yarl import URL pytest_simcore_core_services_selection = [ + "migration", "director", "redis", "rabbit", diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_scheduler.py b/services/director-v2/tests/unit/with_dbs/test_modules_scheduler.py index f5b28e8aa1c..8c62ac9af61 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_scheduler.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_scheduler.py @@ -26,7 +26,7 @@ scheduler_task, ) -pytest_simcore_core_services_selection = ["postgres", "redis", "rabbit"] +pytest_simcore_core_services_selection = ["migration", "postgres", "redis", "rabbit"] pytest_simcore_ops_services_selection = ["adminer", "redis-commander"] diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index efe1c4a93a9..115e922dca1 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -111,6 +111,10 @@ services: - "15672:15672" - "15692" + migration: + ports: + - "8000" + redis: ports: - "6379" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index e1046b012fb..197a970e979 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -153,6 +153,8 @@ services: - DIRECTOR_V2_PORT=${DIRECTOR_V2_PORT:-8000} - STORAGE_HOST=${STORAGE_HOST:-storage} - STORAGE_PORT=${STORAGE_PORT:-8080} + - MIGRATION_HOST=${MIGRATION_HOST:-migration} + - MIGRATION_PORT=${MIGRATION_PORT:-8000} - SWARM_STACK_NAME=${SWARM_STACK_NAME:-simcore} - WEBSERVER_LOGLEVEL=${LOG_LEVEL:-WARNING} env_file: diff --git a/services/web/server/tests/integration/01/test_exporter.py b/services/web/server/tests/integration/01/test_exporter.py index 7cadc5ec43a..cc79cf6c088 100644 --- a/services/web/server/tests/integration/01/test_exporter.py +++ b/services/web/server/tests/integration/01/test_exporter.py @@ -56,6 +56,7 @@ log = logging.getLogger(__name__) pytest_simcore_core_services_selection = [ + "migration", "redis", "rabbit", "catalog", diff --git a/services/web/server/tests/integration/01/test_garbage_collection.py b/services/web/server/tests/integration/01/test_garbage_collection.py index 038c78104f1..47fd1b55e92 100644 --- a/services/web/server/tests/integration/01/test_garbage_collection.py +++ b/services/web/server/tests/integration/01/test_garbage_collection.py @@ -42,6 +42,7 @@ log = logging.getLogger(__name__) pytest_simcore_core_services_selection = [ + "migration", "postgres", "redis", "storage", @@ -371,7 +372,7 @@ async def test_t2_cleanup_resources_after_browser_is_closed( aiopg_engine, redis_client, ): - """ after a GUEST users with one opened project closes browser tab regularly (GC cleans everything) """ + """after a GUEST users with one opened project closes browser tab regularly (GC cleans everything)""" logged_guest_user = await login_guest_user(client) empty_guest_user_project = await new_project(client, logged_guest_user) assert await assert_users_count(aiopg_engine, 1) is True @@ -405,7 +406,7 @@ async def test_t2_cleanup_resources_after_browser_is_closed( async def test_t3_gc_will_not_intervene_for_regular_users_and_their_resources( simcore_services, client, socketio_client_factory: Callable, aiopg_engine ): - """ after a USER disconnects the GC will remove none of its projects or templates nor the user itself """ + """after a USER disconnects the GC will remove none of its projects or templates nor the user itself""" number_of_projects = 5 number_of_templates = 5 logged_user = await login_user(client) diff --git a/services/web/server/tests/integration/01/test_project_workflow.py b/services/web/server/tests/integration/01/test_project_workflow.py index b7685251ef2..379c966808b 100644 --- a/services/web/server/tests/integration/01/test_project_workflow.py +++ b/services/web/server/tests/integration/01/test_project_workflow.py @@ -39,6 +39,7 @@ # Selection of core and tool services started in this swarm fixture (integration) pytest_simcore_core_services_selection = [ + "migration", "catalog", "director", "postgres", diff --git a/services/web/server/tests/integration/02/test_computation.py b/services/web/server/tests/integration/02/test_computation.py index 5e1f0c0de75..ff576053899 100644 --- a/services/web/server/tests/integration/02/test_computation.py +++ b/services/web/server/tests/integration/02/test_computation.py @@ -46,6 +46,7 @@ # Selection of core and tool services started in this swarm fixture (integration) pytest_simcore_core_services_selection = [ + "migration", "redis", "rabbit", "director", From 76798da1ff4c45d6706d8a7592ad45f380422b7b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 21 Jul 2021 13:11:51 +0200 Subject: [PATCH 12/13] removed not neededmmigration service --- .../director-v2/tests/unit/with_dbs/test_modules_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_scheduler.py b/services/director-v2/tests/unit/with_dbs/test_modules_scheduler.py index 8c62ac9af61..f5b28e8aa1c 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_scheduler.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_scheduler.py @@ -26,7 +26,7 @@ scheduler_task, ) -pytest_simcore_core_services_selection = ["migration", "postgres", "redis", "rabbit"] +pytest_simcore_core_services_selection = ["postgres", "redis", "rabbit"] pytest_simcore_ops_services_selection = ["adminer", "redis-commander"] From 55f5f10c9a9bf91f425220f50aa9eed61bcc9a08 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 21 Jul 2021 13:57:15 +0200 Subject: [PATCH 13/13] trying to fix failing tests --- packages/simcore-sdk/tests/integration/test_filemanager.py | 2 +- services/web/server/tests/integration/01/test_exporter.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/simcore-sdk/tests/integration/test_filemanager.py b/packages/simcore-sdk/tests/integration/test_filemanager.py index e37874f4d1b..d916c307b3c 100644 --- a/packages/simcore-sdk/tests/integration/test_filemanager.py +++ b/packages/simcore-sdk/tests/integration/test_filemanager.py @@ -12,7 +12,7 @@ import pytest from simcore_sdk.node_ports import exceptions, filemanager -pytest_simcore_core_services_selection = ["migration", "postgres", "storage"] +pytest_simcore_core_services_selection = ["postgres", "storage"] pytest_simcore_ops_services_selection = [ "minio", diff --git a/services/web/server/tests/integration/01/test_exporter.py b/services/web/server/tests/integration/01/test_exporter.py index cc79cf6c088..7cadc5ec43a 100644 --- a/services/web/server/tests/integration/01/test_exporter.py +++ b/services/web/server/tests/integration/01/test_exporter.py @@ -56,7 +56,6 @@ log = logging.getLogger(__name__) pytest_simcore_core_services_selection = [ - "migration", "redis", "rabbit", "catalog",