✨adding agent service with dyv volumes removal (#3465)

.github/CODEOWNERS

@@ -24,6 +24,7 @@ Makefile                                  @pcrespov @sanderegg
 /scripts/demo/                            @odeimaiz @pcrespov
 /scripts/json-schema-to-openapi-schema    @sanderegg
 /scripts/template-projects/               @odeimaiz @pcrespov
+/services/agent                           @GitHK
 /services/api-server/                     @pcrespov
 /services/autoscaling/                    @Surfict @sanderegg @pcrespov
 /services/catalog/                        @pcrespov @sanderegg

.github/workflows/ci-testing-deploy.yml

@@ -67,6 +67,7 @@ jobs:
       service-library: ${{ steps.filter.outputs.service-library }}
       settings-library: ${{ steps.filter.outputs.settings-library }}
       simcore-sdk: ${{ steps.filter.outputs.simcore-sdk }}
+      agent: ${{ steps.filter.outputs.agent }}
       api: ${{ steps.filter.outputs.api }}
       api-server: ${{ steps.filter.outputs.api-server }}
       autoscaling: ${{ steps.filter.outputs.autoscaling }}
@@ -113,6 +114,9 @@ jobs:
               - 'packages/settings-library/**'
             simcore-sdk:
               - 'packages/**'
+            agent:
+              - 'packages/**'
+              - 'services/agent/**'
             api:
               - 'api/**'
             api-server:
@@ -369,6 +373,53 @@ jobs:
         with:
           flags: unittests #optional
 
+  unit-test-agent:
+    needs: changes
+    if: ${{ needs.changes.outputs.agent == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    name: "[unit] agent"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.9"]
+        os: [ubuntu-20.04]
+        docker_buildx: [v0.8.2]
+        docker_compose: [1.29.1]
+        include:
+          - docker_compose: 1.29.1
+            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: ${{ matrix.docker_buildx }}
+          driver: docker-container
+      - name: setup docker-compose
+        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
+      - name: setup python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          cache: "pip"
+          cache-dependency-path: "services/agent/requirements/ci.txt"
+      - name: install rclone
+        run: sudo ./ci/github/helpers/install_rclone.bash
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - name: install
+        run: ./ci/github/unit-testing/agent.bash install
+      - name: typecheck
+        run: ./ci/github/unit-testing/agent.bash typecheck
+      - name: test
+        if: always()
+        run: ./ci/github/unit-testing/agent.bash test
+      - uses: codecov/codecov-action@v3.1.1
+        with:
+          flags: unittests #optional
+
   unit-test-api:
     needs: changes
     if: ${{ needs.changes.outputs.api == 'true' || github.event_name == 'push' }}
@@ -1133,7 +1184,7 @@ jobs:
           driver: docker-container
       - name: setup docker-compose
         run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
-      - name: install rclone in CI
+      - name: install rclone
         run: sudo ./ci/github/helpers/install_rclone.bash
       - name: setup python environment
         uses: actions/setup-python@v4
@@ -1156,6 +1207,7 @@ jobs:
     if: always()
     needs:
       [
+        unit-test-agent,
         unit-test-api,
         unit-test-api-server,
         unit-test-autoscaling,
@@ -1451,7 +1503,7 @@ jobs:
           driver: docker-container
       - name: setup docker-compose
         run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
-      - name: install rclone in CI
+      - name: install rclone
         run: sudo ./ci/github/helpers/install_rclone.bash
       - name: setup python environment
         uses: actions/setup-python@v4

Makefile

@@ -31,6 +31,7 @@ $(if $(IS_WIN),$(error Windows is not supported in all recipes. Use WSL instead.
 # cat services/docker-compose-build.yml | yq ".services | keys | sort"
 #
 SERVICES_NAMES_TO_BUILD := \
+  agent \
   api-server \
   autoscaling \
   catalog \
@@ -55,6 +56,7 @@ export VCS_STATUS_CLIENT:= $(if $(shell git status -s),'modified/untracked','cle
 export BUILD_DATE       := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 
 # api-versions
+export AGENT_API_VERSION := $(shell cat $(CURDIR)/services/api-server/VERSION)
 export API_SERVER_API_VERSION := $(shell cat $(CURDIR)/services/api-server/VERSION)
 export AUTOSCALING_API_VERSION := $(shell cat $(CURDIR)/services/autoscaling/VERSION)
 export CATALOG_API_VERSION    := $(shell cat $(CURDIR)/services/catalog/VERSION)

ci/github/helpers/install_rclone.bash

@@ -9,9 +9,4 @@ set -o nounset   # abort on unbound variable
 set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
-
-R_CLONE_VERSION="1.58.0"
-curl --silent --location --remote-name "https://downloads.rclone.org/v${R_CLONE_VERSION}/rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
-dpkg --install "rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
-rm "rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
-rclone --version
+exec "$( dirname -- "$0"; )"/../../../scripts/install_rclone.bash

ci/github/unit-testing/agent.bash

@@ -0,0 +1,41 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+IFS=$'\n\t'
+
+install() {
+  bash ci/helpers/ensure_python_pip.bash
+  make devenv
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd services/agent
+  make install-ci
+  popd
+  .venv/bin/pip list --verbose
+}
+
+test() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd services/agent
+  make test-ci-unit
+  popd
+}
+
+typecheck() {
+  pushd services/agent
+  make mypy
+  popd
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

ci/github/unit-testing/dask-task-models-library.bash

@@ -16,14 +16,6 @@ install() {
   .venv/bin/pip list --verbose
 }
 
-codestyle() {
-  # shellcheck source=/dev/null
-  source .venv/bin/activate
-  pushd packages/dask-task-models-library
-  make codestyle-ci
-  popd
-}
-
 test() {
   # shellcheck source=/dev/null
   source .venv/bin/activate

ci/github/unit-testing/dynamic-sidecar.bash

@@ -16,14 +16,6 @@ install() {
   .venv/bin/pip list --verbose
 }
 
-codestyle() {
-  # shellcheck source=/dev/null
-  source .venv/bin/activate
-  pushd services/dynamic-sidecar
-  make codestyle-ci
-  popd
-}
-
 test() {
   # shellcheck source=/dev/null
   source .venv/bin/activate

packages/pytest-simcore/src/pytest_simcore/simcore_services.py

@@ -24,16 +24,17 @@
 log = logging.getLogger(__name__)
 
 
-SERVICES_TO_SKIP = [
-    "dask-sidecar",
+_SERVICES_TO_SKIP = {
+    "agent",  # global mode deploy (NO exposed ports, has http API)
+    "dask-sidecar",  # global mode deploy (NO exposed ports, **NO** http API)
     "migration",
     "postgres",
-    "redis",
     "rabbit",
+    "redis",
     "static-webserver",
-    "whoami",
     "traefik",
-]
+    "whoami",
+}
 # TODO: unify healthcheck policies see  https://github.com/ITISFoundation/osparc-simcore/pull/2281
 SERVICE_PUBLISHED_PORT = {}
 DEFAULT_SERVICE_HEALTHCHECK_ENTRYPOINT = "/v0/"
@@ -114,7 +115,7 @@ def services_endpoint(
 
         # TODO: unify healthcheck policies see  https://github.com/ITISFoundation/osparc-simcore/pull/2281
         # TODO: get health-check cmd from Dockerfile or docker-compose (e.g. postgres?)
-        if service not in SERVICES_TO_SKIP:
+        if service not in _SERVICES_TO_SKIP:
             target_ports = [
                 AIOHTTP_BASED_SERVICE_PORT,
                 FASTAPI_BASED_SERVICE_PORT,

packages/service-library/src/servicelib/docker_constants.py

@@ -0,0 +1,3 @@
+from typing import Final
+
+PREFIX_DYNAMIC_SIDECAR_VOLUMES: Final[str] = "dyv"

packages/service-library/src/servicelib/docker_utils.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 
-DOCKER_TIMESTAMP_LENGTH = len("2020-10-09T12:28:14.771034")
+_DOCKER_TIMESTAMP_LENGTH = len("2020-10-09T12:28:14.771034")
+
 
 
 def to_datetime(docker_timestamp: str) -> datetime:
@@ -10,5 +11,5 @@ def to_datetime(docker_timestamp: str) -> datetime:
     # The 099 before the Z is not clear, therefore we will truncate the last part
     # NOTE: must be in UNIX Timestamp format
     return datetime.strptime(
-        docker_timestamp[:DOCKER_TIMESTAMP_LENGTH], "%Y-%m-%dT%H:%M:%S.%f"
+        docker_timestamp[:_DOCKER_TIMESTAMP_LENGTH], "%Y-%m-%dT%H:%M:%S.%f"
     )

scripts/install_rclone.bash

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+#  Installs the latest version of rclone plugin
+#
+
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
+IFS=$'\n\t'
+
+
+R_CLONE_VERSION="1.58.0"
+curl --silent --location --remote-name "https://downloads.rclone.org/v${R_CLONE_VERSION}/rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
+dpkg --install "rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
+rm "rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
+rclone --version