🏗️ Maintenance week 48: requirements tooling refactor and library upg…

…rades (#2005) * lists of requirement to test different setups * Upgrades sqlalchemy patch * Upgrades prometheous-client to 0.9.0 * Sets a global constraints file unders requirements/constraints.txt for all repo * Fixed target dependencies in sidecar/requirements makefile * Upgrades aiohttp in the entire repo * Moved reqs docs, tools and base makefile under requirements folder * Renamed reqs listings * Downgrades webserver's aiohttp==3.6.3 since test_access_to_studies cookie_jar fails to return auth cookie with aiohttp==3.7.X. Needs further investigation * updated ownership and doc
ITISFoundation · Nov 30, 2020 · 42c5b20 · 42c5b20
1 parent 2834fd9
commit 42c5b20
Show file tree

Hide file tree

Showing 96 changed files with 272 additions and 155 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,27 +1,32 @@
 # Maps code in repository with maintainers
 # Order is important. The last matching pattern has the most precedence.
+# SEE https://docs.github.com/en/free-pro-team@latest/github/creating-cloning-and-archiving-repositories/about-code-owners#example-of-a-codeowners-file
+
 
 # files and folders recursively
 Makefile                                  @pcrespov, @sanderegg
 .codeclimate.yml                          @sanderegg, @pcrespov
 .travis.yml                               @sanderegg
+
+# NOTE: '/' denotes the root of the repository
 /api/                                     @sanderegg, @pcrespov
 /ci/                                      @sanderegg, @pcrespov
 /docs/                                    @pcrespov
-/packages/pytest-simcore                  @pcrespov, @sanderegg
-/packages/models-library                  @sanderegg, @pcrespov
-/packages/service-library                 @pcrespov
-/scripts/demo                             @odeimaiz, @pcrespov
+/packages/pytest-simcore/                 @pcrespov, @sanderegg
+/packages/models-library/                 @sanderegg, @pcrespov
+/packages/service-library/                @pcrespov
+/requirements/                            @pcrespov
+/scripts/demo/                            @odeimaiz, @pcrespov
 /scripts/json-schema-to-openapi-schema    @sanderegg
-/scripts/template-projects                @odeimaiz, @pcrespov
-/services/api-server                      @pcrespov
-/services/director*                       @sanderegg, @pcrespov
-/services/catalog                         @pcrespov, @sanderegg
-/services/migration                       @pcrespov
-/services/sidecar                         @pcrespov, @mguidon
-/services/web/client                      @odeimaiz, @oetiker, @ignapas
-/services/web/server                      @pcrespov
-/services/storage                         @mguidon
+/scripts/template-projects/               @odeimaiz, @pcrespov
+/services/api-server/                     @pcrespov
+/services/director*/                      @sanderegg, @pcrespov
+/services/catalog/                        @pcrespov, @sanderegg
+/services/migration/                      @pcrespov
+/services/sidecar/                        @pcrespov, @mguidon
+/services/web/client/                     @odeimaiz, @oetiker, @ignapas
+/services/web/server/                     @pcrespov
+/services/storage/                        @mguidon
 
 requirements/*                            @pcrespov
 tools/*                                   @pcrespov
diff --git a/api/tests/requirements.in b/api/tests/requirements.in
@@ -1,3 +1,4 @@
+-c ../../requirements/constraints.txt
 
 # testing
 coverage

diff --git a/api/tests/requirements.txt b/api/tests/requirements.txt
@@ -32,7 +32,7 @@ pytest-cov==2.10.1        # via -r requirements.in
 pytest-instafail==0.4.2   # via -r requirements.in
 pytest-sugar==0.9.4       # via -r requirements.in
 pytest==6.1.2             # via -r requirements.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-sugar
-pyyaml==5.3.1             # via openapi-spec-validator
+pyyaml==5.3.1             # via -c ../../requirements/constraints.txt, openapi-spec-validator
 six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging
 strict-rfc3339==0.7       # via openapi-schema-validator
 termcolor==1.1.0          # via pytest-sugar

diff --git a/packages/models-library/requirements/Makefile b/packages/models-library/requirements/Makefile
@@ -1,6 +1,6 @@
 #
 # Targets to pip-compile requirements
 #
-include ../../../scripts/requirements.Makefile
+include ../../../requirements/base.Makefile
 
 # Add here any extra explicit dependency: e.g. _migration.txt: _base.txt
diff --git a/packages/models-library/requirements/_base.in b/packages/models-library/requirements/_base.in
@@ -1,5 +1,6 @@
 #
 # Specifies third-party dependencies for 'models-library'
 #
+-c ../../../requirements/constraints.txt
 
 pydantic[email]
diff --git a/packages/models-library/requirements/_test.in b/packages/models-library/requirements/_test.in
@@ -1,6 +1,7 @@
 #
 # Specifies dependencies required to run 'models-library'
 #
+-c ../../../requirements/constraints.txt
 
 # Adds base AS CONSTRAINT specs, not requirement.
 #  - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt

diff --git a/packages/models-library/requirements/_test.txt b/packages/models-library/requirements/_test.txt
@@ -42,7 +42,7 @@ termcolor==1.1.0          # via pytest-sugar
 toml==0.10.2              # via pylint, pytest
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.4.3  # via aiohttp, yarl
-urllib3==1.26.2           # via requests
+urllib3==1.26.2           # via -c requirements/../../../requirements/constraints.txt, requests
 wrapt==1.12.1             # via astroid
 yarl==1.6.3               # via aiohttp
 zipp==3.4.0               # via importlib-metadata
diff --git a/packages/models-library/requirements/_tools.in b/packages/models-library/requirements/_tools.in
@@ -1,3 +1,4 @@
+-c ../../../requirements/constraints.txt
 -c _base.txt
 -c _test.txt
 

diff --git a/packages/models-library/requirements/_tools.txt b/packages/models-library/requirements/_tools.txt
@@ -21,7 +21,7 @@ nodeenv==1.5.0            # via pre-commit
 pathspec==0.8.1           # via black
 pip-tools==5.4.0          # via -r requirements/../../../requirements/devenv.txt
 pre-commit==2.9.2         # via -r requirements/../../../requirements/devenv.txt
-pyyaml==5.3.1             # via pre-commit
+pyyaml==5.3.1             # via -c requirements/../../../requirements/constraints.txt, pre-commit
 regex==2020.11.13         # via black
 six==1.15.0               # via -c requirements/_test.txt, pip-tools, virtualenv
 toml==0.10.2              # via -c requirements/_test.txt, black, pre-commit

diff --git a/packages/postgres-database/requirements/Makefile b/packages/postgres-database/requirements/Makefile
@@ -1,6 +1,6 @@
 #
 # Targets to pip-compile requirements
 #
-include ../../../scripts/requirements.Makefile
+include ../../../requirements/base.Makefile
 
 _migration.txt: _base.txt
diff --git a/packages/postgres-database/requirements/_base.in b/packages/postgres-database/requirements/_base.in
@@ -1,5 +1,7 @@
 #
 # Specifies third-party dependencies for 'simcore-postgres-database'
 #
-sqlalchemy[postgresql_psycopg2binary]>=1.3.3  # binary + Vulnerability
+-c ../../../requirements/constraints.txt
+
+sqlalchemy[postgresql_psycopg2binary]
 yarl
diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt
@@ -7,6 +7,6 @@
 idna==2.10                # via yarl
 multidict==4.7.6          # via yarl
 psycopg2-binary==2.8.6    # via sqlalchemy
-sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_base.in
+sqlalchemy[postgresql_psycopg2binary]==1.3.20  # via -c requirements/../../../requirements/constraints.txt, -r requirements/_base.in
 typing-extensions==3.7.4.3  # via yarl
 yarl==1.6.0               # via -r requirements/_base.in
diff --git a/packages/postgres-database/requirements/_migration.in b/packages/postgres-database/requirements/_migration.in
@@ -3,9 +3,10 @@
 # NOTE: this is an EXTRA!
 
 # Adds base AS CONSTRAINT specs, not requirement.
+-c ../../../requirements/constraints.txt
 -c _base.txt
 
-urllib3>=1.25.8    # Vulnerability
+urllib3
 
 alembic
 click

diff --git a/packages/postgres-database/requirements/_migration.txt b/packages/postgres-database/requirements/_migration.txt
@@ -17,7 +17,7 @@ python-dateutil==2.8.1    # via alembic
 python-editor==1.0.4      # via alembic
 requests==2.24.0          # via docker
 six==1.15.0               # via docker, python-dateutil, tenacity, websocket-client
-sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -c requirements/_base.txt, alembic
+sqlalchemy[postgresql_psycopg2binary]==1.3.20  # via -c requirements/../../../requirements/constraints.txt, -c requirements/_base.txt, alembic
 tenacity==6.2.0           # via -r requirements/_migration.in
-urllib3==1.25.11          # via -r requirements/_migration.in, requests
+urllib3==1.25.11          # via -c requirements/../../../requirements/constraints.txt, -r requirements/_migration.in, requests
 websocket-client==0.57.0  # via docker
diff --git a/packages/postgres-database/requirements/_test.in b/packages/postgres-database/requirements/_test.in
@@ -1,6 +1,7 @@
 #
 # Specifies dependencies required to run tests on 'simcore-postgres-database'
 #
+-c ../../../requirements/constraints.txt
 
 # Adds base AS CONSTRAINT specs, not requirement.
 #  - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt

diff --git a/packages/postgres-database/requirements/_test.txt b/packages/postgres-database/requirements/_test.txt
@@ -50,16 +50,16 @@ pytest-runner==5.2        # via -r requirements/_test.in
 pytest==6.1.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail
 python-dateutil==2.8.1    # via -c requirements/_migration.txt, faker
 python-dotenv==0.15.0     # via docker-compose
-pyyaml==5.3.1             # via -r requirements/_test.in, docker-compose
+pyyaml==5.3.1             # via -c requirements/../../../requirements/constraints.txt, -r requirements/_test.in, docker-compose
 requests==2.24.0          # via -c requirements/_migration.txt, coveralls, docker, docker-compose
 six==1.15.0               # via -c requirements/_migration.txt, astroid, bcrypt, cryptography, docker, dockerpty, jsonschema, packaging, pynacl, python-dateutil, websocket-client
-sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -c requirements/_base.txt, -c requirements/_migration.txt, aiopg
+sqlalchemy[postgresql_psycopg2binary]==1.3.20  # via -c requirements/../../../requirements/constraints.txt, -c requirements/_base.txt, -c requirements/_migration.txt, aiopg
 text-unidecode==1.3       # via faker
 texttable==1.6.3          # via docker-compose
 toml==0.10.2              # via pylint, pytest
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.4.3  # via -c requirements/_base.txt, aiohttp, yarl
-urllib3==1.25.11          # via -c requirements/_migration.txt, requests
+urllib3==1.25.11          # via -c requirements/../../../requirements/constraints.txt, -c requirements/_migration.txt, requests
 websocket-client==0.57.0  # via -c requirements/_migration.txt, docker, docker-compose
 wrapt==1.12.1             # via astroid
 yarl==1.6.0               # via -c requirements/_base.txt, aiohttp

diff --git a/packages/postgres-database/requirements/_tools.in b/packages/postgres-database/requirements/_tools.in
@@ -1,3 +1,4 @@
+-c ../../../requirements/constraints.txt
 -c _base.txt
 -c _test.txt
 

diff --git a/packages/postgres-database/requirements/_tools.txt b/packages/postgres-database/requirements/_tools.txt
@@ -21,7 +21,7 @@ nodeenv==1.5.0            # via pre-commit
 pathspec==0.8.1           # via black
 pip-tools==5.4.0          # via -r requirements/../../../requirements/devenv.txt
 pre-commit==2.9.2         # via -r requirements/../../../requirements/devenv.txt
-pyyaml==5.3.1             # via -c requirements/_test.txt, pre-commit
+pyyaml==5.3.1             # via -c requirements/../../../requirements/constraints.txt, -c requirements/_test.txt, pre-commit
 regex==2020.11.13         # via black
 six==1.15.0               # via -c requirements/_test.txt, pip-tools, virtualenv
 toml==0.10.2              # via -c requirements/_test.txt, black, pre-commit

diff --git a/packages/pytest-simcore/requirements/coverage.txt b/packages/pytest-simcore/requirements/coverage.txt
@@ -0,0 +1,7 @@
+#
+#  Include these tools for test coverage
+#
+pytest-cov
+coveralls
+coverage
+codecov
diff --git a/packages/pytest-simcore/requirements/mocks.txt b/packages/pytest-simcore/requirements/mocks.txt
@@ -0,0 +1,7 @@
+#
+# Include this libraries for mocking
+#
+
+pytest-mock
+faker
+mock
diff --git a/packages/pytest-simcore/requirements/tests_base.txt b/packages/pytest-simcore/requirements/tests_base.txt
@@ -0,0 +1,21 @@
+#
+# Essential testing libraries and some additional plugins
+#
+pytest
+
+# enhances tests logs
+pytest-icdiff
+pytest-instafail
+pytest-sugar
+
+# enhances test runs
+pytest-xdist
+pytest-runner # TODO: check if REQUIRED ANYMORE ???
+
+# adds functionality
+pytest-lazy-fixture
+
+
+# support testing features built on frameworks
+# pytest-docker
+# pytest-celery
diff --git a/packages/pytest-simcore/requirements/tests_w_aiohttp.txt b/packages/pytest-simcore/requirements/tests_w_aiohttp.txt
@@ -0,0 +1,7 @@
+#
+# Requirements to test aiohttp-specific services
+#
+pytest-aiohttp  # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76
+
+# fixtures
+aioresponses
diff --git a/packages/pytest-simcore/requirements/tests_w_fastapi.txt b/packages/pytest-simcore/requirements/tests_w_fastapi.txt
@@ -0,0 +1,8 @@
+#
+# Requirements to test fastapi-specific services
+#
+
+# fixtures
+asgi_lifespan
+# [https://github.com/encode/httpx/releases/tag/0.15.0] due to PR#1285 [https://github.com/encode/httpx/pull/1285]
+respx<0.13.0
diff --git a/packages/s3wrapper/requirements/Makefile b/packages/s3wrapper/requirements/Makefile
@@ -1,6 +1,6 @@
 #
 # Targets to pip-compile requirements
 #
-include ../../../scripts/requirements.Makefile
+include ../../../requirements/base.Makefile
 
 # Add here any extra explicit dependency: e.g. _migration.txt: _base.txt
diff --git a/packages/s3wrapper/requirements/_base.in b/packages/s3wrapper/requirements/_base.in
@@ -1,5 +1,7 @@
 #
 # Specifies third-party dependencies for 's3wrapper'
 #
-urllib3>=1.25.8 # Vulnerability
+-c ../../../requirements/constraints.txt
+
+urllib3
 minio
diff --git a/packages/s3wrapper/requirements/_base.txt b/packages/s3wrapper/requirements/_base.txt
@@ -10,4 +10,4 @@ minio==6.0.0              # via -r requirements/_base.in
 python-dateutil==2.8.1    # via minio
 pytz==2020.4              # via minio
 six==1.15.0               # via python-dateutil
-urllib3==1.26.2           # via -r requirements/_base.in, minio
+urllib3==1.26.2           # via -c requirements/../../../requirements/constraints.txt, -r requirements/_base.in, minio
diff --git a/packages/s3wrapper/requirements/_test.in b/packages/s3wrapper/requirements/_test.in
@@ -1,6 +1,7 @@
 #
 # Specifies dependencies required to run 's3wrapper'
 #
+-c ../../../requirements/constraints.txt
 
 # Adds base AS CONSTRAINT specs, not requirement.
 #  - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt

diff --git a/packages/s3wrapper/requirements/_test.txt b/packages/s3wrapper/requirements/_test.txt
@@ -40,13 +40,13 @@ pytest-docker==0.10.1     # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
 pytest==6.1.2             # via -r requirements/_test.in, pytest-cov, pytest-docker
 python-dotenv==0.15.0     # via docker-compose
-pyyaml==5.3.1             # via docker-compose
+pyyaml==5.3.1             # via -c requirements/../../../requirements/constraints.txt, docker-compose
 requests==2.25.0          # via -r requirements/_test.in, coveralls, docker, docker-compose
 six==1.15.0               # via -c requirements/_base.txt, astroid, bcrypt, cryptography, docker, dockerpty, jsonschema, packaging, pynacl, websocket-client
 texttable==1.6.3          # via docker-compose
 toml==0.10.2              # via pylint, pytest
 typed-ast==1.4.1          # via astroid
-urllib3==1.26.2           # via -c requirements/_base.txt, requests
+urllib3==1.26.2           # via -c requirements/../../../requirements/constraints.txt, -c requirements/_base.txt, requests
 websocket-client==0.57.0  # via docker, docker-compose
 wrapt==1.12.1             # via astroid
 zipp==3.4.0               # via importlib-metadata

diff --git a/packages/s3wrapper/requirements/_tools.in b/packages/s3wrapper/requirements/_tools.in
@@ -1,3 +1,4 @@
+-c ../../../requirements/constraints.txt
 -c _base.txt
 -c _test.txt
 

diff --git a/packages/s3wrapper/requirements/_tools.txt b/packages/s3wrapper/requirements/_tools.txt
@@ -21,7 +21,7 @@ nodeenv==1.5.0            # via pre-commit
 pathspec==0.8.1           # via black
 pip-tools==5.4.0          # via -r requirements/../../../requirements/devenv.txt
 pre-commit==2.9.2         # via -r requirements/../../../requirements/devenv.txt
-pyyaml==5.3.1             # via -c requirements/_test.txt, pre-commit
+pyyaml==5.3.1             # via -c requirements/../../../requirements/constraints.txt, -c requirements/_test.txt, pre-commit
 regex==2020.11.13         # via black
 six==1.15.0               # via -c requirements/_base.txt, -c requirements/_test.txt, pip-tools, virtualenv
 toml==0.10.2              # via -c requirements/_test.txt, black, pre-commit

diff --git a/packages/service-library/requirements/Makefile b/packages/service-library/requirements/Makefile
@@ -1,6 +1,6 @@
 #
 # Targets to pip-compile requirements
 #
-include ../../../scripts/requirements.Makefile
+include ../../../requirements/base.Makefile
 
 # Add here any extra explicit dependency: e.g. _migration.txt: _base.txt
diff --git a/packages/service-library/requirements/_base.in b/packages/service-library/requirements/_base.in
@@ -1,15 +1,13 @@
 #
 # Specifies third-party dependencies for 'service-library'
 #
+-c ../../../requirements/constraints.txt
 
-sqlalchemy>=1.3.3        # https://nvd.nist.gov/vuln/detail/CVE-2019-7164
-pyyaml>=5.3              # Vulnerable
-psycopg2-binary          # enforces binary version - http://initd.org/psycopg/docs/install.html#binary-install-from-pypi
 openapi-core==0.12.0     # frozen until https://github.com/ITISFoundation/osparc-simcore/pull/1396 is CLOSED
 lazy-object-proxy~=1.4.3 # cannot upgrade due to contraints in openapi-core
-aiozipkin==0.7.1         # breaking changes with new released version 1.0.0 (https://github.com/aio-libs/aiozipkin/releases)
 
 aiohttp
+aiozipkin
 aiopg[sa]
 ujson
 werkzeug
@@ -20,3 +18,6 @@ attrs
 trafaret
 aiodebug
 pydantic
+sqlalchemy
+psycopg2-binary
+pyyaml
diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt
@@ -5,9 +5,9 @@
 #    pip-compile --output-file=requirements/_base.txt requirements/_base.in
 #
 aiodebug==1.1.2           # via -r requirements/_base.in
-aiohttp==3.7.2            # via -r requirements/_base.in, aiozipkin
+aiohttp==3.7.3            # via -r requirements/_base.in, aiozipkin
 aiopg[sa]==1.0.0          # via -r requirements/_base.in
-aiozipkin==0.7.1          # via -r requirements/_base.in
+aiozipkin==0.7.1          # via -c requirements/../../../requirements/constraints.txt, -r requirements/_base.in
 async-timeout==3.0.1      # via aiohttp
 attrs==20.3.0             # via -r requirements/_base.in, aiohttp, jsonschema, openapi-core
 chardet==3.0.4            # via aiohttp
@@ -21,13 +21,13 @@ lazy-object-proxy==1.4.3  # via -r requirements/_base.in, openapi-core
 multidict==4.7.6          # via aiohttp, yarl
 openapi-core==0.12.0      # via -r requirements/_base.in
 openapi-spec-validator==0.2.9  # via openapi-core
-prometheus-client==0.8.0  # via -r requirements/_base.in
+prometheus-client==0.9.0  # via -r requirements/_base.in
 psycopg2-binary==2.8.6    # via -r requirements/_base.in, aiopg, sqlalchemy
 pydantic==1.7.2           # via -r requirements/_base.in
 pyrsistent==0.17.3        # via jsonschema
-pyyaml==5.3.1             # via -r requirements/_base.in, openapi-spec-validator
+pyyaml==5.3.1             # via -c requirements/../../../requirements/constraints.txt, -r requirements/_base.in, openapi-spec-validator
 six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-spec-validator, tenacity
-sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_base.in, aiopg
+sqlalchemy[postgresql_psycopg2binary]==1.3.20  # via -c requirements/../../../requirements/constraints.txt, -r requirements/_base.in, aiopg
 strict-rfc3339==0.7       # via openapi-core
 tenacity==6.2.0           # via -r requirements/_base.in
 trafaret==2.1.0           # via -r requirements/_base.in

diff --git a/packages/service-library/requirements/_test.in b/packages/service-library/requirements/_test.in
@@ -1,6 +1,7 @@
 #
 # Specifies dependencies required to run 'service-library'
 #
+-c ../../../requirements/constraints.txt
 
 # Adds base AS CONSTRAINT specs, not requirement.
 #  - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt