From a78531ada7cd2cec2a27366728388d48cfdc2d25 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Mon, 16 Oct 2023 10:44:20 +0300 Subject: [PATCH 01/12] update tox.ini's update-requirements Updates tox.ini's `update-requirements` as described in canonical/argo-operators#100. Ref canonical/kubeflow-ci#59 --- tox.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 13f98ba..4bcd683 100644 --- a/tox.ini +++ b/tox.ini @@ -32,8 +32,10 @@ allowlist_externals = pip-compile xargs commands = -; uses 'bash -c' because piping didn't work in regular tox commands - bash -c \'find . -type f -name "requirements*.in" | xargs --replace=\{\} pip-compile --resolver=backtracking \{\}' + ; we must preserve the order of compilation, since each *.in file depends on some *.txt file. + ; For example, requirements-unit.in depends on requirements.txt and we must compile first + ; requirements.txt to ensure that requirements-unit.txt get the same dependency as the requirements.txt + bash -c 'for pattern in "requirements.in" "requirements-fmt.in" "requirements*.in"; do find . -type f -name "$pattern" -exec bash -c "cd \$(dirname "{}") && pip-compile --resolver=backtracking \$(basename "{}")" \;; done' deps = pip-tools description = Update requirements files by executing pip-compile on all requirements*.in files, including those in subdirs. From 9d933899d3b51a7b1c9dde98d6578057edbb8c51 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Mon, 16 Oct 2023 10:46:57 +0300 Subject: [PATCH 02/12] requirements*.txt: Update with tox command --- requirements-fmt.txt | 22 ++-- requirements-integration.txt | 225 ++++++++++++++++++----------------- requirements-lint.txt | 56 +++++---- requirements-unit.txt | 119 +++++++++--------- requirements.txt | 55 +++++---- 5 files changed, 250 insertions(+), 227 deletions(-) diff --git a/requirements-fmt.txt b/requirements-fmt.txt index 53094cd..4e025c9 100644 --- a/requirements-fmt.txt +++ b/requirements-fmt.txt @@ -2,21 +2,23 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements-fmt.in +# pip-compile requirements-fmt.in # -black==22.10.0 - # via -r ./requirements-fmt.in -click==8.1.3 +black==23.9.1 + # via -r requirements-fmt.in +click==8.1.7 # via black -isort==5.10.1 - # via -r ./requirements-fmt.in -mypy-extensions==0.4.3 +isort==5.12.0 + # via -r requirements-fmt.in +mypy-extensions==1.0.0 # via black -pathspec==0.10.2 +packaging==23.2 # via black -platformdirs==2.5.4 +pathspec==0.11.2 + # via black +platformdirs==3.11.0 # via black tomli==2.0.1 # via black -typing-extensions==4.4.0 +typing-extensions==4.8.0 # via black diff --git a/requirements-integration.txt b/requirements-integration.txt index 5ca3938..9429c3b 100644 --- a/requirements-integration.txt +++ b/requirements-integration.txt @@ -2,51 +2,46 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements-integration.in +# pip-compile requirements-integration.in # -asttokens==2.2.1 +asttokens==2.4.0 # via stack-data -async-generator==1.10 +attrs==23.1.0 # via - # trio - # trio-websocket -attrs==22.1.0 - # via - # -r ./requirements.txt + # -r requirements.txt # jsonschema # outcome - # pytest # trio backcall==0.2.0 # via ipython bcrypt==4.0.1 # via paramiko -betterproto==2.0.0b5 +betterproto==2.0.0b6 # via - # -r ./requirements.txt + # -r requirements.txt # envoy-data-plane -blinker==1.5 +blinker==1.6.3 # via selenium-wire -brotli==1.0.9 +brotli==1.1.0 # via selenium-wire -cachetools==5.2.0 +cachetools==5.3.1 # via google-auth -certifi==2022.9.24 +certifi==2023.7.22 # via - # -r ./requirements.txt + # -r requirements.txt # kubernetes # requests # selenium # selenium-wire -cffi==1.15.1 +cffi==1.16.0 # via # cryptography # pynacl -chardet==3.0.4 +charset-normalizer==3.3.0 # via - # -r ./requirements.txt + # -r requirements.txt # requests -cryptography==38.0.4 +cryptography==41.0.4 # via # paramiko # pyopenssl @@ -55,89 +50,94 @@ decorator==5.1.1 # ipdb # ipython envoy-data-plane==0.2.5 - # via -r ./requirements.txt -exceptiongroup==1.0.4 + # via -r requirements.txt +exceptiongroup==1.1.3 # via # pytest # trio -executing==1.2.0 + # trio-websocket +executing==2.0.0 # via stack-data -google-auth==2.15.0 +google-auth==2.23.3 # via kubernetes -grpclib==0.4.3 +grpclib==0.4.6 # via - # -r ./requirements.txt + # -r requirements.txt # betterproto h11==0.14.0 # via wsproto h2==4.1.0 # via - # -r ./requirements.txt + # -r requirements.txt # grpclib # selenium-wire hpack==4.0.0 # via - # -r ./requirements.txt + # -r requirements.txt # h2 +hvac==1.2.1 + # via juju hyperframe==6.0.1 # via - # -r ./requirements.txt + # -r requirements.txt # h2 # selenium-wire -idna==2.10 +idna==3.4 # via - # -r ./requirements.txt + # -r requirements.txt # requests # trio -iniconfig==1.1.1 +importlib-resources==6.1.0 + # via + # -r requirements.txt + # jsonschema +iniconfig==2.0.0 # via pytest -ipdb==0.13.9 +ipdb==0.13.13 # via pytest-operator -ipython==8.7.0 +ipython==8.12.3 # via ipdb -jedi==0.18.2 +jedi==0.19.1 # via ipython jinja2==3.1.2 # via pytest-operator -jsonschema==3.2.0 +jsonschema==4.17.3 # via - # -r ./requirements.txt + # -r requirements.txt # serialized-data-interface -juju==3.0.4 +juju==3.2.2 # via - # -r ./requirements-integration.in + # -r requirements-integration.in # pytest-operator -jujubundlelib==0.5.7 - # via theblues kaitaistruct==0.10 # via selenium-wire -kubernetes==17.17.0 +kubernetes==27.2.0 # via juju macaroonbakery==1.3.1 - # via - # juju - # theblues -markupsafe==2.1.1 + # via juju +markupsafe==2.1.3 # via jinja2 matplotlib-inline==0.1.6 # via ipython -multidict==6.0.3 +multidict==6.0.4 # via - # -r ./requirements.txt + # -r requirements.txt # grpclib -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via typing-inspect oauthlib==3.2.2 - # via requests-oauthlib + # via + # kubernetes + # requests-oauthlib oci-image==1.0.0 - # via -r ./requirements.txt -ops==1.5.4 + # via -r requirements.txt +ops==2.7.0 # via - # -r ./requirements.txt + # -r requirements.txt # serialized-data-interface outcome==1.2.0 # via trio -packaging==21.3 +packaging==23.2 # via pytest paramiko==2.12.0 # via juju @@ -147,9 +147,13 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pluggy==1.0.0 +pkgutil-resolve-name==1.3.10 + # via + # -r requirements.txt + # jsonschema +pluggy==1.3.0 # via pytest -prompt-toolkit==3.0.33 +prompt-toolkit==3.0.39 # via ipython protobuf==3.20.3 # via macaroonbakery @@ -157,18 +161,20 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data -pyasn1==0.4.8 +pyasn1==0.5.0 # via # juju # pyasn1-modules # rsa # selenium-wire -pyasn1-modules==0.2.8 +pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pygments==2.13.0 +pygments==2.16.1 # via ipython +pyhcl==0.4.5 + # via hvac pymacaroons==0.13.0 # via macaroonbakery pynacl==1.5.0 @@ -176,74 +182,69 @@ pynacl==1.5.0 # macaroonbakery # paramiko # pymacaroons -pyopenssl==22.1.0 +pyopenssl==23.2.0 + # via selenium-wire +pyparsing==3.1.1 # via selenium-wire -pyparsing==3.0.9 - # via - # packaging - # selenium-wire pyrfc3339==1.1 # via # juju # macaroonbakery -pyrsistent==0.19.2 +pyrsistent==0.19.3 # via - # -r ./requirements.txt + # -r requirements.txt # jsonschema pysocks==1.7.1 # via # selenium-wire # urllib3 -pytest==7.2.0 +pytest==7.4.2 # via # pytest-asyncio # pytest-operator -pytest-asyncio==0.20.2 +pytest-asyncio==0.21.1 # via pytest-operator -pytest-operator==0.22.0 - # via -r ./requirements-integration.in +pytest-operator==0.29.0 + # via -r requirements-integration.in python-dateutil==2.8.2 # via - # -r ./requirements.txt + # -r requirements.txt # betterproto # kubernetes -pytz==2022.6 +pytz==2023.3.post1 # via pyrfc3339 -pyyaml==5.4 +pyyaml==6.0.1 # via - # -r ./requirements.txt + # -r requirements.txt # juju - # jujubundlelib # kubernetes # ops # pytest-operator # serialized-data-interface -requests==2.25.0 +requests==2.31.0 # via - # -r ./requirements.txt + # -r requirements.txt + # hvac # kubernetes # macaroonbakery # requests-oauthlib # serialized-data-interface - # theblues requests-oauthlib==1.3.1 # via kubernetes rsa==4.9 # via google-auth -selenium==4.7.2 +selenium==4.14.0 # via - # -r ./requirements-integration.in + # -r requirements-integration.in # selenium-wire selenium-wire==5.1.0 - # via -r ./requirements-integration.in -serialized-data-interface==0.4.0 - # via -r ./requirements.txt + # via -r requirements-integration.in +serialized-data-interface==0.7.0 + # via -r requirements.txt six==1.16.0 # via - # -r ./requirements.txt + # -r requirements.txt # asttokens - # google-auth - # jsonschema # kubernetes # macaroonbakery # paramiko @@ -253,50 +254,54 @@ sniffio==1.3.0 # via trio sortedcontainers==2.4.0 # via trio -stack-data==0.6.2 +stack-data==0.6.3 # via ipython stringcase==1.2.0 - # via -r ./requirements.txt -theblues==0.5.2 - # via juju -toml==0.10.2 - # via ipdb + # via -r requirements.txt tomli==2.0.1 - # via pytest -toposort==1.7 + # via + # ipdb + # pytest +toposort==1.10 # via juju -traitlets==5.6.0 +traitlets==5.11.2 # via # ipython # matplotlib-inline -trio==0.22.0 +trio==0.22.2 # via # selenium # trio-websocket -trio-websocket==0.9.2 +trio-websocket==0.11.1 # via selenium -typing-extensions==4.4.0 - # via typing-inspect -typing-inspect==0.8.0 +typing-extensions==4.8.0 + # via + # ipython + # typing-inspect +typing-inspect==0.9.0 # via juju -urllib3[socks]==1.26.13 +urllib3[socks]==2.0.6 # via - # -r ./requirements.txt + # -r requirements.txt # kubernetes # requests # selenium -wcwidth==0.2.5 +wcwidth==0.2.8 # via prompt-toolkit -websocket-client==1.4.2 - # via kubernetes -websockets==7.0 +websocket-client==1.6.4 + # via + # -r requirements.txt + # kubernetes + # ops +websockets==8.1 # via juju wsproto==1.2.0 # via # selenium-wire # trio-websocket -zstandard==0.19.0 +zipp==3.17.0 + # via + # -r requirements.txt + # importlib-resources +zstandard==0.21.0 # via selenium-wire - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements-lint.txt b/requirements-lint.txt index 5612ab9..c07a3a9 100644 --- a/requirements-lint.txt +++ b/requirements-lint.txt @@ -2,58 +2,62 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements-lint.in +# pip-compile requirements-lint.in # -black==22.10.0 - # via -r ./requirements-fmt.txt -click==8.1.3 +black==23.9.1 + # via -r requirements-fmt.txt +click==8.1.7 # via - # -r ./requirements-fmt.txt + # -r requirements-fmt.txt # black -codespell==2.2.2 - # via -r ./requirements-lint.in +codespell==2.2.6 + # via -r requirements-lint.in flake8==5.0.4 # via - # -r ./requirements-lint.in + # -r requirements-lint.in # flake8-builtins # pep8-naming # pyproject-flake8 -flake8-builtins==2.0.1 - # via -r ./requirements-lint.in -flake8-copyright==0.2.3 - # via -r ./requirements-lint.in -isort==5.10.1 - # via -r ./requirements-fmt.txt +flake8-builtins==2.1.0 + # via -r requirements-lint.in +flake8-copyright==0.2.4 + # via -r requirements-lint.in +isort==5.12.0 + # via -r requirements-fmt.txt mccabe==0.7.0 # via flake8 -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via - # -r ./requirements-fmt.txt + # -r requirements-fmt.txt # black -pathspec==0.10.2 +packaging==23.2 # via - # -r ./requirements-fmt.txt + # -r requirements-fmt.txt # black -pep8-naming==0.13.2 - # via -r ./requirements-lint.in -platformdirs==2.5.4 +pathspec==0.11.2 # via - # -r ./requirements-fmt.txt + # -r requirements-fmt.txt + # black +pep8-naming==0.13.3 + # via -r requirements-lint.in +platformdirs==3.11.0 + # via + # -r requirements-fmt.txt # black pycodestyle==2.9.1 # via flake8 pyflakes==2.5.0 # via flake8 pyproject-flake8==5.0.4.post1 - # via -r ./requirements-lint.in + # via -r requirements-lint.in tomli==2.0.1 # via - # -r ./requirements-fmt.txt + # -r requirements-fmt.txt # black # pyproject-flake8 -typing-extensions==4.4.0 +typing-extensions==4.8.0 # via - # -r ./requirements-fmt.txt + # -r requirements-fmt.txt # black # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements-unit.txt b/requirements-unit.txt index 5c17039..dce60c5 100644 --- a/requirements-unit.txt +++ b/requirements-unit.txt @@ -2,114 +2,123 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements-unit.in +# pip-compile requirements-unit.in # -attrs==22.1.0 +attrs==23.1.0 # via - # -r ./requirements.txt + # -r requirements.txt # jsonschema - # pytest -betterproto==2.0.0b5 +betterproto==2.0.0b6 # via - # -r ./requirements.txt + # -r requirements.txt # envoy-data-plane -certifi==2022.9.24 +certifi==2023.7.22 # via - # -r ./requirements.txt + # -r requirements.txt # requests -chardet==3.0.4 +charset-normalizer==3.3.0 # via - # -r ./requirements.txt + # -r requirements.txt # requests -coverage==6.5.0 - # via -r ./requirements-unit.in +coverage==7.3.2 + # via -r requirements-unit.in envoy-data-plane==0.2.5 - # via -r ./requirements.txt -exceptiongroup==1.0.4 + # via -r requirements.txt +exceptiongroup==1.1.3 # via pytest -grpclib==0.4.3 +grpclib==0.4.6 # via - # -r ./requirements.txt + # -r requirements.txt # betterproto h2==4.1.0 # via - # -r ./requirements.txt + # -r requirements.txt # grpclib hpack==4.0.0 # via - # -r ./requirements.txt + # -r requirements.txt # h2 hyperframe==6.0.1 # via - # -r ./requirements.txt + # -r requirements.txt # h2 -idna==2.10 +idna==3.4 # via - # -r ./requirements.txt + # -r requirements.txt # requests -iniconfig==1.1.1 +importlib-resources==6.1.0 + # via + # -r requirements.txt + # jsonschema +iniconfig==2.0.0 # via pytest -jsonschema==3.2.0 +jsonschema==4.17.3 # via - # -r ./requirements.txt + # -r requirements.txt # serialized-data-interface -multidict==6.0.3 +multidict==6.0.4 # via - # -r ./requirements.txt + # -r requirements.txt # grpclib oci-image==1.0.0 - # via -r ./requirements.txt -ops==1.5.4 + # via -r requirements.txt +ops==2.7.0 # via - # -r ./requirements.txt + # -r requirements.txt # serialized-data-interface -packaging==21.3 +packaging==23.2 # via pytest -pluggy==1.0.0 +pkgutil-resolve-name==1.3.10 + # via + # -r requirements.txt + # jsonschema +pluggy==1.3.0 # via pytest -pyparsing==3.0.9 - # via packaging -pyrsistent==0.19.2 +pyrsistent==0.19.3 # via - # -r ./requirements.txt + # -r requirements.txt # jsonschema -pytest==7.2.0 +pytest==7.4.2 # via - # -r ./requirements-unit.in + # -r requirements-unit.in # pytest-lazy-fixture # pytest-mock pytest-lazy-fixture==0.6.3 - # via -r ./requirements-unit.in -pytest-mock==3.10.0 - # via -r ./requirements-unit.in + # via -r requirements-unit.in +pytest-mock==3.11.1 + # via -r requirements-unit.in python-dateutil==2.8.2 # via - # -r ./requirements.txt + # -r requirements.txt # betterproto -pyyaml==5.4 +pyyaml==6.0.1 # via - # -r ./requirements.txt + # -r requirements.txt # ops # serialized-data-interface -requests==2.25.0 +requests==2.31.0 # via - # -r ./requirements.txt + # -r requirements.txt # serialized-data-interface -serialized-data-interface==0.4.0 - # via -r ./requirements.txt +serialized-data-interface==0.7.0 + # via -r requirements.txt six==1.16.0 # via - # -r ./requirements.txt - # jsonschema + # -r requirements.txt # python-dateutil stringcase==1.2.0 - # via -r ./requirements.txt + # via -r requirements.txt tomli==2.0.1 # via pytest -urllib3==1.26.13 +urllib3==2.0.6 # via - # -r ./requirements.txt + # -r requirements.txt # requests - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +websocket-client==1.6.4 + # via + # -r requirements.txt + # ops +zipp==3.17.0 + # via + # -r requirements.txt + # importlib-resources diff --git a/requirements.txt b/requirements.txt index f6e0ff8..3b42609 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,19 +2,19 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements.in +# pip-compile requirements.in # -attrs==22.1.0 +attrs==23.1.0 # via jsonschema -betterproto==2.0.0b5 +betterproto==2.0.0b6 # via envoy-data-plane -certifi==2022.9.24 +certifi==2023.7.22 # via requests -chardet==3.0.4 +charset-normalizer==3.3.0 # via requests envoy-data-plane==0.2.5 - # via -r ./requirements.in -grpclib==0.4.3 + # via -r requirements.in +grpclib==0.4.6 # via betterproto h2==4.1.0 # via grpclib @@ -22,38 +22,41 @@ hpack==4.0.0 # via h2 hyperframe==6.0.1 # via h2 -idna==2.10 +idna==3.4 # via requests -jsonschema==3.2.0 +importlib-resources==6.1.0 + # via jsonschema +jsonschema==4.17.3 # via serialized-data-interface -multidict==6.0.3 +multidict==6.0.4 # via grpclib oci-image==1.0.0 - # via -r ./requirements.in -ops==1.5.4 + # via -r requirements.in +ops==2.7.0 # via - # -r ./requirements.in + # -r requirements.in # serialized-data-interface -pyrsistent==0.19.2 +pkgutil-resolve-name==1.3.10 + # via jsonschema +pyrsistent==0.19.3 # via jsonschema python-dateutil==2.8.2 # via betterproto -pyyaml==5.4 +pyyaml==6.0.1 # via # ops # serialized-data-interface -requests==2.25.0 +requests==2.31.0 # via serialized-data-interface -serialized-data-interface==0.4.0 - # via -r ./requirements.in +serialized-data-interface==0.7.0 + # via -r requirements.in six==1.16.0 - # via - # jsonschema - # python-dateutil + # via python-dateutil stringcase==1.2.0 - # via -r ./requirements.in -urllib3==1.26.13 + # via -r requirements.in +urllib3==2.0.6 # via requests - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +websocket-client==1.6.4 + # via ops +zipp==3.17.0 + # via importlib-resources From 1f31e78cf473390e16f94bbd57ca883c8ed85db1 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Mon, 16 Oct 2023 10:47:54 +0300 Subject: [PATCH 03/12] Configure charm according to upstream Modify some values in order to follow upstream configuration from file `third_party/metadata_envoy/envoy.yaml` in pipelines repo. --- src/charm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/charm.py b/src/charm.py index d082614..20adee0 100755 --- a/src/charm.py +++ b/src/charm.py @@ -26,6 +26,7 @@ def get_cluster(service: str, port: int): name=service, connect_timeout=timedelta(seconds=30), type=api.ClusterDiscoveryType.LOGICAL_DNS, + http2_protocol_options=api.core.Http2ProtocolOptions(), lb_policy=api.ClusterLbPolicy.ROUND_ROBIN, hosts=[ api.core.Address( @@ -58,6 +59,7 @@ def get_listener(cluster: str, port: int): "x-accept-response-streaming", "x-grpc-web", "x-user-agent", + "custom-header-1", ] virtual_host = api.route.VirtualHost( @@ -68,7 +70,7 @@ def get_listener(cluster: str, port: int): match=api.route.RouteMatch(prefix="/"), route=api.route.RouteAction( cluster=cluster, - max_grpc_timeout=timedelta(seconds=60), + max_grpc_timeout=timedelta(seconds=0), ), ) ], @@ -77,7 +79,7 @@ def get_listener(cluster: str, port: int): allow_methods=",".join(allowed_methods), allow_headers=",".join(allowed_headers), max_age="1728000", - expose_headers="grpc-status,grpc-message", + expose_headers="grpc-status,grpc-message,custom-header-1", ), ) From 9a0339bfcb18a2a36e762d5f08081e726ef8c5c9 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Mon, 16 Oct 2023 11:02:25 +0300 Subject: [PATCH 04/12] tests(unit): Update expected config --- tests/unit/many_relations.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/unit/many_relations.yaml b/tests/unit/many_relations.yaml index f9ac6e5..2cbf599 100644 --- a/tests/unit/many_relations.yaml +++ b/tests/unit/many_relations.yaml @@ -39,11 +39,11 @@ static_resources: name: local_route virtual_hosts: - cors: - allow_headers: cache-control,content-transfer-encoding,content-type,grpc-timeout,keep-alive,user-agent,x-accept-content-transfer-encoding,x-accept-response-streaming,x-grpc-web,x-user-agent + allow_headers: cache-control,content-transfer-encoding,content-type,grpc-timeout,keep-alive,user-agent,x-accept-content-transfer-encoding,x-accept-response-streaming,x-grpc-web,x-user-agent,custom-header-1 allow_methods: GET,PUT,DELETE,POST,OPTIONS allow_origin: - '*' - expose_headers: grpc-status,grpc-message + expose_headers: grpc-status,grpc-message,custom-header-1 max_age: '1728000' domains: - '*' @@ -53,7 +53,6 @@ static_resources: prefix: / route: cluster: grpc-one - max_grpc_timeout: 60.000s stat_prefix: ingress_http name: envoy.http_connection_manager name: listener-0 @@ -72,11 +71,11 @@ static_resources: name: local_route virtual_hosts: - cors: - allow_headers: cache-control,content-transfer-encoding,content-type,grpc-timeout,keep-alive,user-agent,x-accept-content-transfer-encoding,x-accept-response-streaming,x-grpc-web,x-user-agent + allow_headers: cache-control,content-transfer-encoding,content-type,grpc-timeout,keep-alive,user-agent,x-accept-content-transfer-encoding,x-accept-response-streaming,x-grpc-web,x-user-agent,custom-header-1 allow_methods: GET,PUT,DELETE,POST,OPTIONS allow_origin: - '*' - expose_headers: grpc-status,grpc-message + expose_headers: grpc-status,grpc-message,custom-header-1 max_age: '1728000' domains: - '*' @@ -86,7 +85,6 @@ static_resources: prefix: / route: cluster: grpc-two - max_grpc_timeout: 60.000s stat_prefix: ingress_http name: envoy.http_connection_manager name: listener-0 From 942348fe1a29c27a2811d5285e52a9b35626d196 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Mon, 16 Oct 2023 12:14:28 +0300 Subject: [PATCH 05/12] tests(integration): Trust charms and configure relation names --- tests/integration/test_charm.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 454df17..a4c2c8b 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -37,15 +37,15 @@ async def test_build_and_deploy(ops_test): assert [app.entity_id for app in relation.applications] == [APP_NAME, "mlmd"] assert all([endpoint.name == "grpc" for endpoint in relation.endpoints]) - +@pytest.mark.abort_on_fail async def test_deploy_with_prometheus_and_grafana(ops_test): scrape_config = {"scrape_interval": "30s"} - await ops_test.model.deploy(PROMETHEUS, channel="latest/beta") - await ops_test.model.deploy(GRAFANA, channel="latest/beta") - await ops_test.model.deploy(PROMETHEUS_SCRAPE, channel="latest/beta", config=scrape_config) + await ops_test.model.deploy(PROMETHEUS, channel="latest/beta", trust=True) + await ops_test.model.deploy(GRAFANA, channel="latest/beta", trust = True) + await ops_test.model.deploy(PROMETHEUS_SCRAPE, channel="latest/beta", trust=True, config=scrape_config) await ops_test.model.add_relation(APP_NAME, PROMETHEUS_SCRAPE) - await ops_test.model.add_relation(PROMETHEUS, PROMETHEUS_SCRAPE) - await ops_test.model.add_relation(PROMETHEUS, GRAFANA) + await ops_test.model.add_relation(f"{PROMETHEUS}:metrics-endpoint", f"{PROMETHEUS_SCRAPE}:metrics-endpoint") + await ops_test.model.add_relation(f"{PROMETHEUS}:grafana-dashboard", f"{GRAFANA}:grafana-dashboard") await ops_test.model.add_relation(APP_NAME, GRAFANA) await ops_test.model.add_relation(PROMETHEUS, APP_NAME) From a26343e975bd3acfca81e70fd330911bc68f5683 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Mon, 16 Oct 2023 12:26:07 +0300 Subject: [PATCH 06/12] fix: lint --- tests/integration/test_charm.py | 15 +++++++++++---- tox.ini | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index a4c2c8b..deb7960 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -37,15 +37,22 @@ async def test_build_and_deploy(ops_test): assert [app.entity_id for app in relation.applications] == [APP_NAME, "mlmd"] assert all([endpoint.name == "grpc" for endpoint in relation.endpoints]) + @pytest.mark.abort_on_fail async def test_deploy_with_prometheus_and_grafana(ops_test): scrape_config = {"scrape_interval": "30s"} await ops_test.model.deploy(PROMETHEUS, channel="latest/beta", trust=True) - await ops_test.model.deploy(GRAFANA, channel="latest/beta", trust = True) - await ops_test.model.deploy(PROMETHEUS_SCRAPE, channel="latest/beta", trust=True, config=scrape_config) + await ops_test.model.deploy(GRAFANA, channel="latest/beta", trust=True) + await ops_test.model.deploy( + PROMETHEUS_SCRAPE, channel="latest/beta", trust=True, config=scrape_config + ) await ops_test.model.add_relation(APP_NAME, PROMETHEUS_SCRAPE) - await ops_test.model.add_relation(f"{PROMETHEUS}:metrics-endpoint", f"{PROMETHEUS_SCRAPE}:metrics-endpoint") - await ops_test.model.add_relation(f"{PROMETHEUS}:grafana-dashboard", f"{GRAFANA}:grafana-dashboard") + await ops_test.model.add_relation( + f"{PROMETHEUS}:metrics-endpoint", f"{PROMETHEUS_SCRAPE}:metrics-endpoint" + ) + await ops_test.model.add_relation( + f"{PROMETHEUS}:grafana-dashboard", f"{GRAFANA}:grafana-dashboard" + ) await ops_test.model.add_relation(APP_NAME, GRAFANA) await ops_test.model.add_relation(PROMETHEUS, APP_NAME) diff --git a/tox.ini b/tox.ini index 4bcd683..ed8214f 100644 --- a/tox.ini +++ b/tox.ini @@ -53,7 +53,7 @@ commands = # uncomment the following line if this charm owns a lib # codespell {[vars]lib_path} codespell {toxinidir}/. --skip {toxinidir}/.git --skip {toxinidir}/.tox \ - --skip {toxinidir}/build --skip {toxinidir}/lib --skip {toxinidir}/venv \ + --skip {toxinidir}/build --skip {toxinidir}/lib --skip {toxinidir}/./venv \ --skip {toxinidir}/.mypy_cache \ --skip {toxinidir}/icon.svg --skip *.json.tmpl # pflake8 wrapper supports config from pyproject.toml From 4f1dc1f286f4955531bdaf9273f42426230b9b20 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Mon, 16 Oct 2023 12:44:25 +0300 Subject: [PATCH 07/12] ci: Update integrate workflow environment --- .github/workflows/integrate.yaml | 42 +++++++++----------------------- 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index e48947a..6cab95d 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -51,39 +51,17 @@ jobs: - name: Check out repo uses: actions/checkout@v2 - - uses: balchua/microk8s-actions@v0.2.2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main with: - addons: '["dns", "storage", "rbac"]' - channel: 1.21/stable + provider: microk8s + channel: 1.25-strict/stable + juju-channel: 3.1/stable + microk8s-addons: "dns storage rbac metallb:10.64.140.43-10.64.140.49" - # Avoid race condition with storage taking a long time to initialize - - name: Wait for storage - run: | - sg microk8s -c 'microk8s kubectl rollout status deployment/hostpath-provisioner -n kube-system' + - name: Test + run: sg snap_microk8s -c "tox -vve integration -- --model testing" - - name: Install dependencies - run: | - set -eux - sudo snap install charmcraft --classic --channel=latest/candidate - sudo snap install juju --classic - sudo snap install juju-wait --classic - sudo snap install yq - - - name: Bootstrap Juju - run: | - set -eux - sg microk8s -c 'juju bootstrap microk8s uk8s' - juju add-model ci-test - - - name: Deploy charm - run: | - set -eux - charmcraft pack --destructive-mode - juju deploy ./*.charm \ - --resource oci-image=$(yq eval '.resources.oci-image.upstream-source' metadata.yaml) - juju deploy cs:mlmd - juju relate envoy mlmd - juju wait -wvt 300 - name: Get all run: kubectl get all -A @@ -108,3 +86,7 @@ jobs: - name: Get mlmd operator logs run: kubectl logs --tail 100 -nci-test -ljuju-operator=mlmd if: failure() + + - name: Collect charm debug artifacts + uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main + if: always() From 63a26ddb7bb0cddcf0c8e05242e449e2c7069c85 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Mon, 16 Oct 2023 13:21:05 +0300 Subject: [PATCH 08/12] ci: fix tests --- .github/workflows/integrate.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index 6cab95d..8fe86e5 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -44,7 +44,7 @@ jobs: run: tox -e unit deploy: - name: Test + name: Integration Tests runs-on: ubuntu-20.04 steps: @@ -62,7 +62,6 @@ jobs: - name: Test run: sg snap_microk8s -c "tox -vve integration -- --model testing" - - name: Get all run: kubectl get all -A if: failure() From 9e0d3c75e14884b1835b9875443ad79b59312e95 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Tue, 17 Oct 2023 12:46:28 +0300 Subject: [PATCH 09/12] review(ci): Update microk8s-addons --- .github/workflows/integrate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index 8fe86e5..49516e5 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -57,7 +57,7 @@ jobs: provider: microk8s channel: 1.25-strict/stable juju-channel: 3.1/stable - microk8s-addons: "dns storage rbac metallb:10.64.140.43-10.64.140.49" + microk8s-addons: "dns hostpath-storage rbac" - name: Test run: sg snap_microk8s -c "tox -vve integration -- --model testing" From 6a456af2bad46f35905fd20e93c5b72568269c57 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Tue, 17 Oct 2023 12:54:13 +0300 Subject: [PATCH 10/12] tests(integration): Update channel and assert - Use constant for mlmd charm name. - Use latest/edge channel for mlmd. - Use latest/stable channel for other charms. - Remove redundant assertion and add `idle_period` to the one left. --- tests/integration/test_charm.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index deb7960..5cabb79 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -19,32 +19,31 @@ PROMETHEUS = "prometheus-k8s" GRAFANA = "grafana-k8s" PROMETHEUS_SCRAPE = "prometheus-scrape-config-k8s" +MLMD="mlmd" @pytest.mark.abort_on_fail async def test_build_and_deploy(ops_test): - await ops_test.model.deploy("mlmd") + await ops_test.model.deploy(MLMD, channel="latest/edge") charm = await ops_test.build_charm(".") image_path = METADATA["resources"]["oci-image"]["upstream-source"] resources = {"oci-image": image_path} await ops_test.model.deploy(charm, resources=resources) - await ops_test.model.add_relation(APP_NAME, "mlmd") - await ops_test.model.wait_for_idle(status="active", raise_on_blocked=True) + await ops_test.model.add_relation(APP_NAME, MLMD) + await ops_test.model.wait_for_idle(status="active", raise_on_blocked=True, idle_period=30) - envoy_status = ops_test.model.applications[APP_NAME].units[0].workload_status - assert envoy_status == "active" relation = ops_test.model.relations[0] - assert [app.entity_id for app in relation.applications] == [APP_NAME, "mlmd"] + assert [app.entity_id for app in relation.applications] == [APP_NAME, MLMD] assert all([endpoint.name == "grpc" for endpoint in relation.endpoints]) @pytest.mark.abort_on_fail async def test_deploy_with_prometheus_and_grafana(ops_test): scrape_config = {"scrape_interval": "30s"} - await ops_test.model.deploy(PROMETHEUS, channel="latest/beta", trust=True) - await ops_test.model.deploy(GRAFANA, channel="latest/beta", trust=True) + await ops_test.model.deploy(PROMETHEUS, channel="latest/stable", trust=True) + await ops_test.model.deploy(GRAFANA, channel="latest/stable", trust=True) await ops_test.model.deploy( - PROMETHEUS_SCRAPE, channel="latest/beta", trust=True, config=scrape_config + PROMETHEUS_SCRAPE, channel="latest/stable", trust=True, config=scrape_config ) await ops_test.model.add_relation(APP_NAME, PROMETHEUS_SCRAPE) await ops_test.model.add_relation( From 50dae73869e618c1b476f368fc5b3793fe2a8e92 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Tue, 17 Oct 2023 12:57:54 +0300 Subject: [PATCH 11/12] fix: lint --- tests/integration/test_charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 5cabb79..a97b8fa 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -19,7 +19,7 @@ PROMETHEUS = "prometheus-k8s" GRAFANA = "grafana-k8s" PROMETHEUS_SCRAPE = "prometheus-scrape-config-k8s" -MLMD="mlmd" +MLMD = "mlmd" @pytest.mark.abort_on_fail From e2642e12d05c6ae9ba5c961cfc9f71e1ed99acc4 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Tue, 17 Oct 2023 15:07:07 +0300 Subject: [PATCH 12/12] tests(integration): Remove redundant assertion Remove assertion that doesn't make sense anymore. This PR changed the relations between charms resulting in different result from Prometheus endpoint. --- tests/integration/test_charm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index a97b8fa..986015e 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -68,7 +68,6 @@ async def test_correct_observability_setup(ops_test): ) response = json.loads(r.content.decode("utf-8")) assert response["status"] == "success" - assert len(response["data"]["result"]) == len(ops_test.model.applications[APP_NAME].units) response_metric = response["data"]["result"][0]["metric"] assert response_metric["juju_application"] == APP_NAME