Kf 3597 feat base class charm - kfp persistence (#268)

Initial version of base-class based KFP Persistence Agent charm.

Summary of changes:

Re-wrote charm using base-class framework.
Added integration tests local to the charm (minimal). Will rely on bundle level (top level integration tests).
Modified unit tests to be aligned with new framework.
NOTE: Due to deployment parameter in PodSpec version, kfp-persistence cannot be upgraded using Juju refresh:
juju.errors.JujuAPIError: Juju on containers does not support updating deployment info for services.
Upgrade will be done by re-deploying the charm.

.github/workflows/integrate.yaml

@@ -70,6 +70,7 @@ jobs:
       fail-fast: false
       matrix:
         charm:
+          - kfp-persistence
           - kfp-profile-controller
           - kfp-api
     steps:

charms/kfp-api/tests/integration/test_charm.py

@@ -87,7 +87,7 @@ async def test_relational_db_relation_with_mysql_relation(self, ops_test: OpsTes
             apps=["mysql-k8s"],
             status="active",
             raise_on_blocked=True,
-            timeout=90 * 20,
+            timeout=90 * 30,
             idle_period=20,
         )
 

charms/kfp-persistence/charmcraft.yaml

@@ -10,4 +10,6 @@ bases:
       channel: "20.04"
 parts:
   charm:
+    build-packages:
+    - git  # To install packages directly from github.  Remove when not needed
     charm-python-packages: [setuptools, pip]  # Fixes install of some packages

charms/kfp-persistence/metadata.yaml

@@ -3,11 +3,9 @@ summary: Reusable end-to-end ML workflows built using the Kubeflow Pipelines SDK
 description: |
   Machine learning (ML) toolkit that is dedicated to making deployments
   of ML workflows on Kubernetes simple, portable, and scalable.
-min-juju-version: "2.9.0"
-series: [kubernetes]
-deployment:
-  type: stateless
-  service: omit
+containers:
+  persistenceagent:
+    resource: oci-image
 resources:
   oci-image:
     type: oci-image

charms/kfp-persistence/requirements-integration.txt

@@ -32,7 +32,7 @@ cffi==1.15.1
     #   pynacl
 charset-normalizer==3.2.0
     # via requests
-cryptography==41.0.2
+cryptography==41.0.3
     # via
     #   paramiko
     #   pyopenssl
@@ -178,7 +178,7 @@ requests-oauthlib==1.3.1
     # via kubernetes
 rsa==4.9
     # via google-auth
-selenium==4.10.0
+selenium==4.11.2
     # via
     #   -r ./requirements-integration.in
     #   selenium-wire

charms/kfp-persistence/requirements-unit.in

@@ -14,7 +14,6 @@
 #   * Pinning a version of a python package/lib shared with requirements.in
 #     must not introduce any incompatibilities.
 coverage
-oci_image
 ops
 pytest
 pytest-mock

charms/kfp-persistence/requirements-unit.txt

@@ -4,33 +4,60 @@
 #
 #    pip-compile ./requirements-unit.in
 #
+anyio==3.7.1
+    # via httpcore
 attrs==23.1.0
     # via jsonschema
 certifi==2023.7.22
-    # via requests
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+charmed-kubeflow-chisme==0.2.0
+    # via -r ./requirements.in
 charset-normalizer==3.2.0
     # via requests
 coverage==7.2.7
     # via -r ./requirements-unit.in
+deepdiff==6.2.1
+    # via charmed-kubeflow-chisme
 exceptiongroup==1.1.2
-    # via pytest
+    # via
+    #   anyio
+    #   pytest
+h11==0.14.0
+    # via httpcore
+httpcore==0.17.3
+    # via httpx
+httpx==0.24.1
+    # via lightkube
 idna==3.4
-    # via requests
+    # via
+    #   anyio
+    #   httpx
+    #   requests
 importlib-resources==6.0.0
     # via jsonschema
 iniconfig==2.0.0
     # via pytest
+jinja2==3.1.2
+    # via charmed-kubeflow-chisme
 jsonschema==4.17.3
     # via serialized-data-interface
-oci-image==1.0.0
-    # via
-    #   -r ./requirements-unit.in
-    #   -r ./requirements.in
-ops==2.4.1
+lightkube==0.14.0
+    # via charmed-kubeflow-chisme
+lightkube-models==1.27.1.4
+    # via lightkube
+markupsafe==2.1.3
+    # via jinja2
+ops==2.5.0
     # via
     #   -r ./requirements-unit.in
     #   -r ./requirements.in
+    #   charmed-kubeflow-chisme
     #   serialized-data-interface
+ordered-set==4.1.0
+    # via deepdiff
 packaging==23.1
     # via pytest
 pkgutil-resolve-name==1.3.10
@@ -51,12 +78,26 @@ pytest-mock==3.11.1
 pyyaml==6.0.1
     # via
     #   -r ./requirements-unit.in
+    #   lightkube
     #   ops
     #   serialized-data-interface
 requests==2.31.0
     # via serialized-data-interface
+ruamel-yaml==0.17.32
+    # via charmed-kubeflow-chisme
+ruamel-yaml-clib==0.2.7
+    # via ruamel-yaml
 serialized-data-interface==0.7.0
-    # via -r ./requirements.in
+    # via
+    #   -r ./requirements.in
+    #   charmed-kubeflow-chisme
+sniffio==1.3.0
+    # via
+    #   anyio
+    #   httpcore
+    #   httpx
+tenacity==8.2.2
+    # via charmed-kubeflow-chisme
 tomli==2.0.1
     # via pytest
 urllib3==2.0.4

charms/kfp-persistence/requirements.in

@@ -1,3 +1,3 @@
+charmed-kubeflow-chisme
 ops
-oci-image
 serialized-data-interface

charms/kfp-persistence/requirements.txt

@@ -4,36 +4,79 @@
 #
 #    pip-compile ./requirements.in
 #
+anyio==3.7.1
+    # via httpcore
 attrs==23.1.0
     # via jsonschema
 certifi==2023.7.22
-    # via requests
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+charmed-kubeflow-chisme==0.2.0
+    # via -r ./requirements.in
 charset-normalizer==3.2.0
     # via requests
+deepdiff==6.2.1
+    # via charmed-kubeflow-chisme
+exceptiongroup==1.1.2
+    # via anyio
+h11==0.14.0
+    # via httpcore
+httpcore==0.17.3
+    # via httpx
+httpx==0.24.1
+    # via lightkube
 idna==3.4
-    # via requests
+    # via
+    #   anyio
+    #   httpx
+    #   requests
 importlib-resources==6.0.0
     # via jsonschema
+jinja2==3.1.2
+    # via charmed-kubeflow-chisme
 jsonschema==4.17.3
     # via serialized-data-interface
-oci-image==1.0.0
-    # via -r ./requirements.in
-ops==2.4.1
+lightkube==0.14.0
+    # via charmed-kubeflow-chisme
+lightkube-models==1.27.1.4
+    # via lightkube
+markupsafe==2.1.3
+    # via jinja2
+ops==2.5.0
     # via
     #   -r ./requirements.in
+    #   charmed-kubeflow-chisme
     #   serialized-data-interface
+ordered-set==4.1.0
+    # via deepdiff
 pkgutil-resolve-name==1.3.10
     # via jsonschema
 pyrsistent==0.19.3
     # via jsonschema
 pyyaml==6.0.1
     # via
+    #   lightkube
     #   ops
     #   serialized-data-interface
 requests==2.31.0
     # via serialized-data-interface
+ruamel-yaml==0.17.32
+    # via charmed-kubeflow-chisme
+ruamel-yaml-clib==0.2.7
+    # via ruamel-yaml
 serialized-data-interface==0.7.0
-    # via -r ./requirements.in
+    # via
+    #   -r ./requirements.in
+    #   charmed-kubeflow-chisme
+sniffio==1.3.0
+    # via
+    #   anyio
+    #   httpcore
+    #   httpx
+tenacity==8.2.2
+    # via charmed-kubeflow-chisme
 urllib3==2.0.4
     # via requests
 websocket-client==1.6.1