Add UI/SDK support to pyfunc ensemblers (#171)

* Move getensemblerdirectory method from ensembling_job_service to ensembler_service

* Add holders in appcontext to carry an ensembler service builder

* Make deployment controller utilise pyfunc configs to generate docker image and configs

* Refactor tests with changes to ensembler config schema

* Revert changes to get ensembler directory function

* Create config class for resource request and timeout

* Fix migration script for new py_func_ref_config

* Fix lint comments

* Fix port for ensembler in turing api

* Update e2e test data

* Revert combination of resource request and timeout into a single object

* Revert changes to e2e test data

* Rename py_func_ref_config to pyfunc_config

* Rename pyfunc_config to make it consistent with camelcase PyfuncConfig

* Remove go build tags

* Remove go build tag from integration_config

* Refactor image building step into DeployRouterVersion process

* Rework image/docker naming convention to include run ids

* Add new PyFuncEnsembler class

* Fix typo of file name for ServiceAccountCombobox

* Add PyFunc ensembler config form

* Add PyFunc ensembler config view

* Remove redundant passing of props

* Refactor new name of pyfunc_config

* Update SDK with latest addition of PyfuncEnsemblers

* Add pytests for new pyfunc ensembler config

* Add leftover openapi generated files

* Clean up rebased files

* Add type casting for project_id

* Add optional docker config display to pyfunc view

* Fix optional docker config display to pyfunc view

* Fix OpenAPI return value inconsistencies

* Add missing openapi file

* Fix OpenAPI specs and classes

* Replace EuiSuperSelect with EuiComboBox

* Rearrange statements for clarity

* Remove default ensembler_id and cleanup naming irregularities

* Add version field to Router SDK class

* Refactor wait for status to update current router with latest configs

* Refactor wait for router version back to original

* Fix quotation marks

* Add additional precision to pyfunc ensembler description

* Refactor SDK and OpenAPI specs

api/api/openapi.bundle.yaml

@@ -242,7 +242,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/IdObject'
+                $ref: '#/components/schemas/JobId'
           description: Accepted.
         "400":
           description: Invalid ensembling job
@@ -551,7 +551,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/IdObject_1'
+                $ref: '#/components/schemas/RouterId'
           description: OK
         "400":
           description: Invalid project_id or router_id
@@ -682,7 +682,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/IdObject_1'
+                $ref: '#/components/schemas/RouterIdObject'
           description: OK
         "400":
           description: Invalid project_id or router_id
@@ -863,9 +863,7 @@ paths:
           content:
             application/json:
               schema:
-                items:
-                  $ref: '#/components/schemas/Event'
-                type: array
+                $ref: '#/components/schemas/RouterEvents'
           description: Get events
         "400":
           description: Invalid project_id or router_id
@@ -1694,13 +1692,16 @@ components:
       - failed_submission
       - failed_building
       type: string
+    JobId:
+      $ref: '#/components/schemas/IdObject'
     IdObject:
       example:
         id: 0
       properties:
         id:
           format: int32
           type: integer
+      type: object
     Id:
       format: int32
       type: integer
@@ -2520,7 +2521,7 @@ components:
       - environment_name
       - name
       type: object
-    IdObject_1:
+    RouterId:
       $ref: '#/components/schemas/IdObject'
     RouterIdAndVersion:
       example:
@@ -2534,6 +2535,37 @@ components:
           format: int32
           type: integer
       type: object
+    RouterIdObject:
+      example:
+        router_id: 0
+      properties:
+        router_id:
+          format: int32
+          type: integer
+      type: object
+    RouterEvents:
+      example:
+        events:
+        - event_type: info
+          updated_at: 2000-01-23T04:56:07.000+00:00
+          stage: stage
+          created_at: 2000-01-23T04:56:07.000+00:00
+          id: 0
+          message: message
+          version: 6
+        - event_type: info
+          updated_at: 2000-01-23T04:56:07.000+00:00
+          stage: stage
+          created_at: 2000-01-23T04:56:07.000+00:00
+          id: 0
+          message: message
+          version: 6
+      properties:
+        events:
+          items:
+            $ref: '#/components/schemas/Event'
+          type: array
+      type: object
     Event:
       example:
         event_type: info

api/api/specs/common.yaml

@@ -12,6 +12,7 @@ components:
       format: "int32"
 
     IdObject:
+      type: "object"
       properties:
         id:
           $ref: "#/components/schemas/Id"

api/api/specs/jobs.yaml

@@ -125,15 +125,15 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/IdObject"
+                $ref: "#/components/schemas/JobId"
         400:
           description: "Invalid ensembling job"
         404:
           description: "Ensembling job not found"
 
 components:
   schemas:
-    IdObject:
+    JobId:
       $ref: "common.yaml#/components/schemas/IdObject"
     EnsemblingJob:
       description: A JSON object that represents an ensembling job for batch experiment use cases

api/api/specs/routers.yaml

@@ -152,7 +152,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/IdObject"
+                $ref: "#/components/schemas/RouterId"
         400:
           description: "Invalid project_id or router_id"
         404:
@@ -212,7 +212,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/IdObject"
+                $ref: "#/components/schemas/RouterIdObject"
         400:
           description: "Invalid project_id or router_id"
         404:
@@ -380,18 +380,30 @@ paths:
           content:
             application/json:
               schema:
-                type: "array"
-                items:
-                  $ref: "#/components/schemas/Event"
+                $ref: "#/components/schemas/RouterEvents"
         400:
           description: "Invalid project_id or router_id"
         404:
           description: "No router version found"
 
 components:
   schemas:
-    IdObject:
-      $ref: "common.yaml#/components/schemas/IdObject"
+    RouterId:
+      $ref: "./common.yaml#/components/schemas/IdObject"
+
+    RouterIdObject:
+      type: object
+      properties:
+        router_id:
+          $ref: "common.yaml#/components/schemas/Id"
+
+    RouterEvents:
+      type: object
+      properties:
+        events:
+          type: "array"
+          items:
+            $ref: "#/components/schemas/Event"
 
     RouterIdAndVersion:
       type: object

sdk/samples/router/create_from_existing_router.py

@@ -108,7 +108,6 @@ def main(turing_api: str, project: str):
 
     # Create an ensembler for the router
     ensembler = DockerRouterEnsemblerConfig(
-        id=1,
         image="ealen/echo-server:0.5.1",
         resource_request=ResourceRequest(
             min_replica=1,

sdk/samples/router/general.py

@@ -162,7 +162,6 @@ def main(turing_api: str, project: str):
 
     # Create an ensembler for the router
     ensembler = DockerRouterEnsemblerConfig(
-        id=1,
         image="ealen/echo-server:0.5.1",
         resource_request=ResourceRequest(
             min_replica=1,

sdk/tests/conftest.py

@@ -333,14 +333,24 @@ def generic_ensembler_docker_config(generic_resource_request, generic_env_var):
     )
 
 
-@pytest.fixture(params=["standard", "docker"])
-def ensembler(request, generic_ensembler_standard_config, generic_ensembler_docker_config):
+@pytest.fixture
+def generic_ensembler_pyfunc_config(generic_resource_request):
+    return turing.generated.models.EnsemblerPyfuncConfig(
+        project_id=77,
+        ensembler_id=11,
+        resource_request=generic_resource_request,
+        timeout="500ms"
+    )
+
+
+@pytest.fixture(params=["standard", "docker", "pyfunc"])
+def ensembler(request, generic_ensembler_standard_config, generic_ensembler_docker_config, generic_ensembler_pyfunc_config):
     ensembler_type = request.param
     return turing.generated.models.RouterEnsemblerConfig(
-        id=1,
         type=ensembler_type,
         standard_config=generic_ensembler_standard_config,
         docker_config=generic_ensembler_docker_config,
+        pyfunc_config=generic_ensembler_pyfunc_config,
         created_at=datetime.now() + timedelta(seconds=10),
         updated_at=datetime.now() + timedelta(seconds=10)
     )
@@ -362,6 +372,14 @@ def generic_docker_router_ensembler_config(generic_ensembler_docker_config):
     )
 
 
+@pytest.fixture
+def generic_pyfunc_router_ensembler_config(generic_ensembler_pyfunc_config):
+    return turing.generated.models.RouterEnsemblerConfig(
+        type="pyfunc",
+        pyfunc_config=generic_ensembler_pyfunc_config
+    )
+
+
 @pytest.fixture
 def generic_enricher(generic_resource_request, generic_env_var):
     return turing.generated.models.Enricher(
@@ -508,7 +526,6 @@ def generic_router_config():
             ]
         ),
         ensembler=DockerRouterEnsemblerConfig(
-            id=1,
             image="test.io/just-a-test/turing-ensembler:0.0.0-build.0",
             resource_request=ResourceRequest(
                 min_replica=1,
@@ -559,7 +576,7 @@ def generic_routers(project, num_routers, generic_router_status, generic_router_
 
 @pytest.fixture
 def generic_events():
-    return turing.generated.models.InlineResponse2002(
+    return turing.generated.models.RouterEvents(
         events=[
             turing.generated.models.Event(
                 created_at=datetime.now(),