From 99a6748221b0432ccd3c56b19d495e728bfd4a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Portela=20Afonso?= Date: Fri, 12 Aug 2022 12:24:56 +0100 Subject: [PATCH 1/6] Add support for COS public endpoint This commit aims to introduce support for a public endpoint to a Cloud Object Storage. It adds a new property to the schema JSON to be shown in the Runtime configuration section During the pipeline creation, it try to gets that variable if defined otherwise uses the api one. --- elyra/metadata/schemas/kfp.json | 10 ++++++++++ elyra/pipeline/kfp/processor_kfp.py | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/elyra/metadata/schemas/kfp.json b/elyra/metadata/schemas/kfp.json index 9a5759f39..e1d62add4 100644 --- a/elyra/metadata/schemas/kfp.json +++ b/elyra/metadata/schemas/kfp.json @@ -120,6 +120,16 @@ "ui:placeholder": "https://your-cos-service:port" } }, + "cos_public_endpoint": { + "title": "Public Cloud Object Storage Endpoint", + "description": "The public Cloud Object Storage endpoint", + "type": "string", + "format": "uri", + "uihints": { + "category": "Cloud Object Storage", + "ui:placeholder": "https://your-public-cos-endpoint:port" + } + }, "cos_bucket": { "title": "Cloud Object Storage Bucket Name", "description": "The Cloud Object Storage bucket name", diff --git a/elyra/pipeline/kfp/processor_kfp.py b/elyra/pipeline/kfp/processor_kfp.py index 06744d013..616c48e95 100644 --- a/elyra/pipeline/kfp/processor_kfp.py +++ b/elyra/pipeline/kfp/processor_kfp.py @@ -101,6 +101,7 @@ def process(self, pipeline): # unpack Cloud Object Storage configs cos_endpoint = runtime_configuration.metadata["cos_endpoint"] + cos_public_endpoint = runtime_configuration.metadata.get("cos_public_endpoint", cos_endpoint) cos_bucket = runtime_configuration.metadata["cos_bucket"] # Determine which provider to use to authenticate with Kubeflow @@ -362,7 +363,7 @@ def process(self, pipeline): ) if pipeline.contains_generic_operations(): - object_storage_url = f"{cos_endpoint}" + object_storage_url = f"{cos_public_endpoint}" os_path = join_paths(pipeline.pipeline_parameters.get(COS_OBJECT_PREFIX), pipeline_instance_id) object_storage_path = f"/{cos_bucket}/{os_path}" else: From eb61ad4a139420cc7d6771dab212189060625f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Portela=20Afonso?= Date: Tue, 16 Aug 2022 16:26:47 +0100 Subject: [PATCH 2/6] code review changes --- elyra/metadata/schemas/kfp.json | 2 +- packages/pipeline-editor/src/RuntimesWidget.tsx | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/elyra/metadata/schemas/kfp.json b/elyra/metadata/schemas/kfp.json index e1d62add4..c7a031a8b 100644 --- a/elyra/metadata/schemas/kfp.json +++ b/elyra/metadata/schemas/kfp.json @@ -120,7 +120,7 @@ "ui:placeholder": "https://your-cos-service:port" } }, - "cos_public_endpoint": { + "public_cos_endpoint": { "title": "Public Cloud Object Storage Endpoint", "description": "The public Cloud Object Storage endpoint", "type": "string", diff --git a/packages/pipeline-editor/src/RuntimesWidget.tsx b/packages/pipeline-editor/src/RuntimesWidget.tsx index 2dba26b9e..3835fa03f 100644 --- a/packages/pipeline-editor/src/RuntimesWidget.tsx +++ b/packages/pipeline-editor/src/RuntimesWidget.tsx @@ -64,7 +64,7 @@ class RuntimesDisplay extends MetadataDisplay< > { renderExpandableContent(metadata: IDictionary): JSX.Element { let apiEndpoint = addTrailingSlash(metadata.metadata.api_endpoint); - const cosEndpoint = addTrailingSlash(metadata.metadata.cos_endpoint); + let cosEndpoint = addTrailingSlash(metadata.metadata.cos_endpoint); let githubRepoElement = null; let metadata_props = null; @@ -98,6 +98,11 @@ class RuntimesDisplay extends MetadataDisplay< // user specified a public API endpoint. use it instead of the API endpoint apiEndpoint = addTrailingSlash(metadata.metadata.public_api_endpoint); } + + if (metadata.metadata.public_cos_endpoint) { + // user specified a public COS endpoint. use it instead of the API endpoint + cosEndpoint = addTrailingSlash(metadata.metadata.public_cos_endpoint); + } } return ( From 5c792d9741c36709427ab5cd76ee1a7ce80d2815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Portela=20Afonso?= Date: Tue, 16 Aug 2022 16:38:58 +0100 Subject: [PATCH 3/6] update documentation --- docs/source/user_guide/runtime-conf.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/user_guide/runtime-conf.md b/docs/source/user_guide/runtime-conf.md index f9028c9fe..7b01dbd4b 100644 --- a/docs/source/user_guide/runtime-conf.md +++ b/docs/source/user_guide/runtime-conf.md @@ -323,6 +323,13 @@ This should be the URL address of your S3-compatible Object Storage. If running Example: `https://minio-service.kubeflow:9000` +##### Public Cloud Object Storage endpoint (public_cos_endpoint) + +If the `Cloud Object Storage endpoint` setting identifies a URL that can only be resolved within the kubernetes cluster, Elyra cannot generate valid links to the object storage page. +If your instalation requires a different public URL specify it as `Public Cloud Object Storage endpoint`. + +Example: `https://public-kubernetes-service-url/storage` + ##### Cloud Object Storage bucket name (cos_bucket) Name of the bucket you want Elyra to store pipeline artifacts in. This setting is required. If the bucket doesn't exist, it will be created. The specified bucket name must meet the naming conventions imposed by the Object Storage service. From 85f447a63f9df5ff070f5e210443093e4da05926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Portela=20Afonso?= Date: Wed, 17 Aug 2022 10:08:00 +0100 Subject: [PATCH 4/6] code review changes --- docs/source/user_guide/runtime-conf.md | 4 ++-- elyra/pipeline/kfp/processor_kfp.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/user_guide/runtime-conf.md b/docs/source/user_guide/runtime-conf.md index 7b01dbd4b..7e892e5c8 100644 --- a/docs/source/user_guide/runtime-conf.md +++ b/docs/source/user_guide/runtime-conf.md @@ -325,8 +325,8 @@ Example: `https://minio-service.kubeflow:9000` ##### Public Cloud Object Storage endpoint (public_cos_endpoint) -If the `Cloud Object Storage endpoint` setting identifies a URL that can only be resolved within the kubernetes cluster, Elyra cannot generate valid links to the object storage page. -If your instalation requires a different public URL specify it as `Public Cloud Object Storage endpoint`. +If the `Cloud Object Storage endpoint` setting identifies a URL that can only be resolved within the Kubernetes cluster, Elyra cannot generate valid links to the object storage page. +If your installation requires a different public URL specify it as `Public Cloud Object Storage endpoint`. Example: `https://public-kubernetes-service-url/storage` diff --git a/elyra/pipeline/kfp/processor_kfp.py b/elyra/pipeline/kfp/processor_kfp.py index 616c48e95..09d945c01 100644 --- a/elyra/pipeline/kfp/processor_kfp.py +++ b/elyra/pipeline/kfp/processor_kfp.py @@ -101,7 +101,7 @@ def process(self, pipeline): # unpack Cloud Object Storage configs cos_endpoint = runtime_configuration.metadata["cos_endpoint"] - cos_public_endpoint = runtime_configuration.metadata.get("cos_public_endpoint", cos_endpoint) + cos_public_endpoint = runtime_configuration.metadata.get("public_cos_endpoint", cos_endpoint) cos_bucket = runtime_configuration.metadata["cos_bucket"] # Determine which provider to use to authenticate with Kubeflow From f9ad28cab434c839b0c437014b6e3a6219eb7f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Portela=20Afonso?= Date: Wed, 17 Aug 2022 11:19:40 +0100 Subject: [PATCH 5/6] fix ui integration tests --- tests/support/commands.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/support/commands.ts b/tests/support/commands.ts index 12a1392c5..cc18731e0 100644 --- a/tests/support/commands.ts +++ b/tests/support/commands.ts @@ -80,7 +80,9 @@ Cypress.Commands.add('createRuntimeConfig', ({ type } = {}): void => { ); } - cy.findByLabelText(/object storage endpoint/i).type('http://0.0.0.0:9000'); + cy.findByLabelText(/object storage .* endpoint\*/i).type( + 'http://0.0.0.0:9000' + ); cy.findByLabelText(/object storage username/i).type('minioadmin'); cy.findByLabelText(/object storage password/i).type('minioadmin'); cy.findByLabelText(/object storage bucket/i).type('test-bucket'); From 42f26a7f653f3c156fbfc892993248ab8c4b4b11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Portela=20Afonso?= Date: Thu, 18 Aug 2022 12:16:35 +0100 Subject: [PATCH 6/6] Update tests/support/commands.ts Co-authored-by: Patrick Titzler --- tests/support/commands.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/support/commands.ts b/tests/support/commands.ts index cc18731e0..635f2374a 100644 --- a/tests/support/commands.ts +++ b/tests/support/commands.ts @@ -80,7 +80,7 @@ Cypress.Commands.add('createRuntimeConfig', ({ type } = {}): void => { ); } - cy.findByLabelText(/object storage .* endpoint\*/i).type( + cy.findByLabelText(/^cloud object storage endpoint/i).type( 'http://0.0.0.0:9000' ); cy.findByLabelText(/object storage username/i).type('minioadmin');