From b2a5e26809ca7da52dc6d5840ebcde0f61f94863 Mon Sep 17 00:00:00 2001 From: scottxu Date: Mon, 15 Aug 2022 11:58:10 -0700 Subject: [PATCH] expose IfPresentPlaceholder and ConcatPlaceholder --- .../container_with_concat_placeholder.py | 4 +- .../container_with_if_placeholder.py | 8 ++- sdk/python/kfp/components/placeholders.py | 51 +++++++++++++++---- sdk/python/kfp/components/structures_test.py | 2 +- sdk/python/kfp/dsl/__init__.py | 4 ++ 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/sdk/python/kfp/compiler/test_data/components/container_with_concat_placeholder.py b/sdk/python/kfp/compiler/test_data/components/container_with_concat_placeholder.py index 54c8627edf0..707f71fd50e 100644 --- a/sdk/python/kfp/compiler/test_data/components/container_with_concat_placeholder.py +++ b/sdk/python/kfp/compiler/test_data/components/container_with_concat_placeholder.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from kfp.components import placeholders +from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component from kfp.dsl import ContainerSpec from kfp.dsl import Dataset @@ -26,7 +26,7 @@ def container_with_concat_placeholder(text1: str, text2: Output[Dataset], image='python:3.7', command=[ 'my_program', - placeholders.ConcatPlaceholder(['prefix-', text1, text2.uri]) + ConcatPlaceholder(['prefix-', text1, text2.uri]) ], args=['--output_path', output_path]) diff --git a/sdk/python/kfp/compiler/test_data/components/container_with_if_placeholder.py b/sdk/python/kfp/compiler/test_data/components/container_with_if_placeholder.py index 3298fb8c287..b5e20e8ff9b 100644 --- a/sdk/python/kfp/compiler/test_data/components/container_with_if_placeholder.py +++ b/sdk/python/kfp/compiler/test_data/components/container_with_if_placeholder.py @@ -11,12 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional - -from kfp.components import placeholders from kfp.dsl import container_component from kfp.dsl import ContainerSpec from kfp.dsl import Dataset +from kfp.dsl import IfPresentPlaceholder from kfp.dsl import Output from kfp.dsl import OutputPath @@ -29,11 +27,11 @@ def container_with_if_placeholder(output_path: OutputPath(str), image='python:3.7', command=[ 'my_program', - placeholders.IfPresentPlaceholder( + IfPresentPlaceholder( input_name='optional_input', then=[optional_input], else_=['bye']), '--dataset', - placeholders.IfPresentPlaceholder( + IfPresentPlaceholder( input_name='optional_input', then=[dataset.uri], else_=['bye']) ], args=['--output_path', output_path]) diff --git a/sdk/python/kfp/components/placeholders.py b/sdk/python/kfp/components/placeholders.py index 5b8e7b0afb5..8b1917f8bdd 100644 --- a/sdk/python/kfp/components/placeholders.py +++ b/sdk/python/kfp/components/placeholders.py @@ -259,10 +259,23 @@ class ConcatPlaceholder(base_model.BaseModel, Placeholder): """Placeholder for concatenating multiple strings. May contain other placeholders. - Attributes: - items: Elements to concatenate. + Examples: + :: + + @container_component + def container_with_concat_placeholder(text1: str, text2: Output[Dataset], + output_path: OutputPath(str)): + return ContainerSpec( + image='python:3.7', + command=[ + 'my_program', + ConcatPlaceholder(['prefix-', text1, text2.uri]) + ], + args=['--output_path', output_path] + ) """ items: List[CommandLineElement] + """Elements to concatenate.""" @classmethod def split_cel_concat_string(self, string: str) -> List[str]: @@ -338,18 +351,36 @@ class IfPresentPlaceholder(base_model.BaseModel, Placeholder): """Placeholder for handling cases where an input may or may not be passed. May contain other placeholders. - Attributes: - input_name: name of the input/output. - then: If the input/output specified in name is present - the command-line argument will be replaced at run-time by the - expanded value of then. - else_: If the input/output specified in name is not present, - the command-line argument will be replaced at run-time by the - expanded value of otherwise. + Examples: + :: + + @container_component + def container_with_if_placeholder(output_path: OutputPath(str), + dataset: Output[Dataset], + optional_input: str = 'default'): + return ContainerSpec( + image='python:3.7', + command=[ + 'my_program', + IfPresentPlaceholder( + input_name='optional_input', + then=[optional_input], + else_=['no_input']), '--dataset', + IfPresentPlaceholder( + input_name='optional_input', then=[dataset.uri], else_=['no_dataset']) + ], + args=['--output_path', output_path] + ) """ input_name: str + """name of the input/output.""" + then: List[CommandLineElement] + """If the input/output specified in name is present, the command-line argument will be replaced at run-time by the expanded value of then.""" + else_: Optional[List[CommandLineElement]] = None + """If the input/output specified in name is not present, the command-line argument will be replaced at run-time by the expanded value of otherwise.""" + _aliases = {'input_name': 'inputName', 'else_': 'else'} @classmethod diff --git a/sdk/python/kfp/components/structures_test.py b/sdk/python/kfp/components/structures_test.py index 635801de07f..cd2bba71b6e 100644 --- a/sdk/python/kfp/components/structures_test.py +++ b/sdk/python/kfp/components/structures_test.py @@ -512,7 +512,7 @@ def comp_with_artifact_output(dataset_old: dsl.Output[dsl.Dataset], image='gcr.io/my-image', command=['sh', 'run.sh'], args=[ - placeholders.IfPresentPlaceholder( + dsl.IfPresentPlaceholder( input_name='optional_input', then=[dataset_old], else_=[dataset_new]) diff --git a/sdk/python/kfp/dsl/__init__.py b/sdk/python/kfp/dsl/__init__.py index 1f21b2b8754..2ebcc081e18 100644 --- a/sdk/python/kfp/dsl/__init__.py +++ b/sdk/python/kfp/dsl/__init__.py @@ -42,6 +42,8 @@ 'PIPELINE_JOB_ID_PLACEHOLDER', 'PIPELINE_TASK_NAME_PLACEHOLDER', 'PIPELINE_TASK_ID_PLACEHOLDER', + 'IfPresentPlaceholder', + 'ConcatPlaceholder', ] from kfp.components.component_decorator import component @@ -49,6 +51,8 @@ from kfp.components.importer_node import importer from kfp.components.pipeline_context import pipeline from kfp.components.pipeline_task import PipelineTask +from kfp.components.placeholders import ConcatPlaceholder +from kfp.components.placeholders import IfPresentPlaceholder from kfp.components.structures import ContainerSpec from kfp.components.task_final_status import PipelineTaskFinalStatus from kfp.components.tasks_group import Condition