fix(sdk.v2): Fix the issue that using dsl. prefix in component I/O …

…type annotation breaks component at runtime. (#6714) * Import kfp and kfp.v2.dsl for component runtime * update golden * add release note * update golden
kubeflow · Oct 11, 2021 · 073c819 · 073c819
1 parent 8133c37
commit 073c819
Show file tree

Hide file tree

Showing 17 changed files with 131 additions and 120 deletions.
diff --git a/sdk/RELEASE.md b/sdk/RELEASE.md
@@ -16,9 +16,10 @@
 
 * Fix executor getting None as value when float 0 is passed in. [\#6682](https://github.com/kubeflow/pipelines/pull/6682)
 * Fix function-based components not preserving the namespace of GCPC artifact types. [\#6702](https://github.com/kubeflow/pipelines/pull/6702)
+* Fix `dsl.` prefix in component I/O type annotation breaking component at runtime. [\#6714](https://github.com/kubeflow/pipelines/pull/6714)
+* Update v2 yaml format [\#6661](https://github.com/kubeflow/pipelines/pull/6661)
 * Depends on `typing-extensions>=3.7.4,<4; python_version<"3.9"` [\#6683](https://github.com/kubeflow/pipelines/pull/6683)
 * Depends on `click>=7.1.2,<9` [\#6691](https://github.com/kubeflow/pipelines/pull/6691)
-* Update v2 yaml format [\#6661](https://github.com/kubeflow/pipelines/pull/6661)
 
 ## Documentation Updates
 

diff --git a/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml b/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml
@@ -3,15 +3,15 @@ kind: Workflow
 metadata:
   generateName: my-test-pipeline-
   annotations:
-    pipelines.kubeflow.org/kfp_sdk_version: 1.7.2
-    pipelines.kubeflow.org/pipeline_compilation_time: '2021-09-03T01:12:52.407964'
+    pipelines.kubeflow.org/kfp_sdk_version: 1.8.4
+    pipelines.kubeflow.org/pipeline_compilation_time: '2021-10-11T10:01:10.948014'
     pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "gs://output-directory/v2-artifacts",
       "name": "pipeline-root"}, {"default": "pipeline/my-test-pipeline", "name": "pipeline-name"}],
       "name": "my-test-pipeline"}'
     pipelines.kubeflow.org/v2_pipeline: "true"
   labels:
     pipelines.kubeflow.org/v2_pipeline: "true"
-    pipelines.kubeflow.org/kfp_sdk_version: 1.7.2
+    pipelines.kubeflow.org/kfp_sdk_version: 1.8.4
 spec:
   entrypoint: my-test-pipeline
   templates:
@@ -44,9 +44,9 @@ spec:
       - sh
       - -c
       - (python3 -m ensurepip || python3 -m ensurepip --user) && (PIP_DISABLE_PIP_VERSION_CHECK=1
-        python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.7.2'
+        python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.4'
         || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location
-        'kfp==1.7.2' --user) && "$0" "$@"
+        'kfp==1.8.4' --user) && "$0" "$@"
       - sh
       - -ec
       - |
@@ -55,6 +55,8 @@ spec:
         python3 -m kfp.v2.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
       - |2+
 
+        import kfp
+        from kfp.v2 import dsl
         from kfp.v2.dsl import *
         from typing import *
 
@@ -124,13 +126,13 @@ spec:
         pipelines.kubeflow.org/component_ref: '{}'
         pipelines.kubeflow.org/arguments.parameters: '{"some_int": "12", "uri": "uri-to-import"}'
       labels:
-        pipelines.kubeflow.org/kfp_sdk_version: 1.7.2
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.4
         pipelines.kubeflow.org/pipeline-sdk-type: kfp
         pipelines.kubeflow.org/v2_component: "true"
         pipelines.kubeflow.org/enable_caching: "true"
     initContainers:
     - command: [launcher, --copy, /kfp-launcher/launch]
-      image: gcr.io/ml-pipeline/kfp-launcher:1.7.2
+      image: gcr.io/ml-pipeline/kfp-launcher:1.8.4
       name: kfp-launcher
       mirrorVolumeMounts: true
     volumes:
@@ -141,9 +143,9 @@ spec:
       - sh
       - -c
       - (python3 -m ensurepip || python3 -m ensurepip --user) && (PIP_DISABLE_PIP_VERSION_CHECK=1
-        python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.7.2'
+        python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.4'
         || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location
-        'kfp==1.7.2' --user) && "$0" "$@"
+        'kfp==1.8.4' --user) && "$0" "$@"
       - sh
       - -ec
       - |
@@ -152,6 +154,8 @@ spec:
         python3 -m kfp.v2.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
       - |2+
 
+        import kfp
+        from kfp.v2 import dsl
         from kfp.v2.dsl import *
         from typing import *
 
@@ -226,13 +230,13 @@ spec:
         pipelines.kubeflow.org/component_ref: '{}'
         pipelines.kubeflow.org/arguments.parameters: '{"num_steps": "{{inputs.parameters.preprocess-output_parameter_one}}"}'
       labels:
-        pipelines.kubeflow.org/kfp_sdk_version: 1.7.2
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.4
         pipelines.kubeflow.org/pipeline-sdk-type: kfp
         pipelines.kubeflow.org/v2_component: "true"
         pipelines.kubeflow.org/enable_caching: "true"
     initContainers:
     - command: [launcher, --copy, /kfp-launcher/launch]
-      image: gcr.io/ml-pipeline/kfp-launcher:1.7.2
+      image: gcr.io/ml-pipeline/kfp-launcher:1.8.4
       name: kfp-launcher
       mirrorVolumeMounts: true
     volumes:

diff --git a/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml b/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml
@@ -3,15 +3,15 @@ kind: Workflow
 metadata:
   generateName: my-test-pipeline-with-custom-launcher-
   annotations:
-    pipelines.kubeflow.org/kfp_sdk_version: 1.7.2
-    pipelines.kubeflow.org/pipeline_compilation_time: '2021-09-03T01:12:52.250696'
+    pipelines.kubeflow.org/kfp_sdk_version: 1.8.4
+    pipelines.kubeflow.org/pipeline_compilation_time: '2021-10-11T10:01:10.779353'
     pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "gs://output-directory/v2-artifacts",
       "name": "pipeline-root"}, {"default": "pipeline/my-test-pipeline-with-custom-launcher",
       "name": "pipeline-name"}], "name": "my-test-pipeline-with-custom-launcher"}'
     pipelines.kubeflow.org/v2_pipeline: "true"
   labels:
     pipelines.kubeflow.org/v2_pipeline: "true"
-    pipelines.kubeflow.org/kfp_sdk_version: 1.7.2
+    pipelines.kubeflow.org/kfp_sdk_version: 1.8.4
 spec:
   entrypoint: my-test-pipeline-with-custom-launcher
   templates:
@@ -44,9 +44,9 @@ spec:
       - sh
       - -c
       - (python3 -m ensurepip || python3 -m ensurepip --user) && (PIP_DISABLE_PIP_VERSION_CHECK=1
-        python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.7.2'
+        python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.4'
         || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location
-        'kfp==1.7.2' --user) && "$0" "$@"
+        'kfp==1.8.4' --user) && "$0" "$@"
       - sh
       - -ec
       - |
@@ -55,6 +55,8 @@ spec:
         python3 -m kfp.v2.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
       - |2+
 
+        import kfp
+        from kfp.v2 import dsl
         from kfp.v2.dsl import *
         from typing import *
 
@@ -124,7 +126,7 @@ spec:
         pipelines.kubeflow.org/component_ref: '{}'
         pipelines.kubeflow.org/arguments.parameters: '{"some_int": "12", "uri": "uri-to-import"}'
       labels:
-        pipelines.kubeflow.org/kfp_sdk_version: 1.7.2
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.4
         pipelines.kubeflow.org/pipeline-sdk-type: kfp
         pipelines.kubeflow.org/v2_component: "true"
         pipelines.kubeflow.org/enable_caching: "true"
@@ -141,9 +143,9 @@ spec:
       - sh
       - -c
       - (python3 -m ensurepip || python3 -m ensurepip --user) && (PIP_DISABLE_PIP_VERSION_CHECK=1
-        python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.7.2'
+        python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.4'
         || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location
-        'kfp==1.7.2' --user) && "$0" "$@"
+        'kfp==1.8.4' --user) && "$0" "$@"
       - sh
       - -ec
       - |
@@ -152,6 +154,8 @@ spec:
         python3 -m kfp.v2.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
       - |2+
 
+        import kfp
+        from kfp.v2 import dsl
         from kfp.v2.dsl import *
         from typing import *
 
@@ -226,7 +230,7 @@ spec:
         pipelines.kubeflow.org/component_ref: '{}'
         pipelines.kubeflow.org/arguments.parameters: '{"num_steps": "{{inputs.parameters.preprocess-output_parameter_one}}"}'
       labels:
-        pipelines.kubeflow.org/kfp_sdk_version: 1.7.2
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.4
         pipelines.kubeflow.org/pipeline-sdk-type: kfp
         pipelines.kubeflow.org/v2_component: "true"
         pipelines.kubeflow.org/enable_caching: "true"

diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json
@@ -107,11 +107,11 @@
             "command": [
               "sh",
               "-c",
-              "(python3 -m ensurepip || python3 -m ensurepip --user) && (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.0' --user) && \"$0\" \"$@\"",
+              "(python3 -m ensurepip || python3 -m ensurepip --user) && (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.4' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.4' --user) && \"$0\" \"$@\"",
               "sh",
               "-ec",
               "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-              "\nfrom kfp.v2.dsl import *\nfrom typing import *\n\ndef preprocess(\n    # An input parameter of type string.\n    message: str,\n    # An input parameter of type dict.\n    input_dict_parameter: Dict[str, int],\n    # An input parameter of type list.\n    input_list_parameter: List[str],\n    # Use Output[T] to get a metadata-rich handle to the output artifact\n    # of type `Dataset`.\n    output_dataset_one: Output[Dataset],\n    # A locally accessible filepath for another output artifact of type\n    # `Dataset`.\n    output_dataset_two_path: OutputPath('Dataset'),\n    # A locally accessible filepath for an output parameter of type string.\n    output_parameter_path: OutputPath(str),\n    # A locally accessible filepath for an output parameter of type bool.\n    output_bool_parameter_path: OutputPath(bool),\n    # A locally accessible filepath for an output parameter of type dict.\n    output_dict_parameter_path: OutputPath(Dict[str, int]),\n    # A locally accessible filepath for an output parameter of type list.\n    output_list_parameter_path: OutputPath(List[str]),\n):\n    \"\"\"Dummy preprocessing step.\"\"\"\n\n    # Use Dataset.path to access a local file path for writing.\n    # One can also use Dataset.uri to access the actual URI file path.\n    with open(output_dataset_one.path, 'w') as f:\n        f.write(message)\n\n    # OutputPath is used to just pass the local file path of the output artifact\n    # to the function.\n    with open(output_dataset_two_path, 'w') as f:\n        f.write(message)\n\n    with open(output_parameter_path, 'w') as f:\n        f.write(message)\n\n    with open(output_bool_parameter_path, 'w') as f:\n        f.write(\n            str(True))  # use either `str()` or `json.dumps()` for bool values.\n\n    import json\n    with open(output_dict_parameter_path, 'w') as f:\n        f.write(json.dumps(input_dict_parameter))\n\n    with open(output_list_parameter_path, 'w') as f:\n        f.write(json.dumps(input_list_parameter))\n\n"
+              "\nimport kfp\nfrom kfp.v2 import dsl\nfrom kfp.v2.dsl import *\nfrom typing import *\n\ndef preprocess(\n    # An input parameter of type string.\n    message: str,\n    # An input parameter of type dict.\n    input_dict_parameter: Dict[str, int],\n    # An input parameter of type list.\n    input_list_parameter: List[str],\n    # Use Output[T] to get a metadata-rich handle to the output artifact\n    # of type `Dataset`.\n    output_dataset_one: Output[Dataset],\n    # A locally accessible filepath for another output artifact of type\n    # `Dataset`.\n    output_dataset_two_path: OutputPath('Dataset'),\n    # A locally accessible filepath for an output parameter of type string.\n    output_parameter_path: OutputPath(str),\n    # A locally accessible filepath for an output parameter of type bool.\n    output_bool_parameter_path: OutputPath(bool),\n    # A locally accessible filepath for an output parameter of type dict.\n    output_dict_parameter_path: OutputPath(Dict[str, int]),\n    # A locally accessible filepath for an output parameter of type list.\n    output_list_parameter_path: OutputPath(List[str]),\n):\n    \"\"\"Dummy preprocessing step.\"\"\"\n\n    # Use Dataset.path to access a local file path for writing.\n    # One can also use Dataset.uri to access the actual URI file path.\n    with open(output_dataset_one.path, 'w') as f:\n        f.write(message)\n\n    # OutputPath is used to just pass the local file path of the output artifact\n    # to the function.\n    with open(output_dataset_two_path, 'w') as f:\n        f.write(message)\n\n    with open(output_parameter_path, 'w') as f:\n        f.write(message)\n\n    with open(output_bool_parameter_path, 'w') as f:\n        f.write(\n            str(True))  # use either `str()` or `json.dumps()` for bool values.\n\n    import json\n    with open(output_dict_parameter_path, 'w') as f:\n        f.write(json.dumps(input_dict_parameter))\n\n    with open(output_list_parameter_path, 'w') as f:\n        f.write(json.dumps(input_list_parameter))\n\n"
             ],
             "image": "python:3.7"
           }
@@ -127,11 +127,11 @@
             "command": [
               "sh",
               "-c",
-              "(python3 -m ensurepip || python3 -m ensurepip --user) && (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.0' --user) && \"$0\" \"$@\"",
+              "(python3 -m ensurepip || python3 -m ensurepip --user) && (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.4' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet                 --no-warn-script-location 'kfp==1.8.4' --user) && \"$0\" \"$@\"",
               "sh",
               "-ec",
               "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-              "\nfrom kfp.v2.dsl import *\nfrom typing import *\n\ndef train(\n    # Use InputPath to get a locally accessible path for the input artifact\n    # of type `Dataset`.\n    dataset_one_path: InputPath('Dataset'),\n    # Use Input[T] to get a metadata-rich handle to the input artifact\n    # of type `Dataset`.\n    dataset_two: Input[Dataset],\n    # An input parameter of type string.\n    message: str,\n    # Use Output[T] to get a metadata-rich handle to the output artifact\n    # of type `Dataset`.\n    model: Output[Model],\n    # An input parameter of type bool.\n    input_bool: bool,\n    # An input parameter of type dict.\n    input_dict: Dict[str, int],\n    # An input parameter of type List[str].\n    input_list: List[str],\n    # An input parameter of type int with a default value.\n    num_steps: int = 100,\n):\n    \"\"\"Dummy Training step.\"\"\"\n    with open(dataset_one_path, 'r') as input_file:\n        dataset_one_contents = input_file.read()\n\n    with open(dataset_two.path, 'r') as input_file:\n        dataset_two_contents = input_file.read()\n\n    line = (f'dataset_one_contents: {dataset_one_contents} || '\n            f'dataset_two_contents: {dataset_two_contents} || '\n            f'message: {message} || '\n            f'input_bool: {input_bool}, type {type(input_bool)} || '\n            f'input_dict: {input_dict}, type {type(input_dict)} || '\n            f'input_list: {input_list}, type {type(input_list)} \\n')\n\n    with open(model.path, 'w') as output_file:\n        for i in range(num_steps):\n            output_file.write('Step {}\\n{}\\n=====\\n'.format(i, line))\n\n    # model is an instance of Model artifact, which has a .metadata dictionary\n    # to store arbitrary metadata for the output artifact.\n    model.metadata['accuracy'] = 0.9\n\n"
+              "\nimport kfp\nfrom kfp.v2 import dsl\nfrom kfp.v2.dsl import *\nfrom typing import *\n\ndef train(\n    # Use InputPath to get a locally accessible path for the input artifact\n    # of type `Dataset`.\n    dataset_one_path: InputPath('Dataset'),\n    # Use Input[T] to get a metadata-rich handle to the input artifact\n    # of type `Dataset`.\n    dataset_two: Input[Dataset],\n    # An input parameter of type string.\n    message: str,\n    # Use Output[T] to get a metadata-rich handle to the output artifact\n    # of type `Dataset`.\n    model: Output[Model],\n    # An input parameter of type bool.\n    input_bool: bool,\n    # An input parameter of type dict.\n    input_dict: Dict[str, int],\n    # An input parameter of type List[str].\n    input_list: List[str],\n    # An input parameter of type int with a default value.\n    num_steps: int = 100,\n):\n    \"\"\"Dummy Training step.\"\"\"\n    with open(dataset_one_path, 'r') as input_file:\n        dataset_one_contents = input_file.read()\n\n    with open(dataset_two.path, 'r') as input_file:\n        dataset_two_contents = input_file.read()\n\n    line = (f'dataset_one_contents: {dataset_one_contents} || '\n            f'dataset_two_contents: {dataset_two_contents} || '\n            f'message: {message} || '\n            f'input_bool: {input_bool}, type {type(input_bool)} || '\n            f'input_dict: {input_dict}, type {type(input_dict)} || '\n            f'input_list: {input_list}, type {type(input_list)} \\n')\n\n    with open(model.path, 'w') as output_file:\n        for i in range(num_steps):\n            output_file.write('Step {}\\n{}\\n=====\\n'.format(i, line))\n\n    # model is an instance of Model artifact, which has a .metadata dictionary\n    # to store arbitrary metadata for the output artifact.\n    model.metadata['accuracy'] = 0.9\n\n"
             ],
             "image": "python:3.7"
           }
@@ -249,7 +249,7 @@
       }
     },
     "schemaVersion": "2.0.0",
-    "sdkVersion": "kfp-1.8.0"
+    "sdkVersion": "kfp-1.8.4"
   },
   "runtimeConfig": {
     "gcsOutputDirectory": "dummy_root",
@@ -259,4 +259,4 @@
       }
     }
   }
-}
+}