chore(sdk): partition KFP SDK source code into runtime and non-runtim…

…e code (#9710)

sdk/python/kfp/__init__.py

@@ -20,5 +20,6 @@
 
 TYPE_CHECK = True
 
+from kfp import components
 from kfp import dsl
 from kfp.client import Client

sdk/python/kfp/cli/compile_.py

@@ -22,8 +22,8 @@
 
 import click
 from kfp import compiler
-from kfp.components import base_component
-from kfp.components import graph_component
+from kfp.dsl import base_component
+from kfp.dsl import graph_component
 
 
 def is_pipeline_func(func: Callable) -> bool:

sdk/python/kfp/cli/component.py

@@ -31,9 +31,9 @@
     _DOCKER_IS_PRESENT = False
 
 import kfp as kfp
-from kfp.components import component_factory
-from kfp.components import kfp_config
-from kfp.components import utils
+from kfp.dsl import component_factory
+from kfp.dsl import kfp_config
+from kfp.dsl import utils
 
 _REQUIREMENTS_TXT = 'runtime-requirements.txt'
 

sdk/python/kfp/client/client.py

@@ -32,7 +32,7 @@
 from kfp import compiler
 from kfp.client import auth
 from kfp.client import set_volume_credentials
-from kfp.components import base_component
+from kfp.dsl import base_component
 from kfp.pipeline_spec import pipeline_spec_pb2
 import kfp_server_api
 import yaml

sdk/python/kfp/compiler/compiler.py

@@ -20,8 +20,8 @@
 from typing import Any, Dict, Optional
 
 from kfp.compiler import pipeline_spec_builder as builder
-from kfp.components import base_component
-from kfp.components.types import type_utils
+from kfp.dsl import base_component
+from kfp.dsl.types import type_utils
 
 
 class Compiler:

sdk/python/kfp/compiler/compiler_test.py

@@ -31,17 +31,17 @@
 from kfp.cli import cli
 from kfp.compiler import compiler
 from kfp.compiler import compiler_utils
-from kfp.components import graph_component
-from kfp.components import pipeline_task
-from kfp.components import yaml_component
-from kfp.components.types import type_utils
 from kfp.dsl import Artifact
 from kfp.dsl import ContainerSpec
+from kfp.dsl import graph_component
 from kfp.dsl import Input
 from kfp.dsl import Model
 from kfp.dsl import Output
 from kfp.dsl import OutputPath
+from kfp.dsl import pipeline_task
 from kfp.dsl import PipelineTaskFinalStatus
+from kfp.dsl import yaml_component
+from kfp.dsl.types import type_utils
 from kfp.pipeline_spec import pipeline_spec_pb2
 import yaml
 
@@ -152,18 +152,6 @@ def simple_pipeline():
             with open(target_json_file, 'r') as f:
                 f.read()
 
-    def test_compile_pipeline_with_dsl_graph_component_should_raise_error(self):
-
-        with self.assertRaisesRegex(
-                AttributeError,
-                "module 'kfp.dsl' has no attribute 'graph_component'"):
-
-            @dsl.graph_component
-            def flip_coin_graph_component():
-                flip = flip_coin_op()
-                with dsl.Condition(flip.output == 'heads'):
-                    flip_coin_graph_component()
-
     def test_compile_pipeline_with_misused_inputvalue_should_raise_error(self):
 
         upstream_op = components.load_component_from_text("""

sdk/python/kfp/compiler/compiler_utils.py

@@ -17,11 +17,11 @@
 from copy import deepcopy
 from typing import DefaultDict, Dict, List, Mapping, Set, Tuple, Union
 
-from kfp.components import for_loop
-from kfp.components import pipeline_channel
-from kfp.components import pipeline_context
-from kfp.components import pipeline_task
-from kfp.components import tasks_group
+from kfp.dsl import for_loop
+from kfp.dsl import pipeline_channel
+from kfp.dsl import pipeline_context
+from kfp.dsl import pipeline_task
+from kfp.dsl import tasks_group
 
 GroupOrTaskType = Union[tasks_group.TasksGroup, pipeline_task.PipelineTask]
 

sdk/python/kfp/compiler/compiler_utils_test.py

@@ -16,7 +16,7 @@
 
 from absl.testing import parameterized
 from kfp.compiler import compiler_utils
-from kfp.components import pipeline_channel
+from kfp.dsl import pipeline_channel
 
 
 class TestAdditionalInputNameForPipelineChannel(parameterized.TestCase):

sdk/python/kfp/compiler/pipeline_spec_builder.py

@@ -24,16 +24,16 @@
 from google.protobuf import struct_pb2
 import kfp
 from kfp.compiler import compiler_utils
-from kfp.components import for_loop
-from kfp.components import pipeline_channel
-from kfp.components import pipeline_context
-from kfp.components import pipeline_task
-from kfp.components import placeholders
-from kfp.components import structures
-from kfp.components import tasks_group
-from kfp.components import utils
-from kfp.components.types import artifact_types
-from kfp.components.types import type_utils
+from kfp.dsl import for_loop
+from kfp.dsl import pipeline_channel
+from kfp.dsl import pipeline_context
+from kfp.dsl import pipeline_task
+from kfp.dsl import placeholders
+from kfp.dsl import structures
+from kfp.dsl import tasks_group
+from kfp.dsl import utils
+from kfp.dsl.types import artifact_types
+from kfp.dsl.types import type_utils
 from kfp.pipeline_spec import pipeline_spec_pb2
 import yaml
 

sdk/python/kfp/compiler/read_write_test.py

@@ -22,9 +22,9 @@
 from absl.testing import parameterized
 from kfp import compiler
 from kfp import components
-from kfp.components import placeholders
-from kfp.components import python_component
-from kfp.components import structures
+from kfp.dsl import placeholders
+from kfp.dsl import python_component
+from kfp.dsl import structures
 import yaml
 
 _PROJECT_ROOT = os.path.abspath(os.path.join(__file__, *([os.path.pardir] * 5)))

sdk/python/kfp/components/__init__.py

@@ -24,10 +24,10 @@
     'YamlComponent',
 ]
 
-from kfp.components.base_component import BaseComponent
-from kfp.components.container_component import ContainerComponent
-from kfp.components.python_component import PythonComponent
-from kfp.components.yaml_component import load_component_from_file
-from kfp.components.yaml_component import load_component_from_text
-from kfp.components.yaml_component import load_component_from_url
-from kfp.components.yaml_component import YamlComponent
+from kfp.components.load_yaml_utilities import load_component_from_file
+from kfp.components.load_yaml_utilities import load_component_from_text
+from kfp.components.load_yaml_utilities import load_component_from_url
+from kfp.dsl.base_component import BaseComponent
+from kfp.dsl.container_component_class import ContainerComponent
+from kfp.dsl.python_component import PythonComponent
+from kfp.dsl.yaml_component import YamlComponent

sdk/python/kfp/components/yaml_component.py → sdk/python/kfp/components/load_yaml_utilities.py

@@ -15,49 +15,12 @@
 
 from typing import Optional, Tuple
 
-from google.protobuf import json_format
-from kfp import components
-from kfp.components import structures
-from kfp.pipeline_spec import pipeline_spec_pb2
+from kfp.dsl import structures
+from kfp.dsl import yaml_component
 import requests
 
 
-class YamlComponent(components.BaseComponent):
-    """A component loaded from a YAML file.
-
-    **Note:** ``YamlComponent`` is not intended to be used to construct components directly. Use ``kfp.components.load_component_from_*()`` instead.
-
-    Attribute:
-        component_spec: Component definition.
-        component_yaml: The yaml string that this component is loaded from.
-    """
-
-    def __init__(
-        self,
-        component_spec: structures.ComponentSpec,
-        component_yaml: str,
-    ):
-        super().__init__(component_spec=component_spec)
-        self.component_yaml = component_yaml
-
-    @property
-    def pipeline_spec(self) -> pipeline_spec_pb2.PipelineSpec:
-        """Returns the pipeline spec of the component."""
-        component_dict = structures.load_documents_from_yaml(
-            self.component_yaml)[0]
-        is_v1 = 'implementation' in set(component_dict.keys())
-        if is_v1:
-            return self.component_spec.to_pipeline_spec()
-        else:
-            return json_format.ParseDict(component_dict,
-                                         pipeline_spec_pb2.PipelineSpec())
-
-    def execute(self, *args, **kwargs):
-        """Not implemented."""
-        raise NotImplementedError
-
-
-def load_component_from_text(text: str) -> YamlComponent:
+def load_component_from_text(text: str) -> yaml_component.YamlComponent:
     """Loads a component from text.
 
     Args:
@@ -66,12 +29,12 @@ def load_component_from_text(text: str) -> YamlComponent:
     Returns:
         Component loaded from YAML.
     """
-    return YamlComponent(
+    return yaml_component.YamlComponent(
         component_spec=structures.ComponentSpec.from_yaml_documents(text),
         component_yaml=text)
 
 
-def load_component_from_file(file_path: str) -> YamlComponent:
+def load_component_from_file(file_path: str) -> yaml_component.YamlComponent:
     """Loads a component from a file.
 
     Args:
@@ -91,9 +54,9 @@ def load_component_from_file(file_path: str) -> YamlComponent:
         return load_component_from_text(component_stream.read())
 
 
-def load_component_from_url(url: str,
-                            auth: Optional[Tuple[str,
-                                                 str]] = None) -> YamlComponent:
+def load_component_from_url(
+        url: str,
+        auth: Optional[Tuple[str, str]] = None) -> yaml_component.YamlComponent:
     """Loads a component from a URL.
 
     Args:

sdk/python/kfp/components/yaml_component_test.py → sdk/python/kfp/components/load_yaml_utilities_test.py

@@ -11,15 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Tests for kfp.components.yaml_component."""
+"""Tests for kfp.dsl.yaml_component."""
 
 import os
 import tempfile
 import textwrap
 import unittest
 
-from kfp.components import structures
-from kfp.components import yaml_component
+from kfp import components
+from kfp.dsl import structures
 
 SAMPLE_YAML = textwrap.dedent("""\
 components:
@@ -84,10 +84,10 @@
 ]
 
 
-class YamlComponentTest(unittest.TestCase):
+class LoadYamlTests(unittest.TestCase):
 
     def test_load_component_from_text(self):
-        component = yaml_component.load_component_from_text(SAMPLE_YAML)
+        component = components.load_component_from_text(SAMPLE_YAML)
         self.assertEqual(component.component_spec.name, 'component-1')
         self.assertEqual(component.component_spec.outputs,
                          {'output1': structures.OutputSpec(type='String')})
@@ -101,7 +101,7 @@ def test_load_component_from_file(self):
             path = os.path.join(tmpdir, 'sample_yaml.yaml')
             with open(path, 'w') as f:
                 f.write(SAMPLE_YAML)
-            component = yaml_component.load_component_from_file(path)
+            component = components.load_component_from_file(path)
         self.assertEqual(component.component_spec.name, 'component-1')
         self.assertEqual(component.component_spec.outputs,
                          {'output1': structures.OutputSpec(type='String')})
@@ -112,7 +112,7 @@ def test_load_component_from_file(self):
 
     def test_load_component_from_url(self):
         component_url = 'https://raw.githubusercontent.com/kubeflow/pipelines/7b49eadf621a9054e1f1315c86f95fb8cf8c17c3/sdk/python/kfp/compiler/test_data/components/identity.yaml'
-        component = yaml_component.load_component_from_url(component_url)
+        component = components.load_component_from_url(component_url)
 
         self.assertEqual(component.component_spec.name, 'identity')
         self.assertEqual(component.component_spec.outputs,

sdk/python/kfp/dsl/__init__.py

@@ -57,31 +57,31 @@
 
 from typing import TypeVar
 
-from kfp.components.component_decorator import component
-from kfp.components.container_component_decorator import container_component
-from kfp.components.for_loop import Collected
-from kfp.components.importer_node import importer
-from kfp.components.pipeline_context import pipeline
-from kfp.components.pipeline_task import PipelineTask
-from kfp.components.placeholders import ConcatPlaceholder
-from kfp.components.placeholders import IfPresentPlaceholder
-from kfp.components.structures import ContainerSpec
-from kfp.components.task_final_status import PipelineTaskFinalStatus
-from kfp.components.tasks_group import Condition
-from kfp.components.tasks_group import ExitHandler
-from kfp.components.tasks_group import ParallelFor
-from kfp.components.types.artifact_types import Artifact
-from kfp.components.types.artifact_types import ClassificationMetrics
-from kfp.components.types.artifact_types import Dataset
-from kfp.components.types.artifact_types import HTML
-from kfp.components.types.artifact_types import Markdown
-from kfp.components.types.artifact_types import Metrics
-from kfp.components.types.artifact_types import Model
-from kfp.components.types.artifact_types import SlicedClassificationMetrics
-from kfp.components.types.type_annotations import InputAnnotation
-from kfp.components.types.type_annotations import InputPath
-from kfp.components.types.type_annotations import OutputAnnotation
-from kfp.components.types.type_annotations import OutputPath
+from kfp.dsl.component_decorator import component
+from kfp.dsl.container_component_decorator import container_component
+from kfp.dsl.for_loop import Collected
+from kfp.dsl.importer_node import importer
+from kfp.dsl.pipeline_context import pipeline
+from kfp.dsl.pipeline_task import PipelineTask
+from kfp.dsl.placeholders import ConcatPlaceholder
+from kfp.dsl.placeholders import IfPresentPlaceholder
+from kfp.dsl.structures import ContainerSpec
+from kfp.dsl.task_final_status import PipelineTaskFinalStatus
+from kfp.dsl.tasks_group import Condition
+from kfp.dsl.tasks_group import ExitHandler
+from kfp.dsl.tasks_group import ParallelFor
+from kfp.dsl.types.artifact_types import Artifact
+from kfp.dsl.types.artifact_types import ClassificationMetrics
+from kfp.dsl.types.artifact_types import Dataset
+from kfp.dsl.types.artifact_types import HTML
+from kfp.dsl.types.artifact_types import Markdown
+from kfp.dsl.types.artifact_types import Metrics
+from kfp.dsl.types.artifact_types import Model
+from kfp.dsl.types.artifact_types import SlicedClassificationMetrics
+from kfp.dsl.types.type_annotations import InputAnnotation
+from kfp.dsl.types.type_annotations import InputPath
+from kfp.dsl.types.type_annotations import OutputAnnotation
+from kfp.dsl.types.type_annotations import OutputPath
 
 # hack: constants and custom type generics have to be defined here to be captured by autodoc and autodocsumm used in ./docs/conf.py
 

sdk/python/kfp/components/base_component.py → sdk/python/kfp/dsl/base_component.py

@@ -16,9 +16,9 @@
 import abc
 from typing import List
 
-from kfp.components import pipeline_task
-from kfp.components import structures
-from kfp.components.types import type_utils
+from kfp.dsl import pipeline_task
+from kfp.dsl import structures
+from kfp.dsl.types import type_utils
 from kfp.pipeline_spec import pipeline_spec_pb2