Skip to content

Commit

Permalink
rename TYPE_NAME and VERSION artifact attributes
Browse files Browse the repository at this point in the history
  • Loading branch information
connor-mccarthy committed Aug 24, 2022
1 parent 459e535 commit f3e042c
Show file tree
Hide file tree
Showing 11 changed files with 86 additions and 83 deletions.
4 changes: 2 additions & 2 deletions sdk/python/kfp/compiler/pipeline_spec_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -915,8 +915,8 @@ def populate_metrics_in_dag_outputs(
if artifact_spec.artifact_type.WhichOneof(
'kind'
) == 'schema_title' and artifact_spec.artifact_type.schema_title in [
artifact_types.Metrics.TYPE_NAME,
artifact_types.ClassificationMetrics.TYPE_NAME,
artifact_types.Metrics.schema_title,
artifact_types.ClassificationMetrics.schema_title,
]:
unique_output_name = '{}-{}'.format(task.name, output_name)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@


class VertexModel(dsl.Artifact):
TYPE_NAME = 'google.VertexModel'
schema_title = 'google.VertexModel'


producer_op = components.load_component_from_text("""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

class VertexDataset(dsl.Artifact):
"""An artifact representing a GCPC Vertex Dataset."""
TYPE_NAME = 'google.VertexDataset'
schema_title = 'google.VertexDataset'


consumer_op = components.load_component_from_text("""
Expand Down
16 changes: 8 additions & 8 deletions sdk/python/kfp/components/component_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,24 +142,24 @@ def _annotation_to_type_struct(annotation):
return type_struct

if issubclass(annotation, artifact_types.Artifact
) and not annotation.TYPE_NAME.startswith('system.'):
) and not annotation.schema_title.startswith('system.'):
# For artifact classes not under the `system` namespace,
# use its TYPE_NAME as-is.
type_name = annotation.TYPE_NAME
# use its schema_title as-is.
schema_title = annotation.schema_title
else:
type_name = str(annotation.__name__)
schema_title = str(annotation.__name__)

elif hasattr(annotation,
'__forward_arg__'): # Handling typing.ForwardRef('Type_name')
type_name = str(annotation.__forward_arg__)
schema_title = str(annotation.__forward_arg__)
else:
type_name = str(annotation)
schema_title = str(annotation)

# It's also possible to get the converter by type name
type_struct = type_utils.get_canonical_type_name_for_type(type_name)
type_struct = type_utils.get_canonical_type_name_for_type(schema_title)
if type_struct:
return type_struct
return type_name
return schema_title


def _maybe_make_unique(name: str, names: List[str]):
Expand Down
8 changes: 4 additions & 4 deletions sdk/python/kfp/components/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ def _write_output_artifact_payload(self, name: str, value: Any):

# TODO: extract to a util
@classmethod
def _get_short_type_name(cls, type_name: str) -> str:
def _get_short_type_name(cls, schema_title: str) -> str:
"""Extracts the short form type name.
This method is used for looking up serializer for a given type.
Expand All @@ -146,17 +146,17 @@ def _get_short_type_name(cls, type_name: str) -> str:
str -> str
Args:
type_name: The original type name.
schema_title: The original type name.
Returns:
The short form type name or the original name if pattern doesn't match.
"""
import re
match = re.match('(typing\.)?(?P<type>\w+)(?:\[.+\])?', type_name)
match = re.match('(typing\.)?(?P<type>\w+)(?:\[.+\])?', schema_title)
if match:
return match.group('type')
else:
return type_name
return schema_title

# TODO: merge with type_utils.is_parameter_type
@classmethod
Expand Down
13 changes: 7 additions & 6 deletions sdk/python/kfp/components/for_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
ItemList = List[Union[int, float, str, Dict[str, Any]]]


def _get_loop_item_type(type_name: str) -> Optional[str]:
def _get_loop_item_type(schema_title: str) -> Optional[str]:
"""Extracts the loop item type.
This method is used for extract the item type from a collection type.
Expand All @@ -34,19 +34,20 @@ def _get_loop_item_type(type_name: str) -> Optional[str]:
str -> None
Args:
type_name: The collection type name, like `List`, Sequence`, etc.
schema_title: The collection type name, like `List`, Sequence`, etc.
Returns:
The collection item type or None if no match found.
"""
match = re.match('(typing\.)?(?:\w+)(?:\[(?P<item_type>.+)\])', type_name)
match = re.match('(typing\.)?(?:\w+)(?:\[(?P<item_type>.+)\])',
schema_title)
if match:
return match.group('item_type').lstrip().rstrip()
else:
return None


def _get_subvar_type(type_name: str) -> Optional[str]:
def _get_subvar_type(schema_title: str) -> Optional[str]:
"""Extracts the subvar type.
This method is used for extract the value type from a dictionary type.
Expand All @@ -56,14 +57,14 @@ def _get_subvar_type(type_name: str) -> Optional[str]:
typing.Mapping[str, float] -> float
Args:
type_name: The dictionary type.
schema_title: The dictionary type.
Returns:
The dictionary value type or None if no match found.
"""
match = re.match(
'(typing\.)?(?:\w+)(?:\[\s*(?:\w+)\s*,\s*(?P<value_type>.+)\])',
type_name)
schema_title)
if match:
return match.group('value_type').lstrip().rstrip()
else:
Expand Down
12 changes: 6 additions & 6 deletions sdk/python/kfp/components/importer_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,24 +55,24 @@ def pipeline_with_importer():
train(dataset=importer1.output)
"""
if issubclass(artifact_class, artifact_types.Artifact
) and not artifact_class.TYPE_NAME.startswith('system.'):
) and not artifact_class.schema_title.startswith('system.'):
# For artifact classes not under the `system` namespace,
# use its TYPE_NAME as-is.
type_name = artifact_class.TYPE_NAME
# use its schema_title as-is.
schema_title = artifact_class.schema_title
else:
type_name = artifact_class.__name__
schema_title = artifact_class.__name__

component_spec = structures.ComponentSpec(
name='importer',
implementation=structures.Implementation(
importer=structures.ImporterSpec(
artifact_uri=placeholders.InputValuePlaceholder(
INPUT_KEY).to_placeholder_string(),
type_schema=artifact_class.TYPE_NAME,
type_schema=artifact_class.schema_title,
reimport=reimport,
metadata=metadata)),
inputs={INPUT_KEY: structures.InputSpec(type='String')},
outputs={OUTPUT_KEY: structures.OutputSpec(type=type_name)},
outputs={OUTPUT_KEY: structures.OutputSpec(type=schema_title)},
)

importer = importer_component.ImporterComponent(
Expand Down
24 changes: 12 additions & 12 deletions sdk/python/kfp/components/types/artifact_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
These are only compatible with v2 Pipelines.
"""

from typing import Dict, List, Optional
from typing import Dict, List, Optional, Type

_GCS_LOCAL_MOUNT_PREFIX = '/gcs/'
_MINIO_LOCAL_MOUNT_PREFIX = '/minio/'
Expand Down Expand Up @@ -63,8 +63,8 @@ def my_pipeline():
Note: Other artifacts are used similarly to the usage of ``Artifact`` in the example above (within ``Input[]`` and ``Output[]``).
"""
TYPE_NAME = 'system.Artifact'
VERSION = '0.0.1'
schema_title = 'system.Artifact'
schema_version = '0.0.1'

def __init__(self,
name: Optional[str] = None,
Expand Down Expand Up @@ -110,7 +110,7 @@ class Model(Artifact):
uri: The model's location on disk or cloud storage.
metadata: Arbitrary key-value pairs about the model.
"""
TYPE_NAME = 'system.Model'
schema_title = 'system.Model'

def __init__(self,
name: Optional[str] = None,
Expand Down Expand Up @@ -141,7 +141,7 @@ class Dataset(Artifact):
uri: The dataset's location on disk or cloud storage.
metadata: Arbitrary key-value pairs about the dataset.
"""
TYPE_NAME = 'system.Dataset'
schema_title = 'system.Dataset'

def __init__(self,
name: Optional[str] = None,
Expand All @@ -158,7 +158,7 @@ class Metrics(Artifact):
uri: The metrics artifact's location on disk or cloud storage.
metadata: Key-value scalar metrics.
"""
TYPE_NAME = 'system.Metrics'
schema_title = 'system.Metrics'

def __init__(self,
name: Optional[str] = None,
Expand All @@ -184,7 +184,7 @@ class ClassificationMetrics(Artifact):
uri: The metrics artifact's location on disk or cloud storage.
metadata: The key-value scalar metrics.
"""
TYPE_NAME = 'system.ClassificationMetrics'
schema_title = 'system.ClassificationMetrics'

def __init__(self,
name: Optional[str] = None,
Expand Down Expand Up @@ -347,7 +347,7 @@ class SlicedClassificationMetrics(Artifact):
metadata: Arbitrary key-value pairs about the metrics artifact.
"""

TYPE_NAME = 'system.SlicedClassificationMetrics'
schema_title = 'system.SlicedClassificationMetrics'

def __init__(self,
name: Optional[str] = None,
Expand Down Expand Up @@ -472,7 +472,7 @@ class HTML(Artifact):
uri: The HTML file's location on disk or cloud storage.
metadata: Arbitrary key-value pairs about the HTML file.
"""
TYPE_NAME = 'system.HTML'
schema_title = 'system.HTML'

def __init__(self,
name: Optional[str] = None,
Expand All @@ -489,7 +489,7 @@ class Markdown(Artifact):
uri: The markdown file's location on disk or cloud storage.
metadata: Arbitrary key-value pairs about the markdown file.
"""
TYPE_NAME = 'system.Markdown'
schema_title = 'system.Markdown'

def __init__(self,
name: Optional[str] = None,
Expand All @@ -498,8 +498,8 @@ def __init__(self,
super().__init__(uri=uri, name=name, metadata=metadata)


_SCHEMA_TITLE_TO_TYPE: Dict[str, Artifact] = {
x.TYPE_NAME: x for x in [
_SCHEMA_TITLE_TO_TYPE: Dict[str, Type[Artifact]] = {
x.schema_title: x for x in [
Artifact,
Model,
Dataset,
Expand Down
8 changes: 4 additions & 4 deletions sdk/python/kfp/components/types/type_annotations.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ def maybe_strip_optional_from_annotation(annotation: T) -> T:
return annotation


def get_short_type_name(type_name: str) -> str:
def get_short_type_name(schema_title: str) -> str:
"""Extracts the short form type name.
This method is used for looking up serializer for a given type.
Expand All @@ -242,13 +242,13 @@ def get_short_type_name(type_name: str) -> str:
str -> str
Args:
type_name: The original type name.
schema_title: The original type name.
Returns:
The short form type name or the original name if pattern doesn't match.
"""
match = re.match('(typing\.)?(?P<type>\w+)(?:\[.+\])?', type_name)
match = re.match('(typing\.)?(?P<type>\w+)(?:\[.+\])?', schema_title)
if match:
return match.group('type')
else:
return type_name
return schema_title
Loading

0 comments on commit f3e042c

Please sign in to comment.