feat(sdk): add default schema_version to pipeline (#6366)

* feat(sdk): add default schema_version to pipeline * sync api for go * Fix tests and address comments * Bump pipeline_spec version * Fix v1 tests * rebase to master * sync api for go * Fix tests and address comments * Bump pipeline_spec version * Fix v1 tests
kubeflow · Aug 24, 2021 · 937cacd · 937cacd
1 parent 7634adb
commit 937cacd
Show file tree

Hide file tree

Showing 35 changed files with 226 additions and 137 deletions.
diff --git a/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml b/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml
@@ -101,7 +101,7 @@ spec:
           "INT"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters":
           {"output_parameter_one": {"type": "INT", "path": "/tmp/outputs/output_parameter_one/data"}},
           "outputArtifacts": {"output_dataset_one": {"schemaTitle": "system.Dataset",
-          "instanceSchema": "", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'}
+          "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'}
       envFrom:
       - configMapRef: {name: metadata-grpc-configmap, optional: true}
       image: python:3.7
@@ -200,9 +200,9 @@ spec:
       - {name: KFP_V2_IMAGE, value: 'python:3.7'}
       - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type":
           "INT"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data",
-          "schemaTitle": "system.Dataset", "instanceSchema": ""}}, "outputParameters":
+          "schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters":
           {}, "outputArtifacts": {"model": {"schemaTitle": "system.Model", "instanceSchema":
-          "", "metadataPath": "/tmp/outputs/model/data"}}}'}
+          "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/model/data"}}}'}
       envFrom:
       - configMapRef: {name: metadata-grpc-configmap, optional: true}
       image: python:3.7

diff --git a/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml b/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml
@@ -101,7 +101,7 @@ spec:
           "INT"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters":
           {"output_parameter_one": {"type": "INT", "path": "/tmp/outputs/output_parameter_one/data"}},
           "outputArtifacts": {"output_dataset_one": {"schemaTitle": "system.Dataset",
-          "instanceSchema": "", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'}
+          "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'}
       envFrom:
       - configMapRef: {name: metadata-grpc-configmap, optional: true}
       image: python:3.7
@@ -200,9 +200,9 @@ spec:
       - {name: KFP_V2_IMAGE, value: 'python:3.7'}
       - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type":
           "INT"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data",
-          "schemaTitle": "system.Dataset", "instanceSchema": ""}}, "outputParameters":
+          "schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters":
           {}, "outputArtifacts": {"model": {"schemaTitle": "system.Model", "instanceSchema":
-          "", "metadataPath": "/tmp/outputs/model/data"}}}'}
+          "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/model/data"}}}'}
       envFrom:
       - configMapRef: {name: metadata-grpc-configmap, optional: true}
       image: python:3.7

diff --git a/sdk/python/kfp/compiler/v2_compat.py b/sdk/python/kfp/compiler/v2_compat.py
@@ -153,6 +153,7 @@ def update_op(op: dsl.ContainerOp,
         "metadataPath": op.input_artifact_paths[artifact_name],
         "schemaTitle": spec.artifact_type.schema_title,
         "instanceSchema": spec.artifact_type.instance_schema,
+        "schemaVersion": spec.artifact_type.schema_version,
     }
     runtime_info["inputArtifacts"][artifact_name] = artifact_info
 
@@ -173,6 +174,7 @@ def update_op(op: dsl.ContainerOp,
         # Type used to register output artifacts.
         "schemaTitle": spec.artifact_type.schema_title,
         "instanceSchema": spec.artifact_type.instance_schema,
+        "schemaVersion": spec.artifact_type.schema_version,
         # File used to write out the registered artifact ID.
         "metadataPath": op.file_outputs[artifact_name],
     }

diff --git a/sdk/python/kfp/dsl/component_spec_test.py b/sdk/python/kfp/dsl/component_spec_test.py
@@ -64,7 +64,8 @@ def test_build_component_spec_from_structure(self):
             'artifacts': {
                 'input1': {
                     'artifactType': {
-                        'schemaTitle': 'system.Dataset'
+                        'schemaTitle': 'system.Dataset',
+                        'schemaVersion': '0.0.1'
                     }
                 }
             },
@@ -81,7 +82,8 @@ def test_build_component_spec_from_structure(self):
             'artifacts': {
                 'output1': {
                     'artifactType': {
-                        'schemaTitle': 'system.Model'
+                        'schemaTitle': 'system.Model',
+                        'schemaVersion': '0.0.1'
                     }
                 }
             }
@@ -108,7 +110,8 @@ def test_build_component_spec_from_structure(self):
                   'artifacts': {
                       'input1': {
                           'artifactType': {
-                              'schemaTitle': 'system.Dataset'
+                              'schemaTitle': 'system.Dataset',
+                              'schemaVersion':'0.0.1'
                           }
                       }
                   },
@@ -133,7 +136,8 @@ def test_build_component_spec_from_structure(self):
                   'artifacts': {
                       'pipelineparam--input1': {
                           'artifactType': {
-                              'schemaTitle': 'system.Dataset'
+                              'schemaTitle': 'system.Dataset',
+                              'schemaVersion':'0.0.1'
                           }
                       }
                   },
@@ -182,7 +186,8 @@ def test_build_component_outputs_spec(self):
             'artifacts': {
                 'output1': {
                     'artifactType': {
-                        'schemaTitle': 'system.Dataset'
+                        'schemaTitle': 'system.Dataset',
+                        'schemaVersion': '0.0.1'
                     }
                 }
             },

diff --git a/sdk/python/kfp/dsl/type_utils.py b/sdk/python/kfp/dsl/type_utils.py
@@ -27,4 +27,4 @@
 get_artifact_type_schema = type_utils.type_utils.get_artifact_type_schema
 get_parameter_type = type_utils.get_parameter_type
 get_parameter_type_field_name = type_utils.get_parameter_type_field_name
-get_input_artifact_type_schema = type_utils.type_utils.get_input_artifact_type_schema
+get_input_artifact_type_schema = type_utils.type_utils.get_input_artifact_type_schema
diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/experimental_v2_component.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/experimental_v2_component.json
@@ -123,7 +123,7 @@
       }
     },
     "schemaVersion": "2.0.0",
-    "sdkVersion": "kfp-1.6.6"
+    "sdkVersion": "kfp-1.7.0"
   },
   "runtimeConfig": {
     "gcsOutputDirectory": "dummy_root",

diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json
@@ -14,12 +14,14 @@
           "artifacts": {
             "output_dataset_one": {
               "artifactType": {
-                "schemaTitle": "system.Dataset"
+                "schemaTitle": "system.Dataset",
+                "schemaVersion": "0.0.1"
               }
             },
             "output_dataset_two_path": {
               "artifactType": {
-                "schemaTitle": "system.Dataset"
+                "schemaTitle": "system.Dataset",
+                "schemaVersion": "0.0.1"
               }
             }
           },
@@ -45,12 +47,14 @@
           "artifacts": {
             "dataset_one_path": {
               "artifactType": {
-                "schemaTitle": "system.Dataset"
+                "schemaTitle": "system.Dataset",
+                "schemaVersion": "0.0.1"
               }
             },
             "dataset_two": {
               "artifactType": {
-                "schemaTitle": "system.Dataset"
+                "schemaTitle": "system.Dataset",
+                "schemaVersion": "0.0.1"
               }
             }
           },
@@ -76,7 +80,8 @@
           "artifacts": {
             "model": {
               "artifactType": {
-                "schemaTitle": "system.Model"
+                "schemaTitle": "system.Model",
+                "schemaVersion": "0.0.1"
               }
             }
           }

diff --git a/...hon/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json b/...hon/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json
@@ -57,7 +57,8 @@
           "artifacts": {
             "Output": {
               "artifactType": {
-                "schemaTitle": "system.Dataset"
+                "schemaTitle": "system.Dataset",
+                "schemaVersion": "0.0.1"
               }
             }
           }
@@ -69,7 +70,8 @@
           "artifacts": {
             "artifact": {
               "artifactType": {
-                "schemaTitle": "system.Dataset"
+                "schemaTitle": "system.Dataset",
+                "schemaVersion": "0.0.1"
               }
             }
           }
@@ -78,12 +80,14 @@
           "artifacts": {
             "metrics": {
               "artifactType": {
-                "schemaTitle": "system.Metrics"
+                "schemaTitle": "system.Metrics",
+                "schemaVersion": "0.0.1"
               }
             },
             "model": {
               "artifactType": {
-                "schemaTitle": "system.Model"
+                "schemaTitle": "system.Model",
+                "schemaVersion": "0.0.1"
               }
             }
           },
@@ -316,7 +320,8 @@
         "artifacts": {
           "output-named-tuple-metrics": {
             "artifactType": {
-              "schemaTitle": "system.Metrics"
+              "schemaTitle": "system.Metrics",
+              "schemaVersion": "0.0.1"
             }
           }
         }

diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json
@@ -152,7 +152,7 @@
       }
     },
     "schemaVersion": "2.0.0",
-    "sdkVersion": "kfp-1.6.6"
+    "sdkVersion": "kfp-1.7.0"
   },
   "runtimeConfig": {
     "gcsOutputDirectory": "dummy_root"

diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json
@@ -57,7 +57,7 @@
       }
     },
     "schemaVersion": "2.0.0",
-    "sdkVersion": "kfp-1.6.6"
+    "sdkVersion": "kfp-1.7.0"
   },
   "runtimeConfig": {
     "gcsOutputDirectory": "dummy_root"

diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json
@@ -275,7 +275,7 @@
       }
     },
     "schemaVersion": "2.0.0",
-    "sdkVersion": "kfp-1.6.6"
+    "sdkVersion": "kfp-1.7.0"
   },
   "runtimeConfig": {
     "gcsOutputDirectory": "dummy_root",

diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json
@@ -72,7 +72,8 @@
           "artifacts": {
             "artifact": {
               "artifactType": {
-                "schemaTitle": "system.Dataset"
+                "schemaTitle": "system.Dataset",
+                "schemaVersion": "0.0.1"
               }
             }
           }
@@ -91,7 +92,8 @@
           "artifacts": {
             "artifact": {
               "artifactType": {
-                "schemaTitle": "system.Dataset"
+                "schemaTitle": "system.Dataset",
+                "schemaVersion": "0.0.1"
               }
             }
           }
@@ -103,7 +105,8 @@
           "artifacts": {
             "dataset": {
               "artifactType": {
-                "schemaTitle": "system.Dataset"
+                "schemaTitle": "system.Dataset",
+                "schemaVersion": "0.0.1"
               }
             }
           }
@@ -112,7 +115,8 @@
           "artifacts": {
             "model": {
               "artifactType": {
-                "schemaTitle": "system.Model"
+                "schemaTitle": "system.Model",
+                "schemaVersion": "0.0.1"
               }
             }
           },
@@ -129,7 +133,8 @@
           "artifacts": {
             "dataset": {
               "artifactType": {
-                "schemaTitle": "system.Dataset"
+                "schemaTitle": "system.Dataset",
+                "schemaVersion": "0.0.1"
               }
             }
           }
@@ -138,7 +143,8 @@
           "artifacts": {
             "model": {
               "artifactType": {
-                "schemaTitle": "system.Model"
+                "schemaTitle": "system.Model",
+                "schemaVersion": "0.0.1"
               }
             }
           },
@@ -160,7 +166,8 @@
               }
             },
             "typeSchema": {
-              "schemaTitle": "system.Dataset"
+              "schemaTitle": "system.Dataset",
+              "schemaVersion": "0.0.1"
             }
           }
         },
@@ -170,7 +177,8 @@
               "runtimeParameter": "uri"
             },
             "typeSchema": {
-              "schemaTitle": "system.Dataset"
+              "schemaTitle": "system.Dataset",
+              "schemaVersion": "0.0.1"
             }
           }
         },

diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_output.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_output.json
@@ -226,7 +226,7 @@
       }
     },
     "schemaVersion": "2.0.0",
-    "sdkVersion": "kfp-1.6.6"
+    "sdkVersion": "kfp-1.7.0"
   },
   "runtimeConfig": {
     "gcsOutputDirectory": "dummy_root"

diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_parameter.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_parameter.json
@@ -191,7 +191,7 @@
       }
     },
     "schemaVersion": "2.0.0",
-    "sdkVersion": "kfp-1.6.6"
+    "sdkVersion": "kfp-1.7.0"
   },
   "runtimeConfig": {
     "gcsOutputDirectory": "dummy_root",

diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_static.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_static.json
@@ -174,7 +174,7 @@
       }
     },
     "schemaVersion": "2.0.0",
-    "sdkVersion": "kfp-1.6.6"
+    "sdkVersion": "kfp-1.7.0"
   },
   "runtimeConfig": {
     "gcsOutputDirectory": "dummy_root"

diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json
@@ -678,7 +678,7 @@
       }
     },
     "schemaVersion": "2.0.0",
-    "sdkVersion": "kfp-1.6.6"
+    "sdkVersion": "kfp-1.7.0"
   },
   "runtimeConfig": {
     "gcsOutputDirectory": "dummy_root",