From 4abd6667c62ca7bc728a66cc68789c34c1ae0ce3 Mon Sep 17 00:00:00 2001 From: Fernando Velasquez Date: Thu, 19 Sep 2024 10:28:59 -0400 Subject: [PATCH] Remove data preparation proto definition. Clean up unused code in data preparation action. --- core/actions/data_preparation.ts | 1 - packages/@dataform/core/BUILD | 9 +- protos/BUILD | 1 - protos/data_preparation.proto | 220 ------------------------------- 4 files changed, 1 insertion(+), 230 deletions(-) delete mode 100644 protos/data_preparation.proto diff --git a/core/actions/data_preparation.ts b/core/actions/data_preparation.ts index 7d8dcac6e..b2e64185c 100644 --- a/core/actions/data_preparation.ts +++ b/core/actions/data_preparation.ts @@ -36,7 +36,6 @@ export class DataPreparation extends ActionBuilder { config.filename = resolveActionsConfigFilename(config.filename, configPath); const dataPreparationAsJson = nativeRequire(config.filename).asJson; - const dataPreparationDefinition = this.parseDataPreparationDefinitionJson(dataPreparationAsJson); // Find targets const targets = this.getTargets(dataPreparationAsJson as { diff --git a/packages/@dataform/core/BUILD b/packages/@dataform/core/BUILD index ea47e3842..f953888fc 100644 --- a/packages/@dataform/core/BUILD +++ b/packages/@dataform/core/BUILD @@ -65,12 +65,6 @@ add_license_header_to_file( to_file = "configs.proto", ) -add_license_header_to_file( - name = "data_preparation_proto_with_license", - from_file = "//protos:data_preparation.proto", - to_file = "data_preparation.proto", -) - add_license_header_to_file( name = "bundle_with_license", from_file = "bundle", @@ -99,8 +93,7 @@ pkg_npm_tar( ":bundle_with_license", ":configs_proto_with_license", ":core_proto_with_license", - ":data_preparation_proto_with_license", ":package.json", - "//:version.bzl" + "//:version.bzl", ], ) diff --git a/protos/BUILD b/protos/BUILD index 8f2723d5d..8e4159e98 100644 --- a/protos/BUILD +++ b/protos/BUILD @@ -9,7 +9,6 @@ proto_library( srcs = [ "configs.proto", "core.proto", - "data_preparation.proto", "evaluation.proto", "execution.proto", "profiles.proto", diff --git a/protos/data_preparation.proto b/protos/data_preparation.proto deleted file mode 100644 index abd9ed275..000000000 --- a/protos/data_preparation.proto +++ /dev/null @@ -1,220 +0,0 @@ -syntax = "proto3"; - -package dataform.dataprep; - -option java_package = "com.dataform.dataprep.protos"; -option java_outer_classname = "DataPreparationMeta"; -option java_multiple_files = true; -option go_package = "github.com/dataform-co/dataform/protos/dataform/dataprep"; - -message DataPreparation { - repeated DataPreparationNode nodes = 1; - DataPreparationGenerated generated = 2; - optional DataPreparationConfiguration configuration = 3; -} - -message DataPreparationNode { - string id = 1; - Source source = 2; - repeated Step steps = 3; - NodeGenerated generated = 4; - optional Destination destination = 5; -} - -message DataPreparationConfiguration { - optional TableReference error_table = 1; - optional int32 error_table_data_expiration_days = 2; - optional LoadConfiguration load = 3; - optional DefaultsConfiguration defaults = 4; -} - -message LoadConfiguration { - oneof mode { - SimpleLoadMode replace = 1; - SimpleLoadMode append = 2; - IncrementalLoadMode maximum = 3; - IncrementalLoadMode unique = 4; - } -} - -message SimpleLoadMode {} - -message IncrementalLoadMode { - string column_name = 1; -} - -message Source { - oneof source { - string node_id = 1; - TableReference table = 2; - Join join = 3; - } -} - -message Destination { - oneof destination { - TableReference table = 1; - } -} - -message Step { - string id = 1; - string description = 2; - oneof definition { - ColumnStep column_step = 3; - FilterStep filter_step = 4; - DropColumnsStep drop_columns_step = 6; - KeepColumnsStep keep_columns_step = 7; - } - - StepGenerated generated = 5; -} - -message ColumnStep { - string column_name = 1; - Expression expression = 2; -} - -enum FilterType { - FILTER_TYPE_UNSPECIFIED = 0; - ROW_FILTER_KEEP_ROWS = 1; - VALIDATION = 2; -} - -message FilterStep { - Expression expression = 1; - optional FilterType filter_type = 2; -} - -message DropColumnsStep { - repeated string columns = 1; -} - -message KeepColumnsStep { - repeated string columns = 1; -} - -message Expression { - oneof expression { - string sql = 1; - } -} - -message Join { - string left_node_id = 1; - string right_node_id = 2; - JoinType join_type = 3; - JoinCondition join_condition = 4; -} - -enum JoinType { - JOIN_TYPE_UNSPECIFIED = 0; - JOIN_TYPE_INNER = 1; - JOIN_TYPE_FULL_OUTER = 2; - JOIN_TYPE_LEFT = 3; - JOIN_TYPE_RIGHT = 4; -} - -message JoinCondition { - oneof condition { - Expression expression = 1; - JoinKeys keys = 2; - } -} - -message JoinKeys { - repeated JoinKey keys = 1; -} - -message JoinKey { - string left_column = 1; - string right_column = 2; -} - -message TableReference { - string project = 1; - string dataset = 2; - string table = 3; -} - -message DataPreparationGenerated { - repeated ValidationError validation_errors = 1; - optional string location = 2; -} - -message NodeGenerated { - repeated Section sections = 1; - repeated string sources = 2; - repeated ValidationError validation_errors = 3; - optional Schema output_schema = 4; - SourceGenerated source_generated = 5; - optional DestinationGenerated destination_generated = 6; -} - -message Section { - SectionType type = 1; - string label = 2; -} - -enum SectionType { - SECTION_TYPE_UNSPECIFIED = 0; - SECTION_UNPARSEABLE = 1; - SECTION_SOURCE_TABLE = 2; - SECTION_SQL = 3; - SECTION_DESTINATION_TABLE = 4; - SECTION_JOIN = 5; -} - -message SourceGenerated { - optional SourceSchema source_schema = 4; -} - -message SourceSchema { - oneof source_schema { - Schema node_schema = 1; - Schema table_schema = 2; - JoinSchema join_schema = 3; - } -} - -message JoinSchema { - Schema left_schema = 1; - Schema right_schema = 2; -} - -message DestinationGenerated { - optional Schema schema = 1; -} - -message StepGenerated { - repeated string source_columns = 1; - repeated ValidationError validation_errors = 2; -} - -message Schema { - repeated Field field = 1; -} - -message Field { - string name = 1; - optional string type = 2; - optional string mode = 3; - repeated Field fields = 4; -} - -message ValidationError { - ValidationErrorLevel level = 1; - string description = 2; -} - -enum ValidationErrorLevel { - LEVEL_UNSPECIFIED = 0; - LEVEL_WARN = 1; - LEVEL_ERROR = 2; - LEVEL_FATAL = 3; -} - -message DefaultsConfiguration { - string project = 1; - string dataset = 2; -}