Merge pull request #125 from JoaoRigonNeto/feature/schema_transform_f…

…unction

Adding transform functions to schema declaration

README.md

@@ -38,6 +38,7 @@ resource "pinot_schema" "block_schema" {
     name        = "block_timestamp",
     format      = "1:MILLISECONDS:EPOCH",
     granularity = "1:MILLISECONDS",
+    transform_function = "ago('PT3H')"
   }]
   dimension_field_specs = [{
     name      = "block_number",
@@ -47,7 +48,8 @@ resource "pinot_schema" "block_schema" {
     {
       name      = "block_hash",
       data_type = "STRING",
-      not_null  = true
+      not_null  = true,
+      transform_function = "jsonPathString(block, '$.block_hash')"
   }]
   metric_field_specs = [{
     name      = "block_difficulty",

docs/data-sources/schemas.md

@@ -47,6 +47,7 @@ Optional:
 - `format` (String) The format of the date time.
 - `granularity` (String) The granularity of the date time.
 - `not_null` (Boolean) Whether the dimension is not null.
+- `transform_function` (String) Transform function for specific field.
 
 
 <a id="nestedatt--schemas--dimension_field_specs"></a>
@@ -61,6 +62,7 @@ Optional:
 
 - `not_null` (Boolean) Whether the dimension is not null.
 - `single_value_field` (Boolean) Whether the dimension is a single value field.
+- `transform_function` (String) Transform function for specific field.
 
 
 <a id="nestedatt--schemas--metric_field_specs"></a>
@@ -74,3 +76,4 @@ Required:
 Optional:
 
 - `not_null` (Boolean) Whether the dimension is not null.
+- `transform_function` (String) Transform function for specific field.

docs/resources/schema.md

@@ -21,6 +21,7 @@ resource "pinot_schema" "block_schema" {
     not_null    = false,
     format      = "1:MILLISECONDS:EPOCH",
     granularity = "1:MILLISECONDS",
+    transform_function = "ago('PT3H')"
   }]
   enable_column_based_null_handling = false
   dimension_field_specs = [{
@@ -31,7 +32,8 @@ resource "pinot_schema" "block_schema" {
     {
       name      = "block_hash",
       data_type = "STRING",
-      not_null  = true
+      not_null  = true,
+      transform_function = "jsonPathString(block, '$.block_hash')"
   }]
   metric_field_specs = [{
     name      = "block_difficulty",
@@ -69,6 +71,7 @@ Optional:
 - `format` (String) The format of the date time.
 - `granularity` (String) The granularity of the date time.
 - `not_null` (Boolean) Whether the dimension is not null.
+- `transform_function` (String) Transform function for specific field.
 
 
 <a id="nestedatt--dimension_field_specs"></a>
@@ -83,6 +86,7 @@ Optional:
 
 - `not_null` (Boolean) Whether the dimension is not null.
 - `single_value_field` (Boolean) Whether the dimension is a single value field.
+- `transform_function` (String) Transform function for specific field.
 
 
 <a id="nestedatt--metric_field_specs"></a>
@@ -96,3 +100,4 @@ Required:
 Optional:
 
 - `not_null` (Boolean) Whether the dimension is not null.
+- `transform_function` (String) Transform function for specific field.

examples/resources/pinot_schema/resource.tf

@@ -3,11 +3,10 @@ resource "pinot_schema" "block_schema" {
   date_time_field_specs = [{
     data_type   = "LONG",
     name        = "block_timestamp",
-    not_null    = false,
     format      = "1:MILLISECONDS:EPOCH",
     granularity = "1:MILLISECONDS",
+    transform_function = "ago('PT3H')"
   }]
-  enable_column_based_null_handling = false
   dimension_field_specs = [{
     name      = "block_number",
     data_type = "INT",
@@ -16,7 +15,8 @@ resource "pinot_schema" "block_schema" {
     {
       name      = "block_hash",
       data_type = "STRING",
-      not_null  = true
+      not_null  = true,
+      transform_function = "jsonPathString(block, '$.block_hash')"
   }]
   metric_field_specs = [{
     name      = "block_difficulty",

examples/table_schema/main.tf

@@ -19,6 +19,7 @@ provider "pinot" {
 #     not_null    = false,
 #     format      = "1:MILLISECONDS:EPOCH",
 #     granularity = "1:MILLISECONDS",
+#     transform_function = "ago('PT3H')"
 #   }]
 #   enable_column_based_null_handling = false
 #   dimension_field_specs = [{
@@ -29,7 +30,8 @@ provider "pinot" {
 #     {
 #       name      = "block_hash",
 #       data_type = "STRING",
-#       not_null  = true
+#       not_null  = true,
+#       transform_function = "jsonPathString(block, '$.block_hash')"
 #   }]
 #   metric_field_specs = [{
 #     name      = "block_difficulty",

internal/provider/table_schema_data_source.go

@@ -27,12 +27,16 @@ func DimensionFieldSpecsFromFieldSpecs(fieldSpecs []model.FieldSpec) []dimension
 	var dimensionFieldSpecs []dimensionFieldSpec
 
 	for _, fieldSpec := range fieldSpecs {
-		dimensionFieldSpecs = append(dimensionFieldSpecs, dimensionFieldSpec{
+		dimensionFieldSpec := dimensionFieldSpec{
 			Name:             fieldSpec.Name,
 			DataType:         fieldSpec.DataType,
 			NotNull:          basetypes.NewBoolPointerValue(fieldSpec.NotNull),
 			SingleValueField: basetypes.NewBoolPointerValue(fieldSpec.SingleValueField),
-		})
+		}
+		if fieldSpec.TransformFunction != "" {
+			dimensionFieldSpec.TransformFunction = basetypes.NewStringValue(fieldSpec.TransformFunction)
+		}
+		dimensionFieldSpecs = append(dimensionFieldSpecs, dimensionFieldSpec)
 	}
 
 	return dimensionFieldSpecs
@@ -42,11 +46,15 @@ func MetricFieldSpecsFromFieldSpecs(fieldSpecs []model.FieldSpec) []metricFieldS
 	var metricFieldSpecs []metricFieldSpec
 
 	for _, fieldSpec := range fieldSpecs {
-		metricFieldSpecs = append(metricFieldSpecs, metricFieldSpec{
+		metricFieldSpec := metricFieldSpec{
 			Name:     fieldSpec.Name,
 			DataType: fieldSpec.DataType,
 			NotNull:  basetypes.NewBoolPointerValue(fieldSpec.NotNull),
-		})
+		}
+		if fieldSpec.TransformFunction != "" {
+			metricFieldSpec.TransformFunction = basetypes.NewStringValue(fieldSpec.TransformFunction)
+		}
+		metricFieldSpecs = append(metricFieldSpecs, metricFieldSpec)
 	}
 
 	return metricFieldSpecs
@@ -56,13 +64,17 @@ func DateTimeFieldSpecsFromFieldSpecs(fieldSpecs []model.FieldSpec) []dateTimeFi
 	var dateTimeFieldSpecs []dateTimeFieldSpec
 
 	for _, fieldSpec := range fieldSpecs {
-		dateTimeFieldSpecs = append(dateTimeFieldSpecs, dateTimeFieldSpec{
+		dateTimeFieldSpec := dateTimeFieldSpec{
 			Name:        fieldSpec.Name,
 			DataType:    fieldSpec.DataType,
 			NotNull:     basetypes.NewBoolPointerValue(fieldSpec.NotNull),
 			Format:      fieldSpec.Format,
 			Granularity: fieldSpec.Granularity,
-		})
+		}
+		if fieldSpec.TransformFunction != "" {
+			dateTimeFieldSpec.TransformFunction = basetypes.NewStringValue(fieldSpec.TransformFunction)
+		}
+		dateTimeFieldSpecs = append(dateTimeFieldSpecs, dateTimeFieldSpec)
 	}
 
 	return dateTimeFieldSpecs
@@ -146,7 +158,10 @@ func (d *schemasDataSource) Schema(_ context.Context, req datasource.SchemaReque
 										Description: "Whether the dimension is a single value field.",
 										Optional:    true,
 									},
-								},
+									"transform_function": schema.StringAttribute{
+										Description: "Transform function for specific field.",
+										Optional:    true,
+									}},
 							},
 						},
 						"metric_field_specs": schema.ListNestedAttribute{
@@ -166,6 +181,10 @@ func (d *schemasDataSource) Schema(_ context.Context, req datasource.SchemaReque
 										Description: "Whether the dimension is not null.",
 										Optional:    true,
 									},
+									"transform_function": schema.StringAttribute{
+										Description: "Transform function for specific field.",
+										Optional:    true,
+									},
 								},
 							},
 						},
@@ -194,6 +213,10 @@ func (d *schemasDataSource) Schema(_ context.Context, req datasource.SchemaReque
 										Description: "The granularity of the date time.",
 										Optional:    true,
 									},
+									"transform_function": schema.StringAttribute{
+										Description: "Transform function for specific field.",
+										Optional:    true,
+									},
 								},
 							},
 						},

internal/provider/table_schema_resource.go

@@ -29,24 +29,27 @@ func NewTableSchemaResource() resource.Resource {
 }
 
 type metricFieldSpec struct {
-	Name     string              `tfsdk:"name"`
-	DataType string              `tfsdk:"data_type"`
-	NotNull  basetypes.BoolValue `tfsdk:"not_null"`
+	Name              string                `tfsdk:"name"`
+	DataType          string                `tfsdk:"data_type"`
+	NotNull           basetypes.BoolValue   `tfsdk:"not_null"`
+	TransformFunction basetypes.StringValue `tfsdk:"transform_function"`
 }
 
 type dimensionFieldSpec struct {
-	Name             string              `tfsdk:"name"`
-	DataType         string              `tfsdk:"data_type"`
-	NotNull          basetypes.BoolValue `tfsdk:"not_null"`
-	SingleValueField basetypes.BoolValue `tfsdk:"single_value_field"`
+	Name              string                `tfsdk:"name"`
+	DataType          string                `tfsdk:"data_type"`
+	NotNull           basetypes.BoolValue   `tfsdk:"not_null"`
+	SingleValueField  basetypes.BoolValue   `tfsdk:"single_value_field"`
+	TransformFunction basetypes.StringValue `tfsdk:"transform_function"`
 }
 
 type dateTimeFieldSpec struct {
-	Name        string              `tfsdk:"name"`
-	DataType    string              `tfsdk:"data_type"`
-	NotNull     basetypes.BoolValue `tfsdk:"not_null"`
-	Format      string              `tfsdk:"format"`
-	Granularity string              `tfsdk:"granularity"`
+	Name              string                `tfsdk:"name"`
+	DataType          string                `tfsdk:"data_type"`
+	NotNull           basetypes.BoolValue   `tfsdk:"not_null"`
+	Format            string                `tfsdk:"format"`
+	Granularity       string                `tfsdk:"granularity"`
+	TransformFunction basetypes.StringValue `tfsdk:"transform_function"`
 }
 
 type tableSchemaResourceModel struct {
@@ -112,6 +115,10 @@ func (t *tableSchemaResource) Schema(_ context.Context, _ resource.SchemaRequest
 							Description: "Whether the dimension is a single value field.",
 							Optional:    true,
 						},
+						"transform_function": schema.StringAttribute{
+							Description: "Transform function for specific field.",
+							Optional:    true,
+						},
 					},
 				},
 			},
@@ -132,6 +139,10 @@ func (t *tableSchemaResource) Schema(_ context.Context, _ resource.SchemaRequest
 							Description: "Whether the dimension is not null.",
 							Optional:    true,
 						},
+						"transform_function": schema.StringAttribute{
+							Description: "Transform function for specific field.",
+							Optional:    true,
+						},
 					},
 				},
 			},
@@ -160,6 +171,10 @@ func (t *tableSchemaResource) Schema(_ context.Context, _ resource.SchemaRequest
 							Description: "The granularity of the date time.",
 							Optional:    true,
 						},
+						"transform_function": schema.StringAttribute{
+							Description: "Transform function for specific field.",
+							Optional:    true,
+						},
 					},
 				},
 			},
@@ -319,10 +334,11 @@ func toDimensionFieldSpecs(fieldSpecs []dimensionFieldSpec) []model.FieldSpec {
 	var pinotFieldSpecs []model.FieldSpec
 	for _, fs := range fieldSpecs {
 		pinotFieldSpecs = append(pinotFieldSpecs, model.FieldSpec{
-			Name:             fs.Name,
-			DataType:         fs.DataType,
-			NotNull:          fs.NotNull.ValueBoolPointer(),
-			SingleValueField: fs.SingleValueField.ValueBoolPointer(),
+			Name:              fs.Name,
+			DataType:          fs.DataType,
+			NotNull:           fs.NotNull.ValueBoolPointer(),
+			SingleValueField:  fs.SingleValueField.ValueBoolPointer(),
+			TransformFunction: fs.TransformFunction.ValueString(),
 		})
 	}
 	return pinotFieldSpecs
@@ -333,9 +349,10 @@ func toMetricFieldSpecs(fieldSpecs []metricFieldSpec) []model.FieldSpec {
 	var pinotFieldSpecs []model.FieldSpec
 	for _, fs := range fieldSpecs {
 		pinotFieldSpecs = append(pinotFieldSpecs, model.FieldSpec{
-			Name:     fs.Name,
-			DataType: fs.DataType,
-			NotNull:  fs.NotNull.ValueBoolPointer(),
+			Name:              fs.Name,
+			DataType:          fs.DataType,
+			NotNull:           fs.NotNull.ValueBoolPointer(),
+			TransformFunction: fs.TransformFunction.ValueString(),
 		})
 	}
 	return pinotFieldSpecs
@@ -345,11 +362,12 @@ func toDateTimeFieldSpecs(fieldSpecs []dateTimeFieldSpec) []model.FieldSpec {
 	var pinotFieldSpecs []model.FieldSpec
 	for _, fs := range fieldSpecs {
 		pinotFieldSpecs = append(pinotFieldSpecs, model.FieldSpec{
-			Name:        fs.Name,
-			DataType:    fs.DataType,
-			Format:      fs.Format,
-			Granularity: fs.Granularity,
-			NotNull:     fs.NotNull.ValueBoolPointer(),
+			Name:              fs.Name,
+			DataType:          fs.DataType,
+			Format:            fs.Format,
+			Granularity:       fs.Granularity,
+			NotNull:           fs.NotNull.ValueBoolPointer(),
+			TransformFunction: fs.TransformFunction.ValueString(),
 		})
 	}
 	return pinotFieldSpecs
@@ -359,32 +377,44 @@ func setState(state *tableSchemaResourceModel, schema *model.Schema) {
 
 	dimensionFieldSpecs := make([]dimensionFieldSpec, len(schema.DimensionFieldSpecs))
 	for i, fs := range schema.DimensionFieldSpecs {
-		dimensionFieldSpecs[i] = dimensionFieldSpec{
+		dimensionFieldSpec := dimensionFieldSpec{
 			Name:             fs.Name,
 			DataType:         fs.DataType,
 			NotNull:          basetypes.NewBoolPointerValue(fs.NotNull),
 			SingleValueField: basetypes.NewBoolPointerValue(fs.SingleValueField),
 		}
+		if fs.TransformFunction != "" {
+			dimensionFieldSpec.TransformFunction = basetypes.NewStringValue(fs.TransformFunction)
+		}
+		dimensionFieldSpecs[i] = dimensionFieldSpec
 	}
 
 	metricFieldSpecs := make([]metricFieldSpec, len(schema.MetricFieldSpecs))
 	for i, fs := range schema.MetricFieldSpecs {
-		metricFieldSpecs[i] = metricFieldSpec{
+		metricFieldSpec := metricFieldSpec{
 			Name:     fs.Name,
 			DataType: fs.DataType,
 			NotNull:  basetypes.NewBoolPointerValue(fs.NotNull),
 		}
+		if fs.TransformFunction != "" {
+			metricFieldSpec.TransformFunction = basetypes.NewStringValue(fs.TransformFunction)
+		}
+		metricFieldSpecs[i] = metricFieldSpec
 	}
 
 	dateTimeFieldSpecs := make([]dateTimeFieldSpec, len(schema.DateTimeFieldSpecs))
 	for i, fs := range schema.DateTimeFieldSpecs {
-		dateTimeFieldSpecs[i] = dateTimeFieldSpec{
+		dateTimeFieldSpec := dateTimeFieldSpec{
 			Name:        fs.Name,
 			DataType:    fs.DataType,
 			Format:      fs.Format,
 			Granularity: fs.Granularity,
 			NotNull:     basetypes.NewBoolPointerValue(fs.NotNull),
 		}
+		if fs.TransformFunction != "" {
+			dateTimeFieldSpec.TransformFunction = basetypes.NewStringValue(fs.TransformFunction)
+		}
+		dateTimeFieldSpecs[i] = dateTimeFieldSpec
 	}
 
 	state.SchemaName = types.StringValue(schema.SchemaName)