Add more fields

docs/resources/table.md

@@ -23,6 +23,7 @@ description: |-
 
 ### Optional
 
+- `field_config_list` (Attributes List) field configurations for the table (see [below for nested schema](#nestedatt--field_config_list))
 - `ingestion_config` (Attributes) ingestion configuration for the table i.e kafka (see [below for nested schema](#nestedatt--ingestion_config))
 - `is_dim_table` (Boolean) is dimension table
 - `metadata` (Attributes) metadata for the table (see [below for nested schema](#nestedatt--metadata))
@@ -32,6 +33,46 @@ description: |-
 - `tier_configs` (Attributes List) tier configurations for the table (see [below for nested schema](#nestedatt--tier_configs))
 - `upsert_config` (Attributes) The upsert configuration for the table. (see [below for nested schema](#nestedatt--upsert_config))
 
+<a id="nestedatt--field_config_list"></a>
+### Nested Schema for `field_config_list`
+
+Required:
+
+- `encoding_type` (String) encoding type
+- `index_type` (String) index type
+- `name` (String) name of the field
+
+Optional:
+
+- `index_types` (List of String) index types
+- `indexes` (Attributes) indexes (see [below for nested schema](#nestedatt--field_config_list--indexes))
+- `timestamp_config` (Attributes) timestamp configuration (see [below for nested schema](#nestedatt--field_config_list--timestamp_config))
+
+<a id="nestedatt--field_config_list--indexes"></a>
+### Nested Schema for `field_config_list.indexes`
+
+Optional:
+
+- `inverted` (Attributes) inverted (see [below for nested schema](#nestedatt--field_config_list--indexes--inverted))
+
+<a id="nestedatt--field_config_list--indexes--inverted"></a>
+### Nested Schema for `field_config_list.indexes.inverted`
+
+Required:
+
+- `enabled` (String) enabled
+
+
+
+<a id="nestedatt--field_config_list--timestamp_config"></a>
+### Nested Schema for `field_config_list.timestamp_config`
+
+Optional:
+
+- `granularities` (List of String) granularities
+
+
+
 <a id="nestedatt--ingestion_config"></a>
 ### Nested Schema for `ingestion_config`
 
@@ -41,6 +82,7 @@ Optional:
 - `row_time_value_check` (Boolean) row time value check.
 - `segment_time_value_check` (Boolean) segment time value check.
 - `stream_ingestion_config` (Attributes) stream ingestion configurations (see [below for nested schema](#nestedatt--ingestion_config--stream_ingestion_config))
+- `transform_configs` (Attributes List) transform configurations (see [below for nested schema](#nestedatt--ingestion_config--transform_configs))
 
 <a id="nestedatt--ingestion_config--stream_ingestion_config"></a>
 ### Nested Schema for `ingestion_config.stream_ingestion_config`
@@ -50,6 +92,15 @@ Optional:
 - `stream_config_maps` (List of Map of String) stream configuration
 
 
+<a id="nestedatt--ingestion_config--transform_configs"></a>
+### Nested Schema for `ingestion_config.transform_configs`
+
+Required:
+
+- `column_name` (String) column name
+- `transform_function` (String) transform function
+
+
 
 <a id="nestedatt--metadata"></a>
 ### Nested Schema for `metadata`
@@ -70,6 +121,8 @@ Required:
 
 Optional:
 
+- `deleted_segments_retention_period` (String) The deleted segments retention period for the segments.
+- `replicas_per_partition` (String) The replicas per partition for the segments.
 - `retention_time_unit` (String) The retention time unit for the segments.
 - `retention_time_value` (String) The retention time value for the segments.
 
@@ -80,19 +133,25 @@ Optional:
 Optional:
 
 - `aggregate_metrics` (Boolean) The aggregate metrics for the table.
+- `bloom_filter_columns` (List of String) The bloom filter columns for the table.
 - `column_min_max_value_generator_mode` (String) The column min max value generator mode for the table.
 - `create_inverted_index_during_segment_generation` (Boolean) The create inverted index during segment generation for the table.
 - `enable_default_star_tree` (Boolean) The enable default star tree for the table.
 - `enable_dynamic_star_tree` (Boolean) The enable dynamic star tree for the table.
 - `load_mode` (String) The load mode for the table.
+- `no_dictionary_columns` (List of String) The no dictionary columns for the table.
 - `no_dictionary_size_ratio_threshold` (Number) The no dictionary size ration threshold for the table.
 - `null_handling_enabled` (Boolean) The null handling enabled for the table.
+- `on_heap_dictionary_columns` (List of String) The on heap dictionary columns for the table.
 - `optimize_dictionary` (Boolean) The optimize dictionary for the table.
 - `optimize_dictionary_for_metrics` (Boolean) The optimize dictionary for metrics for the table.
+- `range_index_columns` (List of String) The range index columns for the table.
+- `range_index_version` (Number) The range index version for the table.
 - `segment_name_generator_type` (String) The segment name generator type for the table.
 - `segment_partition_config` (Attributes) The segment partition configuration for the table. (see [below for nested schema](#nestedatt--table_index_config--segment_partition_config))
 - `sorted_column` (List of String) The sorted column for the table.
 - `star_tree_index_configs` (Attributes List) The star tree index configurations for the table. (see [below for nested schema](#nestedatt--table_index_config--star_tree_index_configs))
+- `var_length_dictionary_columns` (List of String) The var length dictionary columns for the table.
 
 <a id="nestedatt--table_index_config--segment_partition_config"></a>
 ### Nested Schema for `table_index_config.segment_partition_config`

examples/tables/main.tf

@@ -53,6 +53,11 @@ locals {
     join("_", [for keyName in regexall("[A-Z]?[a-z]+", key) : lower(keyName)]) => value
   }
 
+  transform_configs = [
+    for value in local.ingestion_config["transform_configs"] :
+    { for key, inner_value in value : join("_", [for keyName in regexall("[A-Z]?[a-z]+", key) : lower(keyName)]) => inner_value }
+  ]
+
   kafka_overrides_secrets = sensitive({
     "stream.kafka.broker.list" : local.kafka_broker,
     "stream.kafka.zk.broker.url" : local.kafka_zk
@@ -137,6 +142,7 @@ resource "pinot_table" "realtime_table" {
     continue_on_error        = true
     row_time_value_check     = true
     stream_ingestion_config  = local.parsed_stream_ingestion_config
+    transform_configs        = local.transform_configs
   })
 
   metadata = local.metadata

examples/tables/realtime_table_example.json

@@ -6,15 +6,22 @@
     "timeType": "MILLISECONDS",
     "replication": "{{ .PinotSegmentsReplication }}",
     "retentionTimeUnit": "DAYS",
-    "retentionTimeValue": "7"
+    "retentionTimeValue": "7",
+    "deletedSegmentsRetentionPeriod": "1d",
+    "replicasPerPartition": "1"
   },
   "tenants": {
     "broker": "{{ .PinotTenantBroker }}",
     "server": "{{ .PinotTenantServer }}"
   },
   "tableIndexConfig": {
-    "rangeIndexVersion": 0,
+    "rangeIndexVersion": 2,
     "autoGeneratedInvertedIndex": false,
+    "onHeapDictionaryColumns": [],
+    "varLengthDictionaryColumns": [],
+    "noDictionaryColumns": [],
+    "bloomFilterColumns": [],
+    "rangeIndexColumns": [],
     "createInvertedIndexDuringSegmentGeneration": false,
     "loadMode": "MMAP",
     "enableDefaultStarTree": false,
@@ -32,7 +39,7 @@
     "columnMajorSegmentBuilderEnabled": false,
     "optimizeDictionary": false,
     "optimizeDictionaryForMetrics": false,
-    "noDictionarySizeRatioThreshold": 0,
+    "noDictionarySizeRatioThreshold": 0.85,
     "sortedColumn": [
       "number"
     ]
@@ -53,7 +60,7 @@
       "timestampConfig": {},
       "indexes": {
         "inverted": {
-          "enabled": ""
+          "enabled": "true"
         }
       },
       "tierOverwrites": null
@@ -79,7 +86,13 @@
       "columnMajorSegmentBuilderEnabled": false
     },
     "continueOnError": true,
-    "rowTimeValueCheck": true
+    "rowTimeValueCheck": true,
+    "transformConfigs": [
+      {
+        "columnName": "hash_json",
+        "transformFunction": "json_format(hash)"
+      }
+    ]
   },
   "tierConfigs": [
     {

go.mod

@@ -3,7 +3,7 @@ module terraform-provider-pinot
 go 1.22
 
 require (
-	github.com/azaurus1/go-pinot-api v0.1.6-0.20240319123203-7eaffa14dc05
+	github.com/azaurus1/go-pinot-api v0.1.6-0.20240320125733-e682d6a80d1e
 	github.com/hashicorp/terraform-plugin-docs v0.18.0
 	github.com/hashicorp/terraform-plugin-framework v1.6.1
 	github.com/hashicorp/terraform-plugin-log v0.9.0

go.sum

@@ -30,6 +30,12 @@ github.com/azaurus1/go-pinot-api v0.1.6-0.20240319121223-b98533335300 h1:qiJFpnS
 github.com/azaurus1/go-pinot-api v0.1.6-0.20240319121223-b98533335300/go.mod h1:Zxi9gp5nAlU95XH1FQlAXLzH/cLGdX6KaisGtVF56t0=
 github.com/azaurus1/go-pinot-api v0.1.6-0.20240319123203-7eaffa14dc05 h1:ZssdKtCRy6hwyu0xKxjUDR9gHT7rdJeKdkpTq60inKM=
 github.com/azaurus1/go-pinot-api v0.1.6-0.20240319123203-7eaffa14dc05/go.mod h1:Zxi9gp5nAlU95XH1FQlAXLzH/cLGdX6KaisGtVF56t0=
+github.com/azaurus1/go-pinot-api v0.1.6-0.20240320083001-97f535629e53 h1:Ppz8yxmII3TOnjLftTMcnaJecTFVWWjd8xcFrmlLtdM=
+github.com/azaurus1/go-pinot-api v0.1.6-0.20240320083001-97f535629e53/go.mod h1:Zxi9gp5nAlU95XH1FQlAXLzH/cLGdX6KaisGtVF56t0=
+github.com/azaurus1/go-pinot-api v0.1.6-0.20240320115331-f25efcbb8cca h1:LF9KgaEWc+i1EFtNZR/Z8bKI3/fUgrp+DJQkubygjio=
+github.com/azaurus1/go-pinot-api v0.1.6-0.20240320115331-f25efcbb8cca/go.mod h1:Zxi9gp5nAlU95XH1FQlAXLzH/cLGdX6KaisGtVF56t0=
+github.com/azaurus1/go-pinot-api v0.1.6-0.20240320125733-e682d6a80d1e h1:tInsBHJcLk0fda/eOc3npWpRmhuQrEC/eqbSKO4wf7Y=
+github.com/azaurus1/go-pinot-api v0.1.6-0.20240320125733-e682d6a80d1e/go.mod h1:Zxi9gp5nAlU95XH1FQlAXLzH/cLGdX6KaisGtVF56t0=
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=

internal/converter/converter.go

@@ -46,6 +46,20 @@ func SetStateFromTable(state *models.TableResourceModel, table *model.Table) {
 		SortedColumn:                               table.TableIndexConfig.SortedColumn,
 		AggregateMetrics:                           types.BoolValue(table.TableIndexConfig.AggregateMetrics),
 		SegmentPartitionConfig:                     convertSegmentPartitionConfig(table),
+		OnHeapDictionaryColumns:                    table.TableIndexConfig.OnHeapDictionaryColumns,
+		VarLengthDictionaryColumns:                 table.TableIndexConfig.VarLengthDictionaryColumns,
+		RangeIndexColumns:                          table.TableIndexConfig.RangeIndexColumns,
+		NoDictionaryColumns:                        table.TableIndexConfig.NoDictionaryColumns,
+		BloomFilterColumns:                         table.TableIndexConfig.BloomFilterColumns,
+		RangeIndexVersion:                          types.Int64Value(int64(table.TableIndexConfig.RangeIndexVersion)),
+	}
+
+	var ingestionTransformConfigs []*models.TransformConfig
+	for _, transformConfig := range table.IngestionConfig.TransformConfigs {
+		ingestionTransformConfigs = append(ingestionTransformConfigs, &models.TransformConfig{
+			ColumnName:        types.StringValue(transformConfig.ColumnName),
+			TransformFunction: types.StringValue(transformConfig.TransformFunction),
+		})
 	}
 
 	state.IngestionConfig = &models.IngestionConfig{
@@ -55,6 +69,7 @@ func SetStateFromTable(state *models.TableResourceModel, table *model.Table) {
 		StreamIngestionConfig: &models.StreamIngestionConfig{
 			StreamConfigMaps: table.IngestionConfig.StreamIngestionConfig.StreamConfigMaps,
 		},
+		TransformConfigs: ingestionTransformConfigs,
 	}
 
 	var tierConfigs []*models.TierConfig

internal/models/pinot.go

@@ -14,6 +14,7 @@ type TableResourceModel struct {
 	TierConfigs      []*TierConfig     `tfsdk:"tier_configs"`
 	IsDimTable       types.Bool        `tfsdk:"is_dim_table"`
 	Metadata         *Metadata         `tfsdk:"metadata"`
+	FieldConfigList  []*FieldConfig    `tfsdk:"field_config_list"`
 }
 
 type TenantsConfig struct {
@@ -23,11 +24,13 @@ type TenantsConfig struct {
 }
 
 type SegmentsConfig struct {
-	TimeType           types.String `tfsdk:"time_type"`
-	Replication        types.String `tfsdk:"replication"`
-	TimeColumnName     types.String `tfsdk:"time_column_name"`
-	RetentionTimeUnit  types.String `tfsdk:"retention_time_unit"`
-	RetentionTimeValue types.String `tfsdk:"retention_time_value"`
+	TimeType                      types.String `tfsdk:"time_type"`
+	Replication                   types.String `tfsdk:"replication"`
+	ReplicasPerPartition          types.String `tfsdk:"replicas_per_partition"`
+	TimeColumnName                types.String `tfsdk:"time_column_name"`
+	RetentionTimeUnit             types.String `tfsdk:"retention_time_unit"`
+	RetentionTimeValue            types.String `tfsdk:"retention_time_value"`
+	DeletedSegmentRetentionPeriod types.String `tfsdk:"deleted_segment_retention_period"`
 }
 
 type UpsertConfig struct {
@@ -39,6 +42,27 @@ type SegmentPartitionConfig struct {
 	ColumnPartitionMap map[string]map[string]string `tfsdk:"column_partition_map"`
 }
 
+type TimestampConfig struct {
+	Granularities []string `tfsdk:"granularities"`
+}
+
+type FiendIndexInverted struct {
+	Enabled types.String `tfsdk:"enabled"`
+}
+
+type FieldIndexes struct {
+	Inverted *FiendIndexInverted `tfsdk:"inverted"`
+}
+
+type FieldConfig struct {
+	Name            types.String     `tfsdk:"name"`
+	EncodingType    types.String     `tfsdk:"encoding_type"`
+	IndexType       types.String     `tfsdk:"index_type"`
+	IndexTypes      []string         `tfsdk:"index_types"`
+	TimestampConfig *TimestampConfig `tfsdk:"timestamp_config"`
+	Indexes         *FieldIndexes    `tfsdk:"indexes"`
+}
+
 type TableIndexConfig struct {
 	SortedColumn                               []string                `tfsdk:"sorted_column"`
 	LoadMode                                   types.String            `tfsdk:"load_mode"`
@@ -54,6 +78,12 @@ type TableIndexConfig struct {
 	AggregateMetrics                           types.Bool              `tfsdk:"aggregate_metrics"`
 	StarTreeIndexConfigs                       []*StarTreeIndexConfigs `tfsdk:"star_tree_index_configs"`
 	SegmentPartitionConfig                     *SegmentPartitionConfig `tfsdk:"segment_partition_config"`
+	NoDictionaryColumns                        []string                `tfsdk:"no_dictionary_columns"`
+	RangeIndexColumns                          []string                `tfsdk:"range_index_columns"`
+	OnHeapDictionaryColumns                    []string                `tfsdk:"on_heap_dictionary_columns"`
+	VarLengthDictionaryColumns                 []string                `tfsdk:"var_length_dictionary_columns"`
+	BloomFilterColumns                         []string                `tfsdk:"bloom_filter_columns"`
+	RangeIndexVersion                          types.Int64             `tfsdk:"range_index_version"`
 }
 
 type AggregationConfig struct {
@@ -70,11 +100,17 @@ type StarTreeIndexConfigs struct {
 	AggregationConfigs              []*AggregationConfig `tfsdk:"aggregation_configs"`
 }
 
+type TransformConfig struct {
+	ColumnName        types.String `tfsdk:"column_name"`
+	TransformFunction types.String `tfsdk:"transform_function"`
+}
+
 type IngestionConfig struct {
 	SegmentTimeValueCheck types.Bool             `tfsdk:"segment_time_value_check"`
 	RowTimeValueCheck     types.Bool             `tfsdk:"row_time_value_check"`
 	ContinueOnError       types.Bool             `tfsdk:"continue_on_error"`
 	StreamIngestionConfig *StreamIngestionConfig `tfsdk:"stream_ingestion_config"`
+	TransformConfigs      []*TransformConfig     `tfsdk:"transform_configs"`
 }
 
 type StreamIngestionConfig struct {