From 67e55c8290a066e830ae4ea46ce391042179026c Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 17 Sep 2024 20:40:47 -0700 Subject: [PATCH 1/5] Read default values. --- clients/mssql/queries.go | 3 ++- clients/mssql/store.go | 1 - clients/shared/table_config.go | 24 +++++++++++++++++------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/clients/mssql/queries.go b/clients/mssql/queries.go index 636aa6052..af08323d3 100644 --- a/clients/mssql/queries.go +++ b/clients/mssql/queries.go @@ -14,7 +14,8 @@ SELECT ELSE DATA_TYPE END AS DATA_TYPE, - CHARACTER_MAXIMUM_LENGTH + CHARACTER_MAXIMUM_LENGTH, + COLUMN_DEFAULT AS DEFAULT_VALUE FROM INFORMATION_SCHEMA.COLUMNS WHERE diff --git a/clients/mssql/store.go b/clients/mssql/store.go index 0c8e31758..66704e93e 100644 --- a/clients/mssql/store.go +++ b/clients/mssql/store.go @@ -72,7 +72,6 @@ func (s *Store) Dedupe(_ sql.TableIdentifier, _ []string, _ bool) error { } func (s *Store) GetTableConfig(tableData *optimization.TableData) (*types.DwhTableConfig, error) { - // TODO: Figure out how to leave a comment. tableID := s.specificIdentifierFor(tableData.TopicConfig(), tableData.Name()) query, args := describeTableQuery(tableID) return shared.GetTableCfgArgs{ diff --git a/clients/shared/table_config.go b/clients/shared/table_config.go index 47bd2502a..4473b8755 100644 --- a/clients/shared/table_config.go +++ b/clients/shared/table_config.go @@ -104,14 +104,24 @@ func (g GetTableCfgArgs) GetTableConfig() (*types.DwhTableConfig, error) { } col := columns.NewColumn(row[g.ColumnNameForName], kindDetails) - // We need to check to make sure the comment is not an empty string - if comment, isOk := row[g.ColumnNameForComment]; isOk && comment != "" { - var _colComment constants.ColComment - if err = json.Unmarshal([]byte(comment), &_colComment); err != nil { - return nil, fmt.Errorf("failed to unmarshal comment %q: %w", comment, err) + strategy := g.Dwh.Dialect().GetDefaultValueStrategy() + switch strategy { + case sql.Backfill: + // We need to check to make sure the comment is not an empty string + if comment, isOk := row[g.ColumnNameForComment]; isOk && comment != "" { + var _colComment constants.ColComment + if err = json.Unmarshal([]byte(comment), &_colComment); err != nil { + return nil, fmt.Errorf("failed to unmarshal comment %q: %w", comment, err) + } + + col.SetBackfilled(_colComment.Backfilled) } - - col.SetBackfilled(_colComment.Backfilled) + case sql.Native: + if _, isOk := row["default_value"]; isOk { + col.SetBackfilled(true) + } + default: + return nil, fmt.Errorf("unknown default value strategy: %q", strategy) } cols.AddColumn(col) From a570a3b76a2748ed091c1995c2d4c816114a7d06 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 17 Sep 2024 20:41:47 -0700 Subject: [PATCH 2/5] Fix spacing. --- clients/mssql/queries.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/mssql/queries.go b/clients/mssql/queries.go index af08323d3..e23feadcf 100644 --- a/clients/mssql/queries.go +++ b/clients/mssql/queries.go @@ -15,7 +15,7 @@ SELECT DATA_TYPE END AS DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, - COLUMN_DEFAULT AS DEFAULT_VALUE + COLUMN_DEFAULT AS DEFAULT_VALUE FROM INFORMATION_SCHEMA.COLUMNS WHERE From e9a3fd0709b78f003f9b8d1de94aecc7c7c5236c Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 17 Sep 2024 20:42:34 -0700 Subject: [PATCH 3/5] Add TODO. --- lib/typing/columns/columns.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/typing/columns/columns.go b/lib/typing/columns/columns.go index 40dd0b9e4..779fdc839 100644 --- a/lib/typing/columns/columns.go +++ b/lib/typing/columns/columns.go @@ -26,7 +26,8 @@ type Column struct { // Whenever we see the same column where there's an opposite value in `toastColumn`, we will trigger a flush ToastColumn bool defaultValue any - backfilled bool + // TODO: Instead of using a boolean, we should be setting the value at some point. + backfilled bool } func (c *Column) PrimaryKey() bool { From a48b27d6a6765cd00f95046f1aba06b762f6947f Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 17 Sep 2024 20:43:37 -0700 Subject: [PATCH 4/5] Add another TODO. --- clients/shared/table_config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clients/shared/table_config.go b/clients/shared/table_config.go index 4473b8755..1543306d7 100644 --- a/clients/shared/table_config.go +++ b/clients/shared/table_config.go @@ -15,6 +15,8 @@ import ( "github.com/artie-labs/transfer/lib/typing/columns" ) +// TODO: Simplify this function + type GetTableCfgArgs struct { Dwh destination.DataWarehouse TableID sql.TableIdentifier From c55014f3ab01f833b230a690bee53dcbadbf99f1 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Wed, 18 Sep 2024 13:20:56 -0700 Subject: [PATCH 5/5] Ensure value is set. --- clients/shared/table_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/shared/table_config.go b/clients/shared/table_config.go index 1543306d7..03698150b 100644 --- a/clients/shared/table_config.go +++ b/clients/shared/table_config.go @@ -119,7 +119,7 @@ func (g GetTableCfgArgs) GetTableConfig() (*types.DwhTableConfig, error) { col.SetBackfilled(_colComment.Backfilled) } case sql.Native: - if _, isOk := row["default_value"]; isOk { + if value, isOk := row["default_value"]; isOk && value != "" { col.SetBackfilled(true) } default: