Added enum for table's type field (#1453)

* added enum for table's type field Signed-off-by: Nick Nalivaika <ikolina@google.com> * update version Signed-off-by: Nick Nalivaika <ikolina@google.com> * simplify code by filling enumType field for tables when loading compiled graphs * small fix for biquery adapter * make linter happy * fix integration tests * do not call setOrValidateTableEnumType when check for table validity, just check enumType value --------- Signed-off-by: Nick Nalivaika <ikolina@google.com>
dataform-co · Feb 15, 2023 · d5dfab1 · d5dfab1
1 parent 09447c4
commit d5dfab1
Show file tree

Hide file tree

Showing 26 changed files with 188 additions and 69 deletions.
diff --git a/api/commands/build.ts b/api/commands/build.ts
@@ -52,6 +52,7 @@ export class Builder {
       prunedGraph.projectConfig,
       prunedGraph.dataformCoreVersion || "1.0.0"
     );
+    prunedGraph.tables.forEach(utils.setOrValidateTableEnumType);
   }
 
   public build(): dataform.ExecutionGraph {
@@ -94,7 +95,7 @@ export class Builder {
     return {
       ...this.toPartialExecutionAction(table),
       type: "table",
-      tableType: table.type,
+      tableType: utils.tableTypeEnumToString(table.enumType),
       tasks: table.disabled
         ? []
         : this.adapter.publishTasks(table, runConfig, tableMetadata).build(),

diff --git a/api/commands/compile.ts b/api/commands/compile.ts
@@ -7,6 +7,7 @@ import deepmerge from "deepmerge";
 import { validWarehouses } from "df/api/dbadapters";
 import { coerceAsError, ErrorWithCause } from "df/common/errors/errors";
 import { decode64 } from "df/common/protos";
+import { setOrValidateTableEnumType } from "df/core/utils";
 import { dataform } from "df/protos/ts";
 
 // Project config properties that are required.
@@ -55,12 +56,16 @@ export async function compile(
 
   const result = await CompileChildProcess.forkProcess().compile(compileConfig);
 
+  let compileResult: dataform.CompiledGraph;
   if (compileConfig.useMain) {
     const decodedResult = decode64(dataform.CoreExecutionResponse, result);
-    return dataform.CompiledGraph.create(decodedResult.compile.compiledGraph);
+    compileResult = dataform.CompiledGraph.create(decodedResult.compile.compiledGraph);
+  } else {
+    compileResult = decode64(dataform.CompiledGraph, result);
   }
 
-  return decode64(dataform.CompiledGraph, result);
+  compileResult.tables.forEach(setOrValidateTableEnumType);
+  return compileResult;
 }
 
 export class CompileChildProcess {

diff --git a/api/commands/prune.ts b/api/commands/prune.ts
@@ -8,6 +8,7 @@ export function prune(
   compiledGraph: dataform.ICompiledGraph,
   runConfig: dataform.IRunConfig
 ): dataform.ICompiledGraph {
+  compiledGraph.tables.forEach(utils.setOrValidateTableEnumType);
   const includedActionNames = computeIncludedActionNames(compiledGraph, runConfig);
   return {
     ...compiledGraph,
@@ -28,7 +29,7 @@ function computeIncludedActionNames(
   runConfig: dataform.IRunConfig
 ): Set<string> {
   // Remove inline tables.
-  const filteredTables = compiledGraph.tables.filter(t => t.type !== "inline");
+  const filteredTables = compiledGraph.tables.filter(t => t.enumType !== dataform.TableType.INLINE);
 
   // Union all tables, operations, assertions.
   const allActions: CompileAction[] = [].concat(

diff --git a/cli/console.ts b/cli/console.ts
@@ -1,5 +1,6 @@
 import { IInitResult } from "df/api/commands/init";
 import { prettyJsonStringify } from "df/api/utils";
+import { setOrValidateTableEnumType, tableTypeEnumToString } from "df/core/utils";
 import { dataform } from "df/protos/ts";
 import * as readlineSync from "readline-sync";
 
@@ -112,10 +113,11 @@ export function printCompiledGraph(graph: dataform.ICompiledGraph, verbose: bool
       (graph.operations ? graph.operations.length : 0);
     writeStdOut(successOutput(`Compiled ${actionCount} action(s).`));
     if (graph.tables && graph.tables.length) {
+      graph.tables.forEach(setOrValidateTableEnumType);
       writeStdOut(`${graph.tables.length} dataset(s):`);
       graph.tables.forEach(compiledTable => {
         writeStdOut(
-          `${datasetString(compiledTable.target, compiledTable.type, compiledTable.disabled)}`,
+          `${datasetString(compiledTable.target, tableTypeEnumToString(compiledTable.enumType), compiledTable.disabled)}`,
           1
         );
       });

diff --git a/core/adapters/base.ts b/core/adapters/base.ts
@@ -2,6 +2,7 @@ import * as semver from "semver";
 
 import { IAdapter } from "df/core/adapters";
 import { Task, Tasks } from "df/core/tasks";
+import { tableTypeEnumToString } from "df/core/utils"
 import { dataform } from "df/protos/ts";
 
 export abstract class Adapter implements IAdapter {
@@ -34,15 +35,15 @@ export abstract class Adapter implements IAdapter {
     }`;
   }
 
-  public baseTableType(type: string) {
-    switch (type) {
-      case "table":
-      case "incremental":
+  public baseTableType(enumType: dataform.TableType) {
+    switch (enumType) {
+      case dataform.TableType.TABLE:
+      case dataform.TableType.INCREMENTAL:
         return dataform.TableMetadata.Type.TABLE;
-      case "view":
+      case dataform.TableType.VIEW:
         return dataform.TableMetadata.Type.VIEW;
       default:
-        throw new Error(`Unexpected table type: ${type}`);
+        throw new Error(`Unexpected table type: ${tableTypeEnumToString(enumType)}`);
     }
   }
 
@@ -133,7 +134,7 @@ from (${query}) as insertions`;
     let preOps = table.preOps;
     if (
       semver.gt(this.dataformCoreVersion, "1.4.8") &&
-      table.type === "incremental" &&
+      table.enumType === dataform.TableType.INCREMENTAL &&
       this.shouldWriteIncrementally(runConfig, tableMetadata)
     ) {
       preOps = table.incrementalPreOps;
@@ -149,7 +150,7 @@ from (${query}) as insertions`;
     let postOps = table.postOps;
     if (
       semver.gt(this.dataformCoreVersion, "1.4.8") &&
-      table.type === "incremental" &&
+      table.enumType === dataform.TableType.INCREMENTAL &&
       this.shouldWriteIncrementally(runConfig, tableMetadata)
     ) {
       postOps = table.incrementalPostOps;

diff --git a/core/adapters/bigquery.ts b/core/adapters/bigquery.ts
@@ -22,14 +22,14 @@ export class BigQueryAdapter extends Adapter implements IAdapter {
 
     this.preOps(table, runConfig, tableMetadata).forEach(statement => tasks.add(statement));
 
-    const baseTableType = this.baseTableType(table.type);
+    const baseTableType = this.baseTableType(table.enumType);
     if (tableMetadata && tableMetadata.type !== baseTableType) {
       tasks.add(
         Task.statement(this.dropIfExists(table.target, this.oppositeTableType(baseTableType)))
       );
     }
 
-    if (table.type === "incremental") {
+    if (table.enumType === dataform.TableType.INCREMENTAL) {
       if (!this.shouldWriteIncrementally(runConfig, tableMetadata)) {
         tasks.add(Task.statement(this.createOrReplace(table)));
       } else {
@@ -94,7 +94,7 @@ export class BigQueryAdapter extends Adapter implements IAdapter {
       ? "materialized "
       : ""
     }${this.tableTypeAsSql(
-      this.baseTableType(table.type)
+      this.baseTableType(table.enumType)
     )} ${this.resolveTarget(table.target)} ${
       table.bigquery && table.bigquery.partitionBy
         ? `partition by ${table.bigquery.partitionBy} `

diff --git a/core/adapters/index.ts b/core/adapters/index.ts
@@ -21,7 +21,7 @@ export interface IAdapter {
   assertTasks(assertion: dataform.IAssertion, projectConfig: dataform.IProjectConfig): Tasks;
 
   dropIfExists(target: dataform.ITarget, type: dataform.TableMetadata.Type): string;
-  baseTableType(type: string): dataform.TableMetadata.Type;
+  baseTableType(enumType: dataform.TableType): dataform.TableMetadata.Type;
 
   indexAssertion(dataset: string, indexCols: string[]): string;
   rowConditionsAssertion(dataset: string, rowConditions: string[]): string;
@@ -135,7 +135,7 @@ export function collectEvaluationQueries(
   } else {
     try {
       if (queryOrAction instanceof dataform.Table) {
-        if (queryOrAction.type === "incremental") {
+        if (queryOrAction.enumType === dataform.TableType.INCREMENTAL) {
           const incrementalTableQueries = queryOrAction.incrementalPreOps.concat(
             queryOrAction.incrementalQuery,
             queryOrAction.incrementalPostOps

diff --git a/core/adapters/presto.ts b/core/adapters/presto.ts
@@ -23,14 +23,14 @@ export class PrestoAdapter extends Adapter implements IAdapter {
 
     this.preOps(table, runConfig, tableMetadata).forEach(statement => tasks.add(statement));
 
-    const baseTableType = this.baseTableType(table.type);
+    const baseTableType = this.baseTableType(table.enumType);
     if (tableMetadata && tableMetadata.type !== baseTableType) {
       tasks.add(
         Task.statement(this.dropIfExists(table.target, this.oppositeTableType(baseTableType)))
       );
     }
 
-    if (table.type === "incremental") {
+    if (table.enumType === dataform.TableType.INCREMENTAL) {
       throw new Error("Incremental table types are not currently supported for Presto.");
     } else {
       tasks.add(Task.statement(this.createOrReplace(table)));

diff --git a/core/adapters/redshift.ts b/core/adapters/redshift.ts
@@ -23,14 +23,14 @@ export class RedshiftAdapter extends Adapter implements IAdapter {
 
     this.preOps(table, runConfig, tableMetadata).forEach(statement => tasks.add(statement));
 
-    const baseTableType = this.baseTableType(table.type);
+    const baseTableType = this.baseTableType(table.enumType);
     if (tableMetadata && tableMetadata.type !== baseTableType) {
       tasks.add(
         Task.statement(this.dropIfExists(table.target, this.oppositeTableType(baseTableType)))
       );
     }
 
-    if (table.type === "incremental") {
+    if (table.enumType === dataform.TableType.INCREMENTAL) {
       if (!this.shouldWriteIncrementally(runConfig, tableMetadata)) {
         tasks.addAll(this.createOrReplace(table));
       } else {
@@ -90,15 +90,15 @@ export class RedshiftAdapter extends Adapter implements IAdapter {
   }
 
   private createOrReplace(table: dataform.ITable) {
-    if (table.type === "view") {
+    if (table.enumType === dataform.TableType.VIEW) {
       const isBindDefined = table.redshift && table.redshift.hasOwnProperty("bind");
       const bindDefaultValue = semver.gte(this.dataformCoreVersion, "1.4.1") ? false : true;
       const bind =
         (isBindDefined ? table.redshift.bind : bindDefaultValue) && this.isBindSupported();
       return (
         Tasks.create()
           // Drop the view in case we are changing the number of column(s) (or their types).
-          .add(Task.statement(this.dropIfExists(table.target, this.baseTableType(table.type))))
+          .add(Task.statement(this.dropIfExists(table.target, this.baseTableType(table.enumType))))
           .add(Task.statement(this.createOrReplaceView(table.target, table.query, bind)))
       );
     }
@@ -108,7 +108,7 @@ export class RedshiftAdapter extends Adapter implements IAdapter {
     });
 
     return Tasks.create()
-      .add(Task.statement(this.dropIfExists(tempTableTarget, this.baseTableType(table.type))))
+      .add(Task.statement(this.dropIfExists(tempTableTarget, this.baseTableType(table.enumType))))
       .add(Task.statement(this.createTable(table, tempTableTarget)))
       .add(Task.statement(this.dropIfExists(table.target, dataform.TableMetadata.Type.TABLE)))
       .add(

diff --git a/core/adapters/snowflake.ts b/core/adapters/snowflake.ts
@@ -25,14 +25,14 @@ export class SnowflakeAdapter extends Adapter implements IAdapter {
 
     this.preOps(table, runConfig, tableMetadata).forEach(statement => tasks.add(statement));
 
-    const baseTableType = this.baseTableType(table.type);
+    const baseTableType = this.baseTableType(table.enumType);
     if (tableMetadata && tableMetadata.type !== baseTableType) {
       tasks.add(
         Task.statement(this.dropIfExists(table.target, this.oppositeTableType(baseTableType)))
       );
     }
 
-    if (table.type === "incremental") {
+    if (table.enumType === dataform.TableType.INCREMENTAL) {
       if (!this.shouldWriteIncrementally(runConfig, tableMetadata)) {
         tasks.add(Task.statement(this.createOrReplace(table)));
       } else {
@@ -80,7 +80,7 @@ export class SnowflakeAdapter extends Adapter implements IAdapter {
   }
 
   private createOrReplace(table: dataform.ITable) {
-    if (table.type === "view") {
+    if (table.enumType === dataform.TableType.VIEW) {
       return this.createOrReplaceView(table.target, table.query, table.snowflake?.secure, table.materialized);
     }
     return `create or replace ${

diff --git a/core/adapters/sqldatawarehouse.ts b/core/adapters/sqldatawarehouse.ts
@@ -26,14 +26,14 @@ export class SQLDataWarehouseAdapter extends Adapter implements IAdapter {
 
     this.preOps(table, runConfig, tableMetadata).forEach(statement => tasks.add(statement));
 
-    const baseTableType = this.baseTableType(table.type);
+    const baseTableType = this.baseTableType(table.enumType);
     if (tableMetadata && tableMetadata.type !== baseTableType) {
       tasks.add(
         Task.statement(this.dropIfExists(table.target, this.oppositeTableType(baseTableType)))
       );
     }
 
-    if (table.type === "incremental") {
+    if (table.enumType === dataform.TableType.INCREMENTAL) {
       if (!this.shouldWriteIncrementally(runConfig, tableMetadata)) {
         tasks.addAll(this.createOrReplace(table, !!tableMetadata));
       } else {
@@ -90,7 +90,7 @@ from (${query}
   }
 
   private createOrReplace(table: dataform.ITable, alreadyExists: boolean) {
-    if (table.type === "view") {
+    if (table.enumType === dataform.TableType.VIEW) {
       return Tasks.create().add(
         Task.statement(
           `${alreadyExists ? "alter" : "create"} view ${this.resolveTarget(table.target)} as ${
@@ -105,7 +105,7 @@ from (${query}
     });
 
     return Tasks.create()
-      .add(Task.statement(this.dropIfExists(tempTableTarget, this.baseTableType(table.type))))
+      .add(Task.statement(this.dropIfExists(tempTableTarget, this.baseTableType(table.enumType))))
       .add(Task.statement(this.createTable(table, tempTableTarget)))
       .add(Task.statement(this.dropIfExists(table.target, dataform.TableMetadata.Type.TABLE)))
       .add(