From baaf178b405db97f9336aa61a5a1c814b03fbd5d Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Wed, 22 Nov 2023 12:04:05 +0000 Subject: [PATCH 1/3] Remove deprecated run cache and gcloud project ID fields --- cli/api/commands/build.ts | 12 +----------- cli/api/commands/run.ts | 16 ---------------- core/main.ts | 6 ------ core/session.ts | 10 ---------- protos/core.proto | 21 ++++----------------- tests/api/projects/common_v2/dataform.json | 1 - tests/cli/cli.spec.ts | 3 +-- tests/integration/bigquery.spec.ts | 4 +--- 8 files changed, 7 insertions(+), 66 deletions(-) diff --git a/cli/api/commands/build.ts b/cli/api/commands/build.ts index 0d0d300cc..c9d691e7b 100644 --- a/cli/api/commands/build.ts +++ b/cli/api/commands/build.ts @@ -13,8 +13,7 @@ export async function build( dbadapter: dbadapters.IDbAdapter ) { runConfig = { - ...runConfig, - useRunCache: false + ...runConfig }; const prunedGraph = prune(compiledGraph, runConfig); @@ -23,15 +22,6 @@ export async function build( targetStringifier, prunedGraph.tables.map(table => table.target) ); - if (runConfig.useRunCache) { - for (const includedAction of [ - ...prunedGraph.tables, - ...prunedGraph.operations, - ...prunedGraph.assertions - ]) { - allInvolvedTargets.add(includedAction.target); - } - } return new Builder( prunedGraph, diff --git a/cli/api/commands/run.ts b/cli/api/commands/run.ts index 673790e4a..88cd66ab4 100644 --- a/cli/api/commands/run.ts +++ b/cli/api/commands/run.ts @@ -383,22 +383,6 @@ export class Runner { } } - let newMetadata: dataform.ITableMetadata; - if (this.graph.projectConfig.useRunCache) { - try { - newMetadata = await this.dbadapter.table(action.target); - } catch (e) { - // Ignore Errors thrown when trying to get new table metadata; just allow the relevant - // warehouseStateAfterRunByTarget entry to be cleared out (below). - } - } - if (newMetadata) { - this.warehouseStateByTarget.set(action.target, newMetadata); - this.notifyListeners(); - } else { - this.warehouseStateByTarget.delete(action.target); - } - if (actionResult.status === dataform.ActionResult.ExecutionStatus.RUNNING) { actionResult.status = dataform.ActionResult.ExecutionStatus.SUCCESSFUL; } diff --git a/core/main.ts b/core/main.ts index 91076c188..e53bc6790 100644 --- a/core/main.ts +++ b/core/main.ts @@ -28,12 +28,6 @@ export function main(coreExecutionRequest: Uint8Array | string): Uint8Array | st const projectConfigOverride = compileRequest.compileConfig.projectConfigOverride ?? {}; - // Stop using the deprecated 'gcloudProjectId' field. - if (!originalProjectConfig.defaultDatabase) { - originalProjectConfig.defaultDatabase = originalProjectConfig.gcloudProjectId; - } - delete originalProjectConfig.gcloudProjectId; - let projectConfig = { ...originalProjectConfig }; // Merge in general project config overrides. diff --git a/core/session.ts b/core/session.ts index 3457a2d35..3e33b88f6 100644 --- a/core/session.ts +++ b/core/session.ts @@ -401,16 +401,6 @@ export class Session { [].concat(compiledGraph.tables, compiledGraph.assertions, compiledGraph.operations) ); - if (this.config.useRunCache) { - this.checkRunCachingCorrectness( - [].concat( - compiledGraph.tables, - compiledGraph.assertions, - compiledGraph.operations.filter(operation => operation.hasOutput) - ) - ); - } - utils.throwIfInvalid(compiledGraph, dataform.CompiledGraph.verify); return compiledGraph; } diff --git a/protos/core.proto b/protos/core.proto index f79db4084..b01e59036 100644 --- a/protos/core.proto +++ b/protos/core.proto @@ -19,28 +19,15 @@ message ProjectConfig { map vars = 14; - int32 concurrent_query_limit = 13; - int32 idempotent_action_retries = 8; - string database_suffix = 15; string schema_suffix = 7; string table_prefix = 11; - // Deprecated. - // Please use 'default_database' instead. - string gcloud_project_id = 6; - // This setting no longer has any effect. - bool use_run_cache = 10; - - reserved 3, 4, 12; -} - -message GenerateIndexConfig { - CompileConfig compile_config = 1; - repeated string include_paths = 2; - repeated string definition_paths = 3; + // TODO(ekrekr): Move these to be CLI flags instead. + int32 concurrent_query_limit = 13; + int32 idempotent_action_retries = 8; - reserved 4; + reserved 3, 4, 6, 10, 12; } message CompileConfig { diff --git a/tests/api/projects/common_v2/dataform.json b/tests/api/projects/common_v2/dataform.json index 8ff63a16e..88dabbe2e 100644 --- a/tests/api/projects/common_v2/dataform.json +++ b/tests/api/projects/common_v2/dataform.json @@ -3,6 +3,5 @@ "defaultDatabase": "tada-analytics", "defaultSchema": "df_integration_test", "assertionSchema": "df_integration_test_assertions", - "useRunCache": true, "defaultLocation": "US" } diff --git a/tests/cli/cli.spec.ts b/tests/cli/cli.spec.ts index ea64f0a9c..0d8de13ef 100644 --- a/tests/cli/cli.spec.ts +++ b/tests/cli/cli.spec.ts @@ -160,8 +160,7 @@ select 1 as \${dataform.projectConfig.vars.testVar2} } }, runConfig: { - fullRefresh: false, - useRunCache: false + fullRefresh: false }, warehouseState: {} }); diff --git a/tests/integration/bigquery.spec.ts b/tests/integration/bigquery.spec.ts index 6dd7ae1be..776ab65d5 100644 --- a/tests/integration/bigquery.spec.ts +++ b/tests/integration/bigquery.spec.ts @@ -257,9 +257,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) suite("evaluate", async () => { test("evaluate from valid compiled graph as valid", async () => { // Create and run the project. - const compiledGraph = await compile("tests/integration/bigquery_project", "evaluate", { - useRunCache: false - }); + const compiledGraph = await compile("tests/integration/bigquery_project", "evaluate"); const executionGraph = await dfapi.build(compiledGraph, {}, dbadapter); await dfapi.run(dbadapter, executionGraph).result(); From 6fe71a4ccc9433912e3796e1465667a3e94280d0 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Wed, 22 Nov 2023 14:12:27 +0000 Subject: [PATCH 2/3] Remove run cache test --- core/session.ts | 22 ---------------------- tests/api/projects.spec.ts | 5 ----- 2 files changed, 27 deletions(-) diff --git a/core/session.ts b/core/session.ts index 3e33b88f6..742c6ccd4 100644 --- a/core/session.ts +++ b/core/session.ts @@ -665,28 +665,6 @@ export class Session { }); } - private checkRunCachingCorrectness(actionsWithOutput: IActionProto[]) { - actionsWithOutput.forEach(action => { - if (action.dependencyTargets?.length > 0) { - return; - } - if ( - [dataform.ActionHermeticity.HERMETIC, dataform.ActionHermeticity.NON_HERMETIC].includes( - action.hermeticity - ) - ) { - return; - } - this.compileError( - new Error( - "Zero-dependency actions which create datasets are required to explicitly declare 'hermetic: (true|false)' when run caching is turned on." - ), - action.fileName, - action.target - ); - }); - } - private removeNonUniqueActionsFromCompiledGraph(compiledGraph: dataform.CompiledGraph) { function getNonUniqueTargets(targets: dataform.ITarget[]): StringifiedSet { const allTargets = new StringifiedSet(targetStringifier); diff --git a/tests/api/projects.spec.ts b/tests/api/projects.spec.ts index 3b880ca6e..70fd5a362 100644 --- a/tests/api/projects.spec.ts +++ b/tests/api/projects.spec.ts @@ -82,11 +82,6 @@ suite("examples", () => { { fileName: "definitions/has_compile_errors/table_with_materialized.sqlx", message: "The 'materialized' option is only valid for BigQuery views" - }, - { - fileName: "definitions/has_compile_errors/view_without_hermetic.sqlx", - message: - "Zero-dependency actions which create datasets are required to explicitly declare 'hermetic: (true|false)' when run caching is turned on." } ]); From 62753d11fdb830b28d12fd9918dc5c2d58c77ebe Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Wed, 22 Nov 2023 14:48:44 +0000 Subject: [PATCH 3/3] Tidying --- cli/api/commands/build.ts | 4 ---- cli/api/commands/run.ts | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cli/api/commands/build.ts b/cli/api/commands/build.ts index c9d691e7b..74305b06a 100644 --- a/cli/api/commands/build.ts +++ b/cli/api/commands/build.ts @@ -12,10 +12,6 @@ export async function build( runConfig: dataform.IRunConfig, dbadapter: dbadapters.IDbAdapter ) { - runConfig = { - ...runConfig - }; - const prunedGraph = prune(compiledGraph, runConfig); const allInvolvedTargets = new StringifiedSet( diff --git a/cli/api/commands/run.ts b/cli/api/commands/run.ts index 88cd66ab4..2793542e8 100644 --- a/cli/api/commands/run.ts +++ b/cli/api/commands/run.ts @@ -383,6 +383,8 @@ export class Runner { } } + this.warehouseStateByTarget.delete(action.target); + if (actionResult.status === dataform.ActionResult.ExecutionStatus.RUNNING) { actionResult.status = dataform.ActionResult.ExecutionStatus.SUCCESSFUL; }