Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: dependent assertions as dependencies #1720

Merged
merged 12 commits into from
May 7, 2024
7 changes: 5 additions & 2 deletions core/actions/assertion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ export const IAssertionConfigProperties = strictKeysOf<IAssertionConfig>()([
"name",
"schema",
"tags",
"type"
"type",
"dependOnDependencyAssertions"
Tuseeq1 marked this conversation as resolved.
Show resolved Hide resolved
]);

/**
Expand Down Expand Up @@ -151,7 +152,9 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
public dependencies(value: Resolvable | Resolvable[]) {
const newDependencies = Array.isArray(value) ? value : [value];
newDependencies.forEach(resolvable => {
this.proto.dependencyTargets.push(resolvableAsTarget(resolvable));
const resolvableTarget = resolvableAsTarget(resolvable);
this.session.actionAssertionMap.set(resolvableTarget, this.proto.target);
this.proto.dependencyTargets.push(resolvableTarget);
});
return this;
}
Expand Down
20 changes: 17 additions & 3 deletions core/actions/notebook.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import * as Path from "df/core/path";
import { Session } from "df/core/session";
import {
actionConfigToCompiledGraphTarget,
addDependenciesToActionDependencyTargets,
nativeRequire,
resolveActionsConfigFilename
} from "df/core/utils";
import { Resolvable } from "df/core/common";
import { dataform } from "df/protos/ts";

/**
Expand All @@ -18,6 +20,10 @@ export class Notebook extends ActionBuilder<dataform.Notebook> {
// TODO: make this field private, to enforce proto update logic to happen in this class.
public proto: dataform.INotebook = dataform.Notebook.create();

// This is Action level flag. If set to true, we will add assertions from all the depenedncies of
// current action as dependencies.
Tuseeq1 marked this conversation as resolved.
Show resolved Hide resolved
public dependOnDependencyAssertions: boolean = false;

constructor(
session?: Session,
config?: dataform.ActionConfig.NotebookConfig,
Expand All @@ -35,9 +41,8 @@ export class Notebook extends ActionBuilder<dataform.Notebook> {
this.proto.target = this.applySessionToTarget(target, config.filename);
this.proto.canonicalTarget = this.applySessionCanonicallyToTarget(target);
this.proto.tags = config.tags;
this.proto.dependencyTargets = config.dependencyTargets.map(dependencyTarget =>
actionConfigToCompiledGraphTarget(dataform.ActionConfig.Target.create(dependencyTarget))
);
this.dependOnDependencyAssertions = config.dependOnDependencyAssertions;
this.dependencies(config.dependencyTargets);
this.proto.fileName = config.filename;
if (config.disabled) {
this.proto.disabled = config.disabled;
Expand All @@ -61,6 +66,15 @@ export class Notebook extends ActionBuilder<dataform.Notebook> {
return this;
}

/**
* @hidden
*/
public dependencies(value: Resolvable | Resolvable[]) {
const newDependencies = Array.isArray(value) ? value : [value];
newDependencies.forEach(resolvable => addDependenciesToActionDependencyTargets(this, resolvable));
return this;
}

/**
* @hidden
*/
Expand Down
25 changes: 19 additions & 6 deletions core/actions/operation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import {
resolveActionsConfigFilename,
setNameAndTarget,
strictKeysOf,
toResolvable
toResolvable,
addDependenciesToActionDependencyTargets
} from "df/core/utils";
import { dataform } from "df/protos/ts";

Expand Down Expand Up @@ -59,7 +60,8 @@ export const IIOperationConfigProperties = strictKeysOf<IOperationConfig>()([
"name",
"schema",
"tags",
"type"
"type",
"dependOnDependencyAssertions"
]);

/**
Expand All @@ -75,6 +77,10 @@ export class Operation extends ActionBuilder<dataform.Operation> {
// We delay contextification until the final compile step, so hold these here for now.
private contextableQueries: Contextable<ICommonContext, string | string[]>;

// This is Action level flag. If set to true, we will add assertions from all the depenedncies of
// current action as dependencies.
public dependOnDependencyAssertions: boolean = false;
Ekrekr marked this conversation as resolved.
Show resolved Hide resolved

constructor(
session?: Session,
config?: dataform.ActionConfig.OperationConfig,
Expand Down Expand Up @@ -105,7 +111,8 @@ export class Operation extends ActionBuilder<dataform.Operation> {
tags: config.tags,
disabled: config.disabled,
hasOutput: config.hasOutput,
description: config.description
description: config.description,
dependOnDependencyAssertions: config.dependOnDependencyAssertions
});

this.queries(nativeRequire(config.filename).query);
Expand All @@ -118,6 +125,9 @@ export class Operation extends ActionBuilder<dataform.Operation> {
IIOperationConfigProperties,
"operation config"
);
if (config.dependOnDependencyAssertions) {
this.setDependOnDependencyAssertions(config.dependOnDependencyAssertions);
}
if (config.dependencies) {
this.dependencies(config.dependencies);
}
Expand Down Expand Up @@ -155,9 +165,7 @@ export class Operation extends ActionBuilder<dataform.Operation> {

public dependencies(value: Resolvable | Resolvable[]) {
const newDependencies = Array.isArray(value) ? value : [value];
newDependencies.forEach(resolvable => {
this.proto.dependencyTargets.push(resolvableAsTarget(resolvable));
});
newDependencies.forEach(resolvable => addDependenciesToActionDependencyTargets(this, resolvable));
return this;
}

Expand Down Expand Up @@ -228,6 +236,11 @@ export class Operation extends ActionBuilder<dataform.Operation> {
return this;
}

public setDependOnDependencyAssertions(dependOnDependencyAssertions: boolean){
this.dependOnDependencyAssertions = dependOnDependencyAssertions;
return this;
}

/**
* @hidden
*/
Expand Down
32 changes: 23 additions & 9 deletions core/actions/table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ import {
strictKeysOf,
tableTypeStringToEnum,
toResolvable,
validateQueryString
validateQueryString,
addDependenciesToActionDependencyTargets,
} from "df/core/utils";
import { dataform } from "df/protos/ts";

Expand Down Expand Up @@ -221,7 +222,8 @@ export const ITableConfigProperties = () =>
"database",
"columns",
"description",
"materialized"
"materialized",
"dependOnDependencyAssertions"
]);

/**
Expand Down Expand Up @@ -265,6 +267,10 @@ export class Table extends ActionBuilder<dataform.Table> {
private uniqueKeyAssertions: Assertion[] = [];
private rowConditionsAssertion: Assertion;

// This is Action level flag. If set to true, we will add assertions from all the depenedncies of
// current action as dependencies.
public dependOnDependencyAssertions: boolean = false;

constructor(
session?: Session,
tableTypeConfig?:
Expand Down Expand Up @@ -340,7 +346,8 @@ export class Table extends ActionBuilder<dataform.Table> {
tags: config.tags,
disabled: config.disabled,
description: config.description,
bigquery: bigqueryOptions
bigquery: bigqueryOptions,
dependOnDependencyAssertions: config.dependOnDependencyAssertions
});
}
if (tableType === "view") {
Expand Down Expand Up @@ -370,7 +377,8 @@ export class Table extends ActionBuilder<dataform.Table> {
materialized: config.materialized,
tags: config.tags,
description: config.description,
bigquery: bigqueryOptions
bigquery: bigqueryOptions,
dependOnDependencyAssertions: config.dependOnDependencyAssertions
});
}
if (tableType === "incremental") {
Expand Down Expand Up @@ -422,7 +430,8 @@ export class Table extends ActionBuilder<dataform.Table> {
uniqueKey: config.uniqueKey,
tags: config.tags,
description: config.description,
bigquery: bigqueryOptions
bigquery: bigqueryOptions,
dependOnDependencyAssertions: config.dependOnDependencyAssertions
});
}
this.query(nativeRequire(tableTypeConfig.filename).query);
Expand All @@ -444,6 +453,9 @@ export class Table extends ActionBuilder<dataform.Table> {
if (config.type) {
this.type(config.type);
}
if (config.dependOnDependencyAssertions) {
this.setDependOnDependencyAssertions(config.dependOnDependencyAssertions);
}
if (config.dependencies) {
this.dependencies(config.dependencies);
}
Expand Down Expand Up @@ -552,10 +564,7 @@ export class Table extends ActionBuilder<dataform.Table> {

public dependencies(value: Resolvable | Resolvable[]) {
const newDependencies = Array.isArray(value) ? value : [value];
newDependencies.forEach(resolvable => {
this.proto.dependencyTargets.push(resolvableAsTarget(resolvable));
});

newDependencies.forEach(resolvable => addDependenciesToActionDependencyTargets(this, resolvable));
return this;
}

Expand Down Expand Up @@ -676,6 +685,11 @@ export class Table extends ActionBuilder<dataform.Table> {
return this;
}

public setDependOnDependencyAssertions(dependOnDependencyAssertions: boolean){
this.dependOnDependencyAssertions = dependOnDependencyAssertions;
return this;
}

/**
* @hidden
*/
Expand Down
7 changes: 7 additions & 0 deletions core/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,11 @@ export interface IDependenciesConfig {
* dependencies, then it should be set to `true`.
*/
hermetic?: boolean;

/**
* If this flag is set to true, assertions depenedent upon any of the dependencies are added as depenedencies as well.
*/
dependOnDependencyAssertions?: boolean;
Tuseeq1 marked this conversation as resolved.
Show resolved Hide resolved
}

/**
Expand Down Expand Up @@ -219,6 +224,8 @@ export interface ITarget {
schema?: string;

name?: string;

includeDependentAssertions?: boolean;
kolina marked this conversation as resolved.
Show resolved Hide resolved
}

/**
Expand Down
Loading
Loading