Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: dependent assertions as dependencies #1720

Merged
merged 12 commits into from
May 7, 2024
7 changes: 5 additions & 2 deletions core/actions/assertion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ export const IAssertionConfigProperties = strictKeysOf<IAssertionConfig>()([
"name",
"schema",
"tags",
"type"
"type",
"dependOnDependencyAssertions"
Tuseeq1 marked this conversation as resolved.
Show resolved Hide resolved
]);

/**
Expand Down Expand Up @@ -151,7 +152,9 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
public dependencies(value: Resolvable | Resolvable[]) {
const newDependencies = Array.isArray(value) ? value : [value];
newDependencies.forEach(resolvable => {
this.proto.dependencyTargets.push(resolvableAsTarget(resolvable));
const resolvableTarget = resolvableAsTarget(resolvable);
this.session.actionAssertionMap.set(resolvableTarget, this.proto.target);
this.proto.dependencyTargets.push(resolvableTarget);
});
return this;
}
Expand Down
19 changes: 16 additions & 3 deletions core/actions/notebook.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import * as Path from "df/core/path";
import { Session } from "df/core/session";
import {
actionConfigToCompiledGraphTarget,
addDependenciesToActionDependencyTargets,
nativeRequire,
resolveActionsConfigFilename
} from "df/core/utils";
import { Resolvable } from "df/core/common";
import { dataform } from "df/protos/ts";

/**
Expand All @@ -18,6 +20,9 @@ export class Notebook extends ActionBuilder<dataform.Notebook> {
// TODO: make this field private, to enforce proto update logic to happen in this class.
public proto: dataform.INotebook = dataform.Notebook.create();

// If true, adds the inline assertions of dependencies as direct dependencies for this action.
public dependOnDependencyAssertions: boolean = false;

constructor(
session?: Session,
config?: dataform.ActionConfig.NotebookConfig,
Expand All @@ -35,9 +40,8 @@ export class Notebook extends ActionBuilder<dataform.Notebook> {
this.proto.target = this.applySessionToTarget(target, config.filename);
this.proto.canonicalTarget = this.applySessionCanonicallyToTarget(target);
this.proto.tags = config.tags;
this.proto.dependencyTargets = config.dependencyTargets.map(dependencyTarget =>
actionConfigToCompiledGraphTarget(dataform.ActionConfig.Target.create(dependencyTarget))
);
this.dependOnDependencyAssertions = config.dependOnDependencyAssertions;
this.dependencies(config.dependencyTargets);
this.proto.fileName = config.filename;
if (config.disabled) {
this.proto.disabled = config.disabled;
Expand All @@ -61,6 +65,15 @@ export class Notebook extends ActionBuilder<dataform.Notebook> {
return this;
}

/**
* @hidden
*/
public dependencies(value: Resolvable | Resolvable[]) {
const newDependencies = Array.isArray(value) ? value : [value];
newDependencies.forEach(resolvable => addDependenciesToActionDependencyTargets(this, resolvable));
return this;
}

/**
* @hidden
*/
Expand Down
32 changes: 22 additions & 10 deletions core/actions/operation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import {
resolveActionsConfigFilename,
setNameAndTarget,
strictKeysOf,
toResolvable
toResolvable,
addDependenciesToActionDependencyTargets
} from "df/core/utils";
import { dataform } from "df/protos/ts";

Expand All @@ -31,10 +32,10 @@ import { dataform } from "df/protos/ts";
*/
export interface IOperationConfig
extends IActionConfig,
IDependenciesConfig,
IDocumentableConfig,
INamedConfig,
ITargetableConfig {
IDependenciesConfig,
IDocumentableConfig,
INamedConfig,
ITargetableConfig {
/**
* Declares that this `operations` action creates a dataset which should be referenceable using the `ref` function.
*
Expand All @@ -59,7 +60,8 @@ export const IIOperationConfigProperties = strictKeysOf<IOperationConfig>()([
"name",
"schema",
"tags",
"type"
"type",
"dependOnDependencyAssertions"
]);

/**
Expand All @@ -75,6 +77,9 @@ export class Operation extends ActionBuilder<dataform.Operation> {
// We delay contextification until the final compile step, so hold these here for now.
private contextableQueries: Contextable<ICommonContext, string | string[]>;

// If true, adds the inline assertions of dependencies as direct dependencies for this action.
public dependOnDependencyAssertions: boolean = false;

constructor(
session?: Session,
config?: dataform.ActionConfig.OperationConfig,
Expand Down Expand Up @@ -105,7 +110,8 @@ export class Operation extends ActionBuilder<dataform.Operation> {
tags: config.tags,
disabled: config.disabled,
hasOutput: config.hasOutput,
description: config.description
description: config.description,
dependOnDependencyAssertions: config.dependOnDependencyAssertions
});

this.queries(nativeRequire(config.filename).query);
Expand All @@ -118,6 +124,9 @@ export class Operation extends ActionBuilder<dataform.Operation> {
IIOperationConfigProperties,
"operation config"
);
if (config.dependOnDependencyAssertions) {
this.setDependOnDependencyAssertions(config.dependOnDependencyAssertions);
}
if (config.dependencies) {
this.dependencies(config.dependencies);
}
Expand Down Expand Up @@ -155,9 +164,7 @@ export class Operation extends ActionBuilder<dataform.Operation> {

public dependencies(value: Resolvable | Resolvable[]) {
const newDependencies = Array.isArray(value) ? value : [value];
newDependencies.forEach(resolvable => {
this.proto.dependencyTargets.push(resolvableAsTarget(resolvable));
});
newDependencies.forEach(resolvable => addDependenciesToActionDependencyTargets(this, resolvable));
return this;
}

Expand Down Expand Up @@ -228,6 +235,11 @@ export class Operation extends ActionBuilder<dataform.Operation> {
return this;
}

public setDependOnDependencyAssertions(dependOnDependencyAssertions: boolean) {
this.dependOnDependencyAssertions = dependOnDependencyAssertions;
return this;
}

/**
* @hidden
*/
Expand Down
41 changes: 27 additions & 14 deletions core/actions/table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ import {
strictKeysOf,
tableTypeStringToEnum,
toResolvable,
validateQueryString
validateQueryString,
addDependenciesToActionDependencyTargets,
} from "df/core/utils";
import { dataform } from "df/protos/ts";

Expand Down Expand Up @@ -157,10 +158,10 @@ const ITableAssertionsProperties = () =>
*/
export interface ITableConfig
extends IActionConfig,
IDependenciesConfig,
IDocumentableConfig,
INamedConfig,
ITargetableConfig {
IDependenciesConfig,
IDocumentableConfig,
INamedConfig,
ITargetableConfig {
/**
* The type of the dataset. For more information on how this setting works, check out some of the [guides](guides)
* on publishing different types of datasets with Dataform.
Expand Down Expand Up @@ -221,7 +222,8 @@ export const ITableConfigProperties = () =>
"database",
"columns",
"description",
"materialized"
"materialized",
"dependOnDependencyAssertions"
]);

/**
Expand Down Expand Up @@ -265,6 +267,9 @@ export class Table extends ActionBuilder<dataform.Table> {
private uniqueKeyAssertions: Assertion[] = [];
private rowConditionsAssertion: Assertion;

// If true, adds the inline assertions of dependencies as direct dependencies for this action.
public dependOnDependencyAssertions: boolean = false;

constructor(
session?: Session,
tableTypeConfig?:
Expand Down Expand Up @@ -340,7 +345,8 @@ export class Table extends ActionBuilder<dataform.Table> {
tags: config.tags,
disabled: config.disabled,
description: config.description,
bigquery: bigqueryOptions
bigquery: bigqueryOptions,
dependOnDependencyAssertions: config.dependOnDependencyAssertions
});
}
if (tableType === "view") {
Expand Down Expand Up @@ -370,7 +376,8 @@ export class Table extends ActionBuilder<dataform.Table> {
materialized: config.materialized,
tags: config.tags,
description: config.description,
bigquery: bigqueryOptions
bigquery: bigqueryOptions,
dependOnDependencyAssertions: config.dependOnDependencyAssertions
});
}
if (tableType === "incremental") {
Expand Down Expand Up @@ -422,7 +429,8 @@ export class Table extends ActionBuilder<dataform.Table> {
uniqueKey: config.uniqueKey,
tags: config.tags,
description: config.description,
bigquery: bigqueryOptions
bigquery: bigqueryOptions,
dependOnDependencyAssertions: config.dependOnDependencyAssertions
});
}
this.query(nativeRequire(tableTypeConfig.filename).query);
Expand All @@ -444,6 +452,9 @@ export class Table extends ActionBuilder<dataform.Table> {
if (config.type) {
this.type(config.type);
}
if (config.dependOnDependencyAssertions) {
this.setDependOnDependencyAssertions(config.dependOnDependencyAssertions);
}
if (config.dependencies) {
this.dependencies(config.dependencies);
}
Expand Down Expand Up @@ -552,10 +563,7 @@ export class Table extends ActionBuilder<dataform.Table> {

public dependencies(value: Resolvable | Resolvable[]) {
const newDependencies = Array.isArray(value) ? value : [value];
newDependencies.forEach(resolvable => {
this.proto.dependencyTargets.push(resolvableAsTarget(resolvable));
});

newDependencies.forEach(resolvable => addDependenciesToActionDependencyTargets(this, resolvable));
return this;
}

Expand Down Expand Up @@ -676,6 +684,11 @@ export class Table extends ActionBuilder<dataform.Table> {
return this;
}

public setDependOnDependencyAssertions(dependOnDependencyAssertions: boolean) {
this.dependOnDependencyAssertions = dependOnDependencyAssertions;
return this;
}

/**
* @hidden
*/
Expand Down Expand Up @@ -740,7 +753,7 @@ export class Table extends ActionBuilder<dataform.Table> {
* @hidden
*/
export class TableContext implements ITableContext {
constructor(private table: Table, private isIncremental = false) {}
constructor(private table: Table, private isIncremental = false) { }

public config(config: ITableConfig) {
this.table.config(config);
Expand Down
7 changes: 7 additions & 0 deletions core/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,11 @@ export interface IDependenciesConfig {
* dependencies, then it should be set to `true`.
*/
hermetic?: boolean;

/**
* If this flag is set to true, assertions depenedent upon any of the dependencies are added as depenedencies as well.
*/
dependOnDependencyAssertions?: boolean;
Tuseeq1 marked this conversation as resolved.
Show resolved Hide resolved
}

/**
Expand Down Expand Up @@ -219,6 +224,8 @@ export interface ITarget {
schema?: string;

name?: string;

includeDependentAssertions?: boolean;
kolina marked this conversation as resolved.
Show resolved Hide resolved
}

/**
Expand Down
Loading
Loading