Skip to content

Commit

Permalink
Always run auto assertions (#959)
Browse files Browse the repository at this point in the history
* Always run auto assertions

* Change parent to be a target, tidied attachment

* Add name fixing

* Change assertion in prune to use target

* Tidy IAction, rename to actionParent, bump version

* Added missing no null check for parent

* Add auto assertions to caching test
  • Loading branch information
Ekrekr authored Aug 21, 2020
1 parent 16e1033 commit 3db05a5
Show file tree
Hide file tree
Showing 8 changed files with 50 additions and 25 deletions.
9 changes: 9 additions & 0 deletions api/commands/prune.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,5 +77,14 @@ function computeIncludedActionNames(
}
}

// Add auto assertions
[...compiledGraph.assertions].forEach(assertion => {
if (!!assertion.parentAction) {
if (includedActionNames.has(utils.targetToName(assertion.parentAction))) {
includedActionNames.add(utils.targetToName(assertion.target));
}
}
});

return includedActionNames;
}
5 changes: 5 additions & 0 deletions core/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ export interface IActionProto {
hermeticity?: dataform.ActionHermeticity;
target?: dataform.ITarget;
canonicalTarget?: dataform.ITarget;
parentAction?: dataform.ITarget;
}

type SqlxConfig = (
Expand Down Expand Up @@ -515,6 +516,10 @@ export class Session {
action.dependencies = (action.dependencyTargets || []).map(dependencyTarget =>
utils.targetToName(dependencyTarget)
);

if (!!action.parentAction) {
action.parentAction = newTargetByOriginalTarget.get(action.parentAction);
}
});
}

Expand Down
28 changes: 8 additions & 20 deletions core/table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -299,8 +299,6 @@ export class Table {
private contextableWhere: Contextable<ITableContext, string>;
private contextablePreOps: Array<Contextable<ITableContext, string | string[]>> = [];
private contextablePostOps: Array<Contextable<ITableContext, string | string[]>> = [];
private uniqueKeyAssertion?: Assertion;
private mergedRowConditionsAssertion?: Assertion;

public config(config: ITableConfig) {
checkExcessProperties(
Expand Down Expand Up @@ -450,12 +448,6 @@ export class Table {
newTags.forEach(t => {
this.proto.tags.push(t);
});
if (!!this.uniqueKeyAssertion) {
this.uniqueKeyAssertion.tags(value);
}
if (!!this.mergedRowConditionsAssertion) {
this.mergedRowConditionsAssertion.tags(value);
}
return this;
}

Expand Down Expand Up @@ -510,11 +502,9 @@ export class Table {
if (!!assertions.uniqueKey) {
const indexCols =
typeof assertions.uniqueKey === "string" ? [assertions.uniqueKey] : assertions.uniqueKey;
this.uniqueKeyAssertion = this.session
.assert(`${this.proto.target.name}_assertions_uniqueKey`, ctx =>
this.session.adapter().indexAssertion(ctx.ref(this.proto.target), indexCols)
)
.tags(this.proto.tags);
this.session.assert(`${this.proto.target.name}_assertions_uniqueKey`, ctx =>
this.session.adapter().indexAssertion(ctx.ref(this.proto.target), indexCols)
).proto.parentAction = this.proto.target;
}
const mergedRowConditions = assertions.rowConditions || [];
if (!!assertions.nonNull) {
Expand All @@ -523,13 +513,11 @@ export class Table {
nonNullCols.forEach(nonNullCol => mergedRowConditions.push(`${nonNullCol} IS NOT NULL`));
}
if (!!mergedRowConditions && mergedRowConditions.length > 0) {
this.mergedRowConditionsAssertion = this.session
.assert(`${this.proto.target.name}_assertions_rowConditions`, ctx =>
this.session
.adapter()
.rowConditionsAssertion(ctx.ref(this.proto.target), mergedRowConditions)
)
.tags(this.proto.tags);
this.session.assert(`${this.proto.target.name}_assertions_rowConditions`, ctx =>
this.session
.adapter()
.rowConditionsAssertion(ctx.ref(this.proto.target), mergedRowConditions)
).proto.parentAction = this.proto.target;
}
return this;
}
Expand Down
3 changes: 3 additions & 0 deletions protos/core.proto
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,9 @@ message Assertion {
repeated string tags = 9;

ActionDescriptor action_descriptor = 10;

// Only present for auto assertions.
Target parent_action = 15;

// Generated.
string file_name = 7;
Expand Down
21 changes: 19 additions & 2 deletions tests/api/api.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,19 @@ suite("@dataform/api", () => {
},
query: "query"
}
],
assertions: [
{
name: "schema.d",
target: {
schema: "schema",
name: "d"
},
parentAction: {
schema: "schema",
name: "b"
}
}
]
});

Expand Down Expand Up @@ -325,20 +338,24 @@ suite("@dataform/api", () => {
const prunedGraph = prune(TEST_GRAPH, { actions: ["schema.a"], includeDependencies: true });
const actionNames = [
...prunedGraph.tables.map(action => action.name),
...prunedGraph.operations.map(action => action.name)
...prunedGraph.operations.map(action => action.name),
...prunedGraph.assertions.map(action => action.name)
];
expect(actionNames).includes("schema.a");
expect(actionNames).includes("schema.b");
expect(actionNames).includes("schema.d");
});

test("prune actions with --actions without dependencies", () => {
const prunedGraph = prune(TEST_GRAPH, { actions: ["schema.a"], includeDependencies: false });
const actionNames = [
...prunedGraph.tables.map(action => action.name),
...prunedGraph.operations.map(action => action.name)
...prunedGraph.operations.map(action => action.name),
...prunedGraph.assertions.map(action => action.name)
];
expect(actionNames).includes("schema.a");
expect(actionNames).not.includes("schema.b");
expect(actionNames).not.includes("schema.d");
});
});

Expand Down
2 changes: 0 additions & 2 deletions tests/api/examples.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,6 @@ suite("examples", () => {
name: "example_table_with_tags"
})
]);
expect(exampleTableWithTagsUniqueKeyAssertion.tags).eql(["tag1", "tag2", "tag3"]);

// Check table-with-tags's row conditions assertion
const exampleTableWithTagsRowConditionsAssertion = graph.assertions.filter(
Expand All @@ -347,7 +346,6 @@ suite("examples", () => {
name: "example_table_with_tags"
})
]);
expect(exampleTableWithTagsRowConditionsAssertion.tags).eql(["tag1", "tag2", "tag3"]);

// Check sample data
const exampleSampleData = graph.tables.find(
Expand Down
5 changes: 5 additions & 0 deletions tests/integration/bigquery.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,11 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after })
// Should run because the dataset has changed in the warehouse.
"dataform-integration-tests.df_integration_test_run_caching.sample_data_2":
dataform.ActionResult.ExecutionStatus.SUCCESSFUL,
// Should run because they are auto assertions.
"dataform-integration-tests.df_integration_test_assertions_run_caching.sample_data_2_assertions_uniqueKey":
dataform.ActionResult.ExecutionStatus.SUCCESSFUL,
"dataform-integration-tests.df_integration_test_assertions_run_caching.sample_data_2_assertions_rowConditions":
dataform.ActionResult.ExecutionStatus.SUCCESSFUL,
// Should run because an input to dataset has changed in the warehouse.
"dataform-integration-tests.df_integration_test_run_caching.depends_on_sample_data_3":
dataform.ActionResult.ExecutionStatus.SUCCESSFUL,
Expand Down
2 changes: 1 addition & 1 deletion version.bzl
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
# NOTE: If you change the format of this line, you must change the bash command
# in /scripts/publish to extract the version string correctly.
DF_VERSION = "1.9.0"
DF_VERSION = "1.9.1"

0 comments on commit 3db05a5

Please sign in to comment.