Add validation for extra properties in column descriptors (#714)

* Add validation for extra properties in column descriptors

* Expand excess property testing

* Add warehouse checks and fix lints

* Add expect, newline eof

* Bump version

core/assertion.ts

@@ -5,7 +5,13 @@ import {
   Resolvable
 } from "@dataform/core/common";
 import { Session } from "@dataform/core/session";
-import * as utils from "@dataform/core/utils";
+import {
+  checkExcessProperties,
+  resolvableAsTarget,
+  setNameAndTarget,
+  strictKeysOf,
+  toResolvable
+} from "@dataform/core/utils";
 import { dataform } from "@dataform/protos";
 
 /**
@@ -28,6 +34,16 @@ export interface IAssertionConfig extends ITargetableConfig, IDependenciesConfig
   description?: string;
 }
 
+export const IAssertionConfigProperties = strictKeysOf<IAssertionConfig>()([
+  "database",
+  "schema",
+  "name",
+  "description",
+  "type",
+  "tags",
+  "dependencies"
+]);
+
 /**
  * @hidden
  */
@@ -46,6 +62,7 @@ export class Assertion {
   private contextableQuery: AContextable<string>;
 
   public config(config: IAssertionConfig) {
+    checkExcessProperties(config, IAssertionConfigProperties, "assertion config");
     if (config.dependencies) {
       this.dependencies(config.dependencies);
     }
@@ -72,7 +89,7 @@ export class Assertion {
   public dependencies(value: Resolvable | Resolvable[]) {
     const newDependencies = Array.isArray(value) ? value : [value];
     newDependencies.forEach(resolvable => {
-      this.proto.dependencyTargets.push(utils.resolvableAsTarget(resolvable));
+      this.proto.dependencyTargets.push(resolvableAsTarget(resolvable));
     });
     return this;
   }
@@ -93,7 +110,7 @@ export class Assertion {
   }
 
   public database(database: string) {
-    utils.setNameAndTarget(
+    setNameAndTarget(
       this.session,
       this.proto,
       this.proto.target.name,
@@ -104,7 +121,7 @@ export class Assertion {
   }
 
   public schema(schema: string) {
-    utils.setNameAndTarget(this.session, this.proto, this.proto.target.name, schema);
+    setNameAndTarget(this.session, this.proto, this.proto.target.name, schema);
     return this;
   }
 
@@ -137,8 +154,8 @@ export class AssertionContext implements ICommonContext {
   }
 
   public ref(ref: Resolvable | string[], ...rest: string[]) {
-    ref = utils.toResolvable(ref, rest);
-    if (!utils.resolvableAsTarget(ref)) {
+    ref = toResolvable(ref, rest);
+    if (!resolvableAsTarget(ref)) {
       const message = `Action name is not specified`;
       this.assertion.session.compileError(new Error(message));
       return "";
@@ -148,7 +165,7 @@ export class AssertionContext implements ICommonContext {
   }
 
   public resolve(ref: Resolvable | string[], ...rest: string[]) {
-    return this.assertion.session.resolve(utils.toResolvable(ref, rest));
+    return this.assertion.session.resolve(toResolvable(ref, rest));
   }
 
   public dependencies(name: Resolvable | Resolvable[]) {

core/column_descriptors.ts

@@ -0,0 +1,90 @@
+import {
+  IColumnsDescriptor,
+  IRecordDescriptor,
+  IRecordDescriptorProperties
+} from "@dataform/core/common";
+import * as utils from "@dataform/core/utils";
+import { dataform } from "@dataform/protos";
+
+/**
+ * @hidden
+ */
+export class ColumnDescriptors {
+  public static mapToColumnProtoArray(columns: IColumnsDescriptor): dataform.IColumnDescriptor[] {
+    return utils.flatten(
+      Object.keys(columns).map(column =>
+        ColumnDescriptors.mapColumnDescriptionToProto([column], columns[column])
+      )
+    );
+  }
+
+  public static mapColumnDescriptionToProto(
+    currentPath: string[],
+    description: string | IRecordDescriptor
+  ): dataform.IColumnDescriptor[] {
+    if (typeof description === "string") {
+      return [
+        dataform.ColumnDescriptor.create({
+          description,
+          path: currentPath
+        })
+      ];
+    }
+    utils.checkExcessProperties(
+      description,
+      IRecordDescriptorProperties(),
+      `${currentPath.join(".")} column descriptor`
+    );
+    const columnDescriptor: dataform.IColumnDescriptor[] = !!description
+      ? [
+          dataform.ColumnDescriptor.create({
+            path: currentPath,
+            description: description.description,
+            displayName: description.displayName,
+            dimensionType: ColumnDescriptors.mapDimensionType(description.dimension),
+            aggregation: ColumnDescriptors.mapAggregation(description.aggregator),
+            expression: description.expression
+          })
+        ]
+      : [];
+    const nestedColumns = description.columns ? Object.keys(description.columns) : [];
+    return columnDescriptor.concat(
+      utils.flatten(
+        nestedColumns.map(nestedColumn =>
+          ColumnDescriptors.mapColumnDescriptionToProto(
+            currentPath.concat([nestedColumn]),
+            description.columns[nestedColumn]
+          )
+        )
+      )
+    );
+  }
+
+  public static mapAggregation(aggregation: string) {
+    switch (aggregation) {
+      case "sum":
+        return dataform.ColumnDescriptor.Aggregation.SUM;
+      case "distinct":
+        return dataform.ColumnDescriptor.Aggregation.DISTINCT;
+      case "derived":
+        return dataform.ColumnDescriptor.Aggregation.DERIVED;
+      case undefined:
+        return undefined;
+      default:
+        throw new Error(`'${aggregation}' is not a valid aggregation option.`);
+    }
+  }
+
+  public static mapDimensionType(dimensionType: string) {
+    switch (dimensionType) {
+      case "category":
+        return dataform.ColumnDescriptor.DimensionType.CATEGORY;
+      case "timestamp":
+        return dataform.ColumnDescriptor.DimensionType.TIMESTAMP;
+      case undefined:
+        return undefined;
+      default:
+        throw new Error(`'${dimensionType}' is not a valid dimension type.`);
+    }
+  }
+}

core/common.ts

@@ -1,3 +1,5 @@
+import { strictKeysOf } from "@dataform/core/utils";
+
 /**
  * Context methods are available when evaluating contextable SQL code, such as
  * within SQLX files, or when using a [Contextable](#Contextable) argument with the JS API.
@@ -49,10 +51,31 @@ export interface ICommonContext {
  * @hidden
  */
 export interface ICommonConfig {
+  /**
+   * The type of the action.
+   *
+   * @hidden
+   */
+  type?: string;
+
   /**
    * A list of user-defined tags with which the action should be labeled.
    */
   tags?: string[];
+
+  /**
+   * The name of the action.
+   *
+   * @hidden
+   */
+  name?: string;
+
+  /**
+   * Dependencies of the action.
+   *
+   * @hidden
+   */
+  dependencies?: Resolvable | Resolvable[];
 }
 
 /**
@@ -138,6 +161,21 @@ export interface IRecordDescriptor {
   expression?: string;
 }
 
+/**
+ * @hidden
+ *
+ * TODO: This needs to be a method, I'm really not sure why, but it hits a runtime failure otherwise.
+ */
+export const IRecordDescriptorProperties = () =>
+  strictKeysOf<IRecordDescriptor>()([
+    "description",
+    "columns",
+    "displayName",
+    "dimension",
+    "aggregator",
+    "expression"
+  ]);
+
 /**
  * A reference to a dataset within the warehouse.
  */

core/declaration.ts

@@ -1,12 +1,24 @@
+import { ColumnDescriptors } from "@dataform/core/column_descriptors";
 import { IColumnsDescriptor, IDocumentableConfig, ITargetableConfig } from "@dataform/core/common";
-import { mapToColumnProtoArray, Session } from "@dataform/core/session";
+import { Session } from "@dataform/core/session";
+import { checkExcessProperties, strictKeysOf } from "@dataform/core/utils";
 import { dataform } from "@dataform/protos";
-
 /**
  * Configuration options for `declaration` action types.
  */
 export interface IDeclarationConfig extends IDocumentableConfig, ITargetableConfig {}
 
+export const IDeclarationConfigProperties = strictKeysOf<IDeclarationConfig>()([
+  "type",
+  "name",
+  "tags",
+  "schema",
+  "database",
+  "columns",
+  "description",
+  "dependencies"
+]);
+
 /**
  * @hidden
  */
@@ -16,6 +28,7 @@ export class Declaration {
   public session: Session;
 
   public config(config: IDeclarationConfig) {
+    checkExcessProperties(config, IDeclarationConfigProperties, "declaration config");
     if (config.description) {
       this.description(config.description);
     }
@@ -37,7 +50,7 @@ export class Declaration {
     if (!this.proto.actionDescriptor) {
       this.proto.actionDescriptor = {};
     }
-    this.proto.actionDescriptor.columns = mapToColumnProtoArray(columns);
+    this.proto.actionDescriptor.columns = ColumnDescriptors.mapToColumnProtoArray(columns);
     return this;
   }
 

core/operation.ts

@@ -1,13 +1,14 @@
+import { ColumnDescriptors } from "@dataform/core/column_descriptors";
 import {
+  Contextable,
   IColumnsDescriptor,
   ICommonContext,
   IDependenciesConfig,
   IDocumentableConfig,
   ITargetableConfig,
   Resolvable
 } from "@dataform/core/common";
-import { Contextable } from "@dataform/core/common";
-import { mapToColumnProtoArray, Session } from "@dataform/core/session";
+import { Session } from "@dataform/core/session";
 import * as utils from "@dataform/core/utils";
 import { dataform } from "@dataform/protos";
 
@@ -108,7 +109,7 @@ export class Operation {
     if (!this.proto.actionDescriptor) {
       this.proto.actionDescriptor = {};
     }
-    this.proto.actionDescriptor.columns = mapToColumnProtoArray(columns);
+    this.proto.actionDescriptor.columns = ColumnDescriptors.mapToColumnProtoArray(columns);
     return this;
   }