diff --git a/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt b/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt index 2881a176db..827fd006b3 100644 --- a/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt +++ b/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt @@ -83,10 +83,10 @@ class PartiQLSchemaInferencerTests { @Execution(ExecutionMode.CONCURRENT) fun testJoins(tc: TestCase) = runTest(tc) - // @ParameterizedTest - // @MethodSource("excludeCases") - // @Execution(ExecutionMode.CONCURRENT) - // fun testExclude(tc: TestCase) = runTest(tc) + @ParameterizedTest + @MethodSource("excludeCases") + @Execution(ExecutionMode.CONCURRENT) + fun testExclude(tc: TestCase) = runTest(tc) @ParameterizedTest @MethodSource("orderByCases") diff --git a/partiql-plan/src/main/resources/partiql_plan_0_1.ion b/partiql-plan/src/main/resources/partiql_plan_0_1.ion index c3835b2933..920df2587d 100644 --- a/partiql-plan/src/main/resources/partiql_plan_0_1.ion +++ b/partiql-plan/src/main/resources/partiql_plan_0_1.ion @@ -291,7 +291,30 @@ rel::{ ], }, - err::{}, + exclude::{ + input: rel, + items: list::[item], + _: [ + item::{ + root: '.identifier.symbol', + steps: list::[step], + }, + step::[ + attr::{ + symbol: '.identifier.symbol', + }, + pos::{ + index: int, + }, + struct_wildcard::{}, + collection_wildcard::{}, + ], + ], + }, + + err::{ + message: string, + }, ], _: [ prop::[ diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/Env.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/Env.kt index 55f807059d..161c2cd3d6 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/Env.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/Env.kt @@ -383,7 +383,7 @@ internal class Env( /** * Searches for the path within the given struct, returning null if not found. */ - internal fun inferStructLookup(struct: StructType, path: BindingPath): StaticType? { + private fun inferStructLookup(struct: StructType, path: BindingPath): StaticType? { var curr: StaticType = struct for (step in path.steps) { if (curr !is StructType) { diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/Errors.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/Errors.kt index 857de1b5b8..98992ae41a 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/Errors.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/Errors.kt @@ -117,6 +117,12 @@ sealed class PlanningProblemDetails( severity = ProblemSeverity.ERROR, messageFormatter = { "${actualTypes.joinToString()} is/are incompatible data types for the '$operator' operator." } ) + + data class UnresolvedExcludeExprRoot(val root: String) : + PlanningProblemDetails( + ProblemSeverity.ERROR, + { "Exclude expression given an unresolvable root '$root'" } + ) } private fun quotationHint(caseSensitive: Boolean) = diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/transforms/RelConverter.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/transforms/RelConverter.kt index c0f840659c..418762424e 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/transforms/RelConverter.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/transforms/RelConverter.kt @@ -17,6 +17,7 @@ package org.partiql.planner.transforms import org.partiql.ast.AstNode +import org.partiql.ast.Exclude import org.partiql.ast.Expr import org.partiql.ast.From import org.partiql.ast.GroupBy @@ -36,6 +37,12 @@ import org.partiql.plan.relOpAggregate import org.partiql.plan.relOpAggregateAgg import org.partiql.plan.relOpErr import org.partiql.plan.relOpExcept +import org.partiql.plan.relOpExclude +import org.partiql.plan.relOpExcludeItem +import org.partiql.plan.relOpExcludeStepAttr +import org.partiql.plan.relOpExcludeStepCollectionWildcard +import org.partiql.plan.relOpExcludeStepPos +import org.partiql.plan.relOpExcludeStepStructWildcard import org.partiql.plan.relOpFilter import org.partiql.plan.relOpIntersect import org.partiql.plan.relOpJoin @@ -60,7 +67,6 @@ import org.partiql.plan.rexOpTupleUnionArgSpread import org.partiql.plan.rexOpTupleUnionArgStruct import org.partiql.plan.rexOpVarResolved import org.partiql.planner.Env -import org.partiql.types.ListType import org.partiql.types.StaticType import org.partiql.value.PartiQLValueExperimental import org.partiql.value.boolValue @@ -72,7 +78,7 @@ import org.partiql.value.stringValue internal object RelConverter { // IGNORE — so we don't have to non-null assert on operator inputs - private val nil = rel(relType(emptyList(), emptySet()), relOpErr()) + private val nil = rel(relType(emptyList(), emptySet()), relOpErr("nil")) /** * Here we convert an SFW to composed [Rel]s, then apply the appropriate relation-value projection to get a [Rex]. @@ -225,6 +231,7 @@ internal object RelConverter { rel = convertOrderBy(rel, sel.orderBy) rel = convertLimit(rel, sel.limit) rel = convertOffset(rel, sel.offset) + rel = convertExclude(rel, sel.exclude) // append SQL projection if present rel = when (val projection = sel.select) { is Select.Project -> visitSelectProject(projection, rel) @@ -528,6 +535,29 @@ internal object RelConverter { return rel(type, op) } + private fun convertExclude(input: Rel, exclude: Exclude?): Rel { + if (exclude == null) { + return input + } + val type = input.type // PlanTyper handles typing the exclusion + val items = exclude.exprs.map { convertExcludeItem(it) } + val op = relOpExclude(input, items) + return rel(type, op) + } + + private fun convertExcludeItem(expr: Exclude.ExcludeExpr): Rel.Op.Exclude.Item { + val root = AstToPlan.convert(expr.root) + val steps = expr.steps.map { convertExcludeStep(it) } + return relOpExcludeItem(root, steps) + } + + private fun convertExcludeStep(step: Exclude.Step): Rel.Op.Exclude.Step = when (step) { + is Exclude.Step.ExcludeTupleAttr -> relOpExcludeStepAttr(AstToPlan.convert(step.symbol)) + is Exclude.Step.ExcludeCollectionIndex -> relOpExcludeStepPos(step.index) + is Exclude.Step.ExcludeCollectionWildcard -> relOpExcludeStepCollectionWildcard() + is Exclude.Step.ExcludeTupleWildcard -> relOpExcludeStepStructWildcard() + } + // /** // * Converts a GROUP AS X clause to a binding of the form: // * ``` diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/typer/PlanTyper.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/typer/PlanTyper.kt index 3b6a4b881d..02e0c25556 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/typer/PlanTyper.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/typer/PlanTyper.kt @@ -154,7 +154,7 @@ internal class PlanTyper( // only UNPIVOT a struct if (rex.type !is StructType) { handleUnexpectedType(rex.type, expected = setOf(StaticType.STRUCT)) - return rel(ctx!!, relOpErr()) + return rel(ctx!!, relOpErr("UNPIVOT on non-STRUCT type ${rex.type}")) } // compute element type @@ -291,6 +291,47 @@ internal class PlanTyper( return rel(type, op) } + /** + * Initial implementation of `EXCLUDE` schema inference. Until an RFC is finalized for `EXCLUDE` + * (https://github.com/partiql/partiql-spec/issues/39), this behavior is considered experimental and subject to + * change. + * + * So far this implementation includes + * - Excluding tuple bindings (e.g. t.a.b.c) + * - Excluding tuple wildcards (e.g. t.a.*.b) + * - Excluding collection indexes (e.g. t.a[0].b -- behavior subject to change; see below discussion) + * - Excluding collection wildcards (e.g. t.a[*].b) + * + * There are still discussion points regarding the following edge cases: + * - EXCLUDE on a tuple bindingibute that doesn't exist -- give an error/warning? + * - currently no error + * - EXCLUDE on a tuple bindingibute that has duplicates -- give an error/warning? exclude one? exclude both? + * - currently excludes both w/ no error + * - EXCLUDE on a collection index as the last step -- mark element type as optional? + * - currently element type as-is + * - EXCLUDE on a collection index w/ remaining path steps -- mark last step's type as optional? + * - currently marks last step's type as optional + * - EXCLUDE on a binding tuple variable (e.g. SELECT ... EXCLUDE t FROM t) -- error? + * - currently a parser error + * - EXCLUDE on a union type -- give an error/warning? no-op? exclude on each type in union? + * - currently exclude on each union type + * - If SELECT list includes an bindingibute that is excluded, we could consider giving an error in PlanTyper or + * some other semantic pass + * - currently does not give an error + */ + override fun visitRelOpExclude(node: Rel.Op.Exclude, ctx: Rel.Type?): Rel { + // compute input schema + val input = visitRel(node.input, ctx) + + // apply exclusions to the input schema + val init = input.type.schema.map { it.copy() } + val schema = node.items.fold((init)) { bindings, item -> excludeBindings(bindings, item) } + + // rewrite + val type = ctx!!.copy(schema) + return rel(type, node) + } + override fun visitRelOpAggregate(node: Rel.Op.Aggregate, ctx: Rel.Type?): Rel { TODO("Type RelOp Aggregate") } @@ -898,6 +939,17 @@ internal class PlanTyper( ) } + private fun handleUnresolvedExcludeRoot(root: String) { + onProblem( + Problem( + sourceLocation = UNKNOWN_PROBLEM_LOCATION, + details = PlanningProblemDetails.UnresolvedExcludeExprRoot(root) + ) + ) + } + + // HELPERS + private fun Identifier.normalize(): String = when (this) { is Identifier.Qualified -> (listOf(root.normalize()) + steps.map { it.normalize() }).joinToString(".") is Identifier.Symbol -> when (caseSensitivity) { @@ -930,4 +982,25 @@ internal class PlanTyper( private fun StructType.withNullableFields(): StructType { return copy(fields.map { it.copy(value = it.value.asNullable()) }) } + + private fun excludeBindings(input: List, item: Rel.Op.Exclude.Item): List { + var matchedRoot = false + val output = input.map { + if (item.root.isEquivalentTo(it.name)) { + matchedRoot = true + // recompute the StaticType of this binding after apply the exclusions + val type = it.type.exclude(item.steps, false) + it.copy(type = type) + } else { + it + } + } + if (!matchedRoot) handleUnresolvedExcludeRoot(item.root.symbol) + return output + } + + private fun Identifier.Symbol.isEquivalentTo(other: String): Boolean = when (caseSensitivity) { + Identifier.CaseSensitivity.SENSITIVE -> symbol.equals(other) + Identifier.CaseSensitivity.INSENSITIVE -> symbol.equals(other, ignoreCase = true) + } } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/typer/TypeUtils.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/typer/TypeUtils.kt index 46d35f4d11..64592ed29d 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/typer/TypeUtils.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/typer/TypeUtils.kt @@ -1,11 +1,14 @@ package org.partiql.planner.typer +import org.partiql.plan.Identifier +import org.partiql.plan.Rel import org.partiql.types.AnyOfType import org.partiql.types.AnyType import org.partiql.types.BagType import org.partiql.types.BlobType import org.partiql.types.BoolType import org.partiql.types.ClobType +import org.partiql.types.CollectionType import org.partiql.types.DateType import org.partiql.types.DecimalType import org.partiql.types.FloatType @@ -125,3 +128,101 @@ private fun StaticType.asRuntimeType(): PartiQLValueType = when (this) { is TimeType -> PartiQLValueType.TIME is TimestampType -> PartiQLValueType.TIMESTAMP } + +/** + * Applies the given exclusion path to produce the reduced StaticType + * + * @param steps + * @param lastStepOptional + * @return + */ +internal fun StaticType.exclude(steps: List, lastStepOptional: Boolean = true): StaticType = + when (this) { + is StructType -> this.exclude(steps, lastStepOptional) + is CollectionType -> this.exclude(steps, lastStepOptional) + is AnyOfType -> StaticType.unionOf( + this.types.map { it.exclude(steps, lastStepOptional) }.toSet() + ) + else -> this + }.flatten() + +/** + * Applies exclusions to struct fields. + * + * @param steps + * @param lastStepOptional + * @return + */ +internal fun StructType.exclude(steps: List, lastStepOptional: Boolean = true): StaticType { + val step = steps.first() + val output = fields.map { field -> + val newField = if (steps.size == 1) { + if (lastStepOptional) { + StructType.Field(field.key, field.value.asOptional()) + } else { + null + } + } else { + val k = field.key + val v = field.value.exclude(steps.drop(1), lastStepOptional) + StructType.Field(k, v) + } + when (step) { + is Rel.Op.Exclude.Step.Attr -> { + if (step.symbol.isEquivalentTo(field.key)) { + newField + } else { + field + } + } + is Rel.Op.Exclude.Step.StructWildcard -> newField + else -> field + } + }.filterNotNull() + return this.copy(fields = output) +} + +/** + * Applies exclusions to collection element type. + * + * @param steps + * @param lastStepOptional + * @return + */ +internal fun CollectionType.exclude(steps: List, lastStepOptional: Boolean = true): StaticType { + var e = this.elementType + when (steps.first()) { + is Rel.Op.Exclude.Step.Pos -> { + if (steps.size > 1) { + e = e.exclude(steps.drop(1), true) + } + } + is Rel.Op.Exclude.Step.CollectionWildcard -> { + if (steps.size > 1) { + e = e.exclude(steps.drop(1), lastStepOptional) + } + // currently no change to elementType if collection wildcard is last element; this behavior could + // change based on RFC definition + } + else -> { + // currently no change to elementType and no error thrown; could consider an error/warning in + // the future + } + } + return when (this) { + is BagType -> this.copy(e) + is ListType -> this.copy(e) + is SexpType -> this.copy(e) + } +} + +/** + * Compare an identifier to a struct field; handling case-insensitive comparisons. + * + * @param other + * @return + */ +private fun Identifier.Symbol.isEquivalentTo(other: String): Boolean = when (caseSensitivity) { + Identifier.CaseSensitivity.SENSITIVE -> symbol.equals(other) + Identifier.CaseSensitivity.INSENSITIVE -> symbol.equals(other, ignoreCase = true) +}