From 1c3c79a40f686fd6526b9d715301c388a9eab315 Mon Sep 17 00:00:00 2001 From: "R. C. Howell" Date: Fri, 8 Dec 2023 11:33:58 -0800 Subject: [PATCH] Refactors partiql-tests-runner for multiple engines (#1289) --- .github/workflows/conformance-report.yml | 4 +- test/partiql-tests-runner/README.md | 6 +- test/partiql-tests-runner/build.gradle.kts | 6 +- .../org/partiql/runner/ConformanceTest.kt | 44 +++ .../partiql/runner/ConformanceTestReport.kt | 44 +++ .../src/test/kotlin/org/partiql/runner/Ion.kt | 8 + .../test/kotlin/org/partiql/runner/Schema.kt | 51 ---- .../partiql/runner/report/ReportGenerator.kt | 55 ++++ .../org/partiql/runner/schema/Assertion.kt | 9 + .../partiql/runner/schema/EquivalenceClass.kt | 6 + .../org/partiql/runner/schema/Namespace.kt | 10 + .../org/partiql/runner/{ => schema}/Parse.kt | 6 +- .../org/partiql/runner/schema/TestCase.kt | 35 +++ .../runner/{TestRunner.kt => skip/Failing.kt} | 272 +----------------- .../org/partiql/runner/test/TestExecutor.kt | 59 ++++ .../org/partiql/runner/test/TestLoader.kt | 49 ++++ .../TestProvider.kt} | 36 ++- .../org/partiql/runner/test/TestRunner.kt | 54 ++++ .../runner/test/executor/LegacyExecutor.kt | 46 +++ .../ValueEquals.kt} | 45 ++- 20 files changed, 489 insertions(+), 356 deletions(-) create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTest.kt create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestReport.kt create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Ion.kt delete mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Schema.kt create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/report/ReportGenerator.kt create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Assertion.kt create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/EquivalenceClass.kt create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Namespace.kt rename test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/{ => schema}/Parse.kt (97%) create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/TestCase.kt rename test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/{TestRunner.kt => skip/Failing.kt} (65%) create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestExecutor.kt create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestLoader.kt rename test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/{ArgumentsProviderBase.kt => test/TestProvider.kt} (52%) create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestRunner.kt create mode 100644 test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/executor/LegacyExecutor.kt rename test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/{PartiQLEqualityChecker.kt => util/ValueEquals.kt} (75%) diff --git a/.github/workflows/conformance-report.yml b/.github/workflows/conformance-report.yml index 1c2b7da21..f7a25de85 100644 --- a/.github/workflows/conformance-report.yml +++ b/.github/workflows/conformance-report.yml @@ -27,7 +27,7 @@ jobs: # Run the conformance tests and save to an Ion file. - name: gradle test of the conformance tests (can fail) and save to Ion file continue-on-error: true - run: gradle :test:partiql-tests-runner:test --tests "*ConformanceTestsReportRunner" -PconformanceReport + run: gradle :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport # Upload conformance report for future viewing and comparison with future runs. - name: Upload `conformance_test_results.ion` uses: actions/upload-artifact@v3 @@ -86,7 +86,7 @@ jobs: continue-on-error: true run: | cd ${{ github.event.pull_request.base.sha }} - gradle :test:partiql-tests-runner:test --tests "*ConformanceTestsReportRunner" -PconformanceReport + gradle :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport - name: (If download of target branch conformance report fails) Move conformance test report of target branch to ./artifact directory if: ${{ steps.download-report.outcome == 'failure' }} continue-on-error: true diff --git a/test/partiql-tests-runner/README.md b/test/partiql-tests-runner/README.md index 9379ff849..9ad5c065f 100644 --- a/test/partiql-tests-runner/README.md +++ b/test/partiql-tests-runner/README.md @@ -12,11 +12,11 @@ This package enables: ```shell # default, test data from partiql-tests submodule will be used -./gradlew :test:partiql-tests-runner:test --tests "*ConformanceTestsReportRunner" -PconformanceReport +./gradlew :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport # override test data location PARTIQL_TESTS_DATA=/path/to/partiql-tests/data \ -./gradlew :test:partiql-tests-runner:test --tests "*ConformanceTestsReportRunner" -PconformanceReport +./gradlew :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport ``` The report is written into file `test/partiql-tests-runner/conformance_test_results.ion`. @@ -24,7 +24,7 @@ The report is written into file `test/partiql-tests-runner/conformance_test_resu The above project property `-PconformanceReport` is checked in `test/partiql-tests-runner/build.gradle.kts`, to exclude the conformance test suite from executing during a normal project-build test run. -Unfortunately, this also disables running `ConformanceTestsReportRunner` in a UI runner. +Unfortunately, this also disables running `ConformanceTestReport` in a UI runner. To make that possible locally, temporarily comment out the check in `test/partiql-tests-runner/build.gradle.kts`. ## Compare Conformance Reports locally diff --git a/test/partiql-tests-runner/build.gradle.kts b/test/partiql-tests-runner/build.gradle.kts index 0c58bcdaf..f6c7cdf27 100644 --- a/test/partiql-tests-runner/build.gradle.kts +++ b/test/partiql-tests-runner/build.gradle.kts @@ -40,11 +40,11 @@ tasks.test { environment(Env.PARTIQL_EVAL, file("$tests/eval/").absolutePath) environment(Env.PARTIQL_EQUIV, file("$tests/eval-equiv/").absolutePath) - // To make it possible to run ConformanceTestsReportRunner in unit test UI runner, comment out this check: + // To make it possible to run ConformanceTestReport in unit test UI runner, comment out this check: if (!project.hasProperty("conformanceReport")) { - exclude("org/partiql/runner/TestRunner\$ConformanceTestsReportRunner.class") + exclude("org/partiql/runner/ConformanceTestReport.class") } // May 2023: Disabled conformance testing during regular project build, because fail lists are out of date. - exclude("org/partiql/runner/TestRunner\$DefaultConformanceTestRunner.class") + exclude("org/partiql/runner/ConformanceTest.class") } diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTest.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTest.kt new file mode 100644 index 000000000..b08d6489c --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTest.kt @@ -0,0 +1,44 @@ +package org.partiql.runner + +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ArgumentsSource +import org.partiql.runner.schema.TestCase +import org.partiql.runner.skip.LANG_KOTLIN_EVAL_EQUIV_FAIL_LIST +import org.partiql.runner.skip.LANG_KOTLIN_EVAL_FAIL_LIST +import org.partiql.runner.test.TestProvider +import org.partiql.runner.test.TestRunner +import org.partiql.runner.test.executor.LegacyExecutor + +/** + * Runs the conformance tests with an expected list of failing tests. Ensures that tests not in the failing list + * succeed with the expected result. Ensures that tests included in the failing list fail. + * + * These tests are included in the normal test/building. + * Update May 2023: Now excluded from the normal build, because the fail lists are out of date. + * TODO: Come up with a low-burden method of maintaining fail / exclusion lists. + */ +class ConformanceTest { + + private val factory = LegacyExecutor.Factory + private val runner = TestRunner(factory) + + // Tests the eval tests with the Kotlin implementation + @ParameterizedTest(name = "{arguments}") + @ArgumentsSource(TestProvider.Eval::class) + fun validatePartiQLEvalTestData(tc: TestCase) { + when (tc) { + is TestCase.Eval -> runner.test(tc, LANG_KOTLIN_EVAL_FAIL_LIST) + else -> error("Unsupported test case category") + } + } + + // Tests the eval equivalence tests with the Kotlin implementation + @ParameterizedTest(name = "{arguments}") + @ArgumentsSource(TestProvider.Equiv::class) + fun validatePartiQLEvalEquivTestData(tc: TestCase) { + when (tc) { + is TestCase.Equiv -> runner.test(tc, LANG_KOTLIN_EVAL_EQUIV_FAIL_LIST) + else -> error("Unsupported test case category") + } + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestReport.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestReport.kt new file mode 100644 index 000000000..56da4943a --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestReport.kt @@ -0,0 +1,44 @@ +package org.partiql.runner + +import org.junit.jupiter.api.extension.ExtendWith +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ArgumentsSource +import org.partiql.runner.report.ReportGenerator +import org.partiql.runner.schema.TestCase +import org.partiql.runner.test.TestProvider +import org.partiql.runner.test.TestRunner +import org.partiql.runner.test.executor.LegacyExecutor + +/** + * Runs the conformance tests without a fail list, so we can document the passing/failing tests in the conformance + * report. + * + * These tests are excluded from normal testing/building unless the `conformanceReport` gradle property is + * specified (i.e. `gradle test ... -PconformanceReport`) + */ +@ExtendWith(ReportGenerator::class) +class ConformanceTestReport { + + private val factory = LegacyExecutor.Factory + private val runner = TestRunner(factory) + + // Tests the eval tests with the Kotlin implementation without a fail list + @ParameterizedTest(name = "{arguments}") + @ArgumentsSource(TestProvider.Eval::class) + fun validatePartiQLEvalTestData(tc: TestCase) { + when (tc) { + is TestCase.Eval -> runner.test(tc, emptyList()) + else -> error("Unsupported test case category") + } + } + + // Tests the eval equivalence tests with the Kotlin implementation without a fail list + @ParameterizedTest(name = "{arguments}") + @ArgumentsSource(TestProvider.Equiv::class) + fun validatePartiQLEvalEquivTestData(tc: TestCase) { + when (tc) { + is TestCase.Equiv -> runner.test(tc, emptyList()) + else -> error("Unsupported test case category") + } + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Ion.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Ion.kt new file mode 100644 index 000000000..25a1b88ec --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Ion.kt @@ -0,0 +1,8 @@ +package org.partiql.runner + +import com.amazon.ion.system.IonSystemBuilder + +/** + * IonSystem for legacy pipelines and value comparison. + */ +public val ION = IonSystemBuilder.standard().build() diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Schema.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Schema.kt deleted file mode 100644 index 401fb636e..000000000 --- a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Schema.kt +++ /dev/null @@ -1,51 +0,0 @@ -package org.partiql.runner - -import com.amazon.ion.IonStruct -import com.amazon.ion.IonValue -import org.partiql.lang.eval.CompileOptions - -data class Namespace( - var env: IonStruct, - val namespaces: MutableList, - val testCases: MutableList, - val equivClasses: MutableMap> -) - -data class EquivalenceClass(val id: String, val statements: List) - -sealed class Assertion { - data class EvaluationSuccess(val expectedResult: IonValue) : Assertion() - object EvaluationFailure : Assertion() - // TODO: other assertion and test categories: https://github.com/partiql/partiql-tests/issues/35 -} - -sealed class TestCase { - abstract val name: String - abstract val env: IonStruct - abstract val compileOptions: CompileOptions - abstract val assertion: Assertion -} - -data class EvalTestCase( - override val name: String, - val statement: String, - override val env: IonStruct, - override val compileOptions: CompileOptions, - override val assertion: Assertion -) : TestCase() { - override fun toString(): String { - return name + ", compileOption: " + compileOptions.typingMode - } -} - -data class EvalEquivTestCase( - override val name: String, - val statements: List, - override val env: IonStruct, - override val compileOptions: CompileOptions, - override val assertion: Assertion -) : TestCase() { - override fun toString(): String { - return name + ", compileOption: " + compileOptions.typingMode - } -} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/report/ReportGenerator.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/report/ReportGenerator.kt new file mode 100644 index 000000000..b3bd79fd5 --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/report/ReportGenerator.kt @@ -0,0 +1,55 @@ +package org.partiql.runner.report + +import com.amazon.ion.IonType +import com.amazon.ion.system.IonTextWriterBuilder +import org.junit.jupiter.api.extension.AfterAllCallback +import org.junit.jupiter.api.extension.ExtensionContext +import org.junit.jupiter.api.extension.TestWatcher +import java.io.File + +class ReportGenerator : TestWatcher, AfterAllCallback { + var failingTests = emptySet() + var passingTests = emptySet() + var ignoredTests = emptySet() + override fun testFailed(context: ExtensionContext?, cause: Throwable?) { + failingTests += context?.displayName ?: "" + super.testFailed(context, cause) + } + + override fun testSuccessful(context: ExtensionContext?) { + passingTests += context?.displayName ?: "" + super.testSuccessful(context) + } + + override fun afterAll(p0: ExtensionContext?) { + val file = File("./conformance_test_results.ion") + val outputStream = file.outputStream() + val writer = IonTextWriterBuilder.pretty().build(outputStream) + writer.stepIn(IonType.STRUCT) // in: outer struct + + // set struct field for passing + writer.setFieldName("passing") + writer.stepIn(IonType.LIST) + passingTests.forEach { passingTest -> + writer.writeString(passingTest) + } + writer.stepOut() + // set struct field for failing + writer.setFieldName("failing") + writer.stepIn(IonType.LIST) + failingTests.forEach { failingTest -> + writer.writeString(failingTest) + } + writer.stepOut() + + // set struct field for ignored + writer.setFieldName("ignored") + writer.stepIn(IonType.LIST) + ignoredTests.forEach { ignoredTest -> + writer.writeString(ignoredTest) + } + writer.stepOut() + + writer.stepOut() // out: outer struct + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Assertion.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Assertion.kt new file mode 100644 index 000000000..31968d22d --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Assertion.kt @@ -0,0 +1,9 @@ +package org.partiql.runner.schema + +import com.amazon.ion.IonValue + +sealed class Assertion { + data class EvaluationSuccess(val expectedResult: IonValue) : Assertion() + object EvaluationFailure : Assertion() + // TODO: other assertion and test categories: https://github.com/partiql/partiql-tests/issues/35 +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/EquivalenceClass.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/EquivalenceClass.kt new file mode 100644 index 000000000..80e627cf7 --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/EquivalenceClass.kt @@ -0,0 +1,6 @@ +package org.partiql.runner.schema + +data class EquivalenceClass( + val id: String, + val statements: List, +) diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Namespace.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Namespace.kt new file mode 100644 index 000000000..afb7b0fe9 --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Namespace.kt @@ -0,0 +1,10 @@ +package org.partiql.runner.schema + +import com.amazon.ion.IonStruct + +data class Namespace( + var env: IonStruct, + val namespaces: MutableList, + val testCases: MutableList, + val equivClasses: MutableMap> +) diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Parse.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Parse.kt similarity index 97% rename from test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Parse.kt rename to test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Parse.kt index 43a98091b..3c70a7966 100644 --- a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Parse.kt +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Parse.kt @@ -1,4 +1,4 @@ -package org.partiql.runner +package org.partiql.runner.schema import com.amazon.ion.IonList import com.amazon.ion.IonStruct @@ -47,7 +47,7 @@ private fun parseTestCase(testStruct: IonStruct, curNamespace: Namespace): List< when (statement.type) { // statement being an IonString indicates that this is an Eval test case - IonType.STRING -> EvalTestCase( + IonType.STRING -> TestCase.Eval( name = name, statement = statement.stringValue() ?: error("Expected `statement` to be a string"), env = env.asIonStruct(), @@ -57,7 +57,7 @@ private fun parseTestCase(testStruct: IonStruct, curNamespace: Namespace): List< // statement being an IonSymbol indicates that this is an eval equivalence test case IonType.SYMBOL -> { val equivClassId = statement.stringValue() ?: error("Expected `statement` to be a symbol") - EvalEquivTestCase( + TestCase.Equiv( name = name, statements = curNamespace.equivClasses[equivClassId] ?: error("Equiv class $equivClassId not defined in current namespace"), env = env.asIonStruct(), diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/TestCase.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/TestCase.kt new file mode 100644 index 000000000..793acef92 --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/TestCase.kt @@ -0,0 +1,35 @@ +package org.partiql.runner.schema + +import com.amazon.ion.IonStruct +import org.partiql.lang.eval.CompileOptions + +sealed class TestCase { + abstract val name: String + abstract val env: IonStruct + abstract val compileOptions: CompileOptions + abstract val assertion: Assertion + + data class Equiv( + override val name: String, + val statements: List, + override val env: IonStruct, + override val compileOptions: CompileOptions, + override val assertion: Assertion + ) : TestCase() { + override fun toString(): String { + return name + ", compileOption: " + compileOptions.typingMode + } + } + + data class Eval( + override val name: String, + val statement: String, + override val env: IonStruct, + override val compileOptions: CompileOptions, + override val assertion: Assertion + ) : TestCase() { + override fun toString(): String { + return name + ", compileOption: " + compileOptions.typingMode + } + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/TestRunner.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/skip/Failing.kt similarity index 65% rename from test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/TestRunner.kt rename to test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/skip/Failing.kt index 6d822d7eb..c605b4791 100644 --- a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/TestRunner.kt +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/skip/Failing.kt @@ -1,92 +1,11 @@ -/* - * Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ +package org.partiql.runner.skip -package org.partiql.runner - -import com.amazon.ion.IonType -import com.amazon.ion.system.IonSystemBuilder -import com.amazon.ion.system.IonTextWriterBuilder -import org.junit.jupiter.api.extension.AfterAllCallback -import org.junit.jupiter.api.extension.ExtendWith -import org.junit.jupiter.api.extension.ExtensionContext -import org.junit.jupiter.api.extension.TestWatcher -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.ArgumentsSource -import org.partiql.lang.CompilerPipeline -import org.partiql.lang.SqlException import org.partiql.lang.eval.CompileOptions -import org.partiql.lang.eval.EvaluationSession -import org.partiql.lang.eval.ExprValue import org.partiql.lang.eval.TypingMode -import org.partiql.lang.eval.toIonValue -import java.io.File - -private val PARTIQL_EVAL_TEST_DATA_DIR = System.getenv("PARTIQL_EVAL_TESTS_DATA") -private val PARTIQL_EVAL_EQUIV_TEST_DATA_DIR = System.getenv("PARTIQL_EVAL_EQUIV_TESTS_DATA") - -private val ION = IonSystemBuilder.standard().build() private val COERCE_EVAL_MODE_COMPILE_OPTIONS = CompileOptions.build { typingMode(TypingMode.PERMISSIVE) } private val ERROR_EVAL_MODE_COMPILE_OPTIONS = CompileOptions.build { typingMode(TypingMode.LEGACY) } -class ConformanceTestReportGenerator : TestWatcher, AfterAllCallback { - var failingTests = emptySet() - var passingTests = emptySet() - var ignoredTests = emptySet() - override fun testFailed(context: ExtensionContext?, cause: Throwable?) { - failingTests += context?.displayName ?: "" - super.testFailed(context, cause) - } - - override fun testSuccessful(context: ExtensionContext?) { - passingTests += context?.displayName ?: "" - super.testSuccessful(context) - } - - override fun afterAll(p0: ExtensionContext?) { - val file = File("./conformance_test_results.ion") - val outputStream = file.outputStream() - val writer = IonTextWriterBuilder.pretty().build(outputStream) - writer.stepIn(IonType.STRUCT) // in: outer struct - - // set struct field for passing - writer.setFieldName("passing") - writer.stepIn(IonType.LIST) - passingTests.forEach { passingTest -> - writer.writeString(passingTest) - } - writer.stepOut() - // set struct field for failing - writer.setFieldName("failing") - writer.stepIn(IonType.LIST) - failingTests.forEach { failingTest -> - writer.writeString(failingTest) - } - writer.stepOut() - - // set struct field for ignored - writer.setFieldName("ignored") - writer.stepIn(IonType.LIST) - ignoredTests.forEach { ignoredTest -> - writer.writeString(ignoredTest) - } - writer.stepOut() - - writer.stepOut() // out: outer struct - } -} - /* The fail lists defined in this file show how the current Kotlin implementation diverges from the PartiQL spec. Most of the divergent behavior is due to `partiql-lang-kotlin` not having a STRICT typing mode/ERROR eval mode. The @@ -100,7 +19,7 @@ aggregation functions) and due to not supporting coercions. The remaining divergent behavior causing certain conformance tests to fail are likely bugs. Tracking issue: https://github.com/partiql/partiql-lang-kotlin/issues/804. */ -private val LANG_KOTLIN_EVAL_FAIL_LIST = listOf( +val LANG_KOTLIN_EVAL_FAIL_LIST = listOf( // from the spec: no explicit CAST to string means the query is "treated as an array navigation with wrongly typed // data" and will return `MISSING` Pair("tuple navigation with array notation without explicit CAST to string", COERCE_EVAL_MODE_COMPILE_OPTIONS), @@ -296,7 +215,7 @@ private val LANG_KOTLIN_EVAL_FAIL_LIST = listOf( Pair("""OCTET_LENGTH special character""", ERROR_EVAL_MODE_COMPILE_OPTIONS), ) -private val LANG_KOTLIN_EVAL_EQUIV_FAIL_LIST = listOf( +val LANG_KOTLIN_EVAL_EQUIV_FAIL_LIST = listOf( // partiql-lang-kotlin gives a parser error for tuple path navigation in which the path expression is a string // literal // e.g. { 'a': 1, 'b': 2}.'a' -> 1 (see section 4 of spec) @@ -338,188 +257,3 @@ private val LANG_KOTLIN_EVAL_EQUIV_FAIL_LIST = listOf( Pair("equiv aliases from select clause", COERCE_EVAL_MODE_COMPILE_OPTIONS), Pair("equiv aliases from select clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), ) - -/** - * Checks all the PartiQL conformance test data in [PARTIQL_EVAL_TEST_DATA_DIR] conforms to the test data schema. - */ -class TestRunner { - private fun parseTestFile(file: File): Namespace { - val loadedData = file.readText() - val dataInIon = ION.loader.load(loadedData) - val emptyNamespace = Namespace( - env = ION.newEmptyStruct(), - namespaces = mutableListOf(), - testCases = mutableListOf(), - equivClasses = mutableMapOf() - ) - dataInIon.forEach { d -> - parseNamespace(emptyNamespace, d) - } - return emptyNamespace - } - - private fun allTestsFromNamespace(ns: Namespace): List { - return ns.testCases + ns.namespaces.fold(listOf()) { acc, subns -> - acc + allTestsFromNamespace(subns) - } - } - - private fun loadTests(path: String): List { - val allFiles = File(path).walk() - .filter { it.isFile } - .filter { it.path.endsWith(".ion") } - .toList() - val filesAsNamespaces = allFiles.map { file -> - parseTestFile(file) - } - - val allTestCases = filesAsNamespaces.flatMap { ns -> - allTestsFromNamespace(ns) - } - return allTestCases - } - - private fun runEvalTestCase(evalTC: EvalTestCase, expectedFailedTests: List>) { - val compilerPipeline = CompilerPipeline.builder().compileOptions(evalTC.compileOptions).build() - val globals = ExprValue.of(evalTC.env).bindings - val session = EvaluationSession.build { globals(globals) } - try { - val expression = compilerPipeline.compile(evalTC.statement) - val actualResult = expression.eval(session) - when (evalTC.assertion) { - is Assertion.EvaluationSuccess -> { - val actualResultAsIon = actualResult.toIonValue(ION) - if (!expectedFailedTests.contains(Pair(evalTC.name, evalTC.compileOptions)) && !PartiQLEqualityChecker().areEqual(evalTC.assertion.expectedResult, actualResultAsIon)) { - error("Expected: ${evalTC.assertion.expectedResult}\nActual: $actualResultAsIon\nMode: ${evalTC.compileOptions.typingMode}") - } - } - is Assertion.EvaluationFailure -> { - if (!expectedFailedTests.contains(Pair(evalTC.name, evalTC.compileOptions))) { - error("Expected error to be thrown but none was thrown.\n${evalTC.name}\nActual result: ${actualResult.toIonValue(ION)}") - } - } - } - } catch (e: SqlException) { - when (evalTC.assertion) { - is Assertion.EvaluationSuccess -> { - if (!expectedFailedTests.contains(Pair(evalTC.name, evalTC.compileOptions))) { - error("Expected success but exception thrown: $e") - } - } - is Assertion.EvaluationFailure -> { - // Expected failure and test threw when evaluated - } - } - } - } - - private fun runEvalEquivTestCase(evalEquivTestCase: EvalEquivTestCase, expectedFailedTests: List>) { - val compilerPipeline = CompilerPipeline.builder().compileOptions(evalEquivTestCase.compileOptions).build() - val globals = ExprValue.of(evalEquivTestCase.env).bindings - val session = EvaluationSession.build { globals(globals) } - val statements = evalEquivTestCase.statements - - statements.forEach { statement -> - try { - val expression = compilerPipeline.compile(statement) - val actualResult = expression.eval(session) - when (evalEquivTestCase.assertion) { - is Assertion.EvaluationSuccess -> { - val actualResultAsIon = actualResult.toIonValue(ION) - if (!expectedFailedTests.contains(Pair(evalEquivTestCase.name, evalEquivTestCase.compileOptions)) && !PartiQLEqualityChecker().areEqual(evalEquivTestCase.assertion.expectedResult, actualResultAsIon)) { - error("Expected and actual results differ:\nExpected: ${evalEquivTestCase.assertion.expectedResult}\nActual: $actualResultAsIon\nMode: ${evalEquivTestCase.compileOptions.typingMode}") - } - } - is Assertion.EvaluationFailure -> { - if (!expectedFailedTests.contains(Pair(evalEquivTestCase.name, evalEquivTestCase.compileOptions))) { - error("Expected error to be thrown but none was thrown.\n${evalEquivTestCase.name}\nActual result: ${actualResult.toIonValue(ION)}") - } - } - } - } catch (e: SqlException) { - when (evalEquivTestCase.assertion) { - is Assertion.EvaluationSuccess -> { - if (!expectedFailedTests.contains(Pair(evalEquivTestCase.name, evalEquivTestCase.compileOptions))) { - error("Expected success but exception thrown: $e") - } - } - is Assertion.EvaluationFailure -> { - // Expected failure and test threw when evaluated - } - } - } - } - } - - /** - * Runs the conformance tests with an expected list of failing tests. Ensures that tests not in the failing list - * succeed with the expected result. Ensures that tests included in the failing list fail. - * - * These tests are included in the normal test/building. - * Update May 2023: Now excluded from the normal build, because the fail lists are out of date. - * TODO: Come up with a low-burden method of maintaining fail / exclusion lists. - */ - class DefaultConformanceTestRunner { - // Tests the eval tests with the Kotlin implementation - @ParameterizedTest(name = "{arguments}") - @ArgumentsSource(EvalTestCases::class) - fun validatePartiQLEvalTestData(tc: TestCase) { - when (tc) { - is EvalTestCase -> TestRunner().runEvalTestCase(tc, LANG_KOTLIN_EVAL_FAIL_LIST) - else -> error("Unsupported test case category") - } - } - - // Tests the eval equivalence tests with the Kotlin implementation - @ParameterizedTest(name = "{arguments}") - @ArgumentsSource(EvalEquivTestCases::class) - fun validatePartiQLEvalEquivTestData(tc: TestCase) { - when (tc) { - is EvalEquivTestCase -> TestRunner().runEvalEquivTestCase(tc, LANG_KOTLIN_EVAL_EQUIV_FAIL_LIST) - else -> error("Unsupported test case category") - } - } - } - - /** - * Runs the conformance tests without a fail list, so we can document the passing/failing tests in the conformance - * report. - * - * These tests are excluded from normal testing/building unless the `conformanceReport` gradle property is - * specified (i.e. `gradle test ... -PconformanceReport`) - */ - @ExtendWith(ConformanceTestReportGenerator::class) - class ConformanceTestsReportRunner { - // Tests the eval tests with the Kotlin implementation without a fail list - @ParameterizedTest(name = "{arguments}") - @ArgumentsSource(EvalTestCases::class) - fun validatePartiQLEvalTestData(tc: TestCase) { - when (tc) { - is EvalTestCase -> TestRunner().runEvalTestCase(tc, emptyList()) - else -> error("Unsupported test case category") - } - } - - // Tests the eval equivalence tests with the Kotlin implementation without a fail list - @ParameterizedTest(name = "{arguments}") - @ArgumentsSource(EvalEquivTestCases::class) - fun validatePartiQLEvalEquivTestData(tc: TestCase) { - when (tc) { - is EvalEquivTestCase -> TestRunner().runEvalEquivTestCase(tc, emptyList()) - else -> error("Unsupported test case category") - } - } - } - - class EvalTestCases : ArgumentsProviderBase() { - override fun getParameters(): List { - return TestRunner().loadTests(PARTIQL_EVAL_TEST_DATA_DIR) - } - } - - class EvalEquivTestCases : ArgumentsProviderBase() { - override fun getParameters(): List { - return TestRunner().loadTests(PARTIQL_EVAL_EQUIV_TEST_DATA_DIR) - } - } -} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestExecutor.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestExecutor.kt new file mode 100644 index 000000000..566995e57 --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestExecutor.kt @@ -0,0 +1,59 @@ +package org.partiql.runner.test + +import com.amazon.ion.IonStruct +import com.amazon.ion.IonValue +import org.partiql.lang.eval.CompileOptions + +interface TestExecutor { + + /** + * Compile the given statement. + * + * @param statement + * @return + */ + fun prepare(statement: String): T + + /** + * Execute the statement, returning a value we can assert on. + * + * @param statement + * @return + */ + fun execute(statement: T): V + + /** + * Compare the equality of two values. + * + * @param actual + * @param expect + */ + fun compare(actual: V, expect: V): Boolean + + /** + * Read an IonValue to the value type [V] used by this executor. + * + * @param value + * @return + */ + fun fromIon(value: IonValue): V + + /** + * Write a value [V] to an IonValue for debug printing. + * + * @param value + * @return + */ + fun toIon(value: V): IonValue + + /** + * CompileOptions varies for each test, need a way to programmatically create an executor. + * + * @param T + * @param V + */ + interface Factory { + + fun create(env: IonStruct, options: CompileOptions): TestExecutor + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestLoader.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestLoader.kt new file mode 100644 index 000000000..abdb5b2c3 --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestLoader.kt @@ -0,0 +1,49 @@ +package org.partiql.runner.test + +import org.partiql.runner.ION +import org.partiql.runner.schema.Namespace +import org.partiql.runner.schema.TestCase +import org.partiql.runner.schema.parseNamespace +import java.io.File + +/** + * Checks all the PartiQL conformance test data in [PARTIQL_EVAL_TEST_DATA_DIR] conforms to the test data schema. + */ +object TestLoader { + + fun load(path: String): List { + val allFiles = File(path).walk() + .filter { it.isFile } + .filter { it.path.endsWith(".ion") } + .toList() + val filesAsNamespaces = allFiles.map { file -> + parseTestFile(file) + } + + val allTestCases = filesAsNamespaces.flatMap { ns -> + allTestsFromNamespace(ns) + } + return allTestCases + } + + private fun parseTestFile(file: File): Namespace { + val loadedData = file.readText() + val dataInIon = ION.loader.load(loadedData) + val emptyNamespace = Namespace( + env = ION.newEmptyStruct(), + namespaces = mutableListOf(), + testCases = mutableListOf(), + equivClasses = mutableMapOf() + ) + dataInIon.forEach { d -> + parseNamespace(emptyNamespace, d) + } + return emptyNamespace + } + + private fun allTestsFromNamespace(ns: Namespace): List { + return ns.testCases + ns.namespaces.fold(listOf()) { acc, subns -> + acc + allTestsFromNamespace(subns) + } + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ArgumentsProviderBase.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestProvider.kt similarity index 52% rename from test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ArgumentsProviderBase.kt rename to test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestProvider.kt index 771c7bc25..46a6a31ea 100644 --- a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ArgumentsProviderBase.kt +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestProvider.kt @@ -1,24 +1,13 @@ -/* - * Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package org.partiql.runner +package org.partiql.runner.test import org.junit.jupiter.api.extension.ExtensionContext import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.ArgumentsProvider import java.util.stream.Stream +private val PARTIQL_EVAL_TEST_DATA_DIR = System.getenv("PARTIQL_EVAL_TESTS_DATA") +private val PARTIQL_EVAL_EQUIV_TEST_DATA_DIR = System.getenv("PARTIQL_EVAL_EQUIV_TESTS_DATA") + /** * Reduces some of the boilerplate associated with the style of parameterized testing frequently * utilized in this package. @@ -29,12 +18,21 @@ import java.util.stream.Stream * * Classes that derive from this class can be defined near the `@ParameterizedTest` functions instead. */ -abstract class ArgumentsProviderBase : ArgumentsProvider { - - abstract fun getParameters(): List +sealed class TestProvider(private val root: String) : ArgumentsProvider { @Throws(Exception::class) override fun provideArguments(extensionContext: ExtensionContext): Stream? { - return getParameters().map { Arguments.of(it) }.stream() + return TestLoader.load(root).map { Arguments.of(it) }.stream() } + + /** + * Evaluation tests + * + */ + class Eval : TestProvider(PARTIQL_EVAL_TEST_DATA_DIR) + + /** + * Equivalence tests + */ + class Equiv : TestProvider(PARTIQL_EVAL_EQUIV_TEST_DATA_DIR) } diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestRunner.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestRunner.kt new file mode 100644 index 000000000..014e15e1e --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestRunner.kt @@ -0,0 +1,54 @@ +package org.partiql.runner.test + +import org.partiql.lang.eval.CompileOptions +import org.partiql.runner.schema.Assertion +import org.partiql.runner.schema.TestCase + +/** + * TestRunner delegates execution to the underlying TestExecutor, but orchestrates test assertions. + */ +class TestRunner(private val factory: TestExecutor.Factory) { + + fun test(case: TestCase.Eval, skipList: List>) { + if (skipList.contains((Pair(case.name, case.compileOptions)))) { + return + } + val executor = factory.create(case.env, case.compileOptions) + val input = case.statement + run(input, case, executor) + } + + fun test(case: TestCase.Equiv, skipList: List>) { + if (skipList.contains((Pair(case.name, case.compileOptions)))) { + return + } + val executor = factory.create(case.env, case.compileOptions) + case.statements.forEach { run(it, case, executor) } + } + + private fun run(input: String, case: TestCase, executor: TestExecutor) { + val assertion = case.assertion + try { + val statement = executor.prepare(input) + val actual = executor.execute(statement) + when (assertion) { + is Assertion.EvaluationSuccess -> { + val expect = executor.fromIon(assertion.expectedResult) + if (!executor.compare(actual, expect)) { + val ion = executor.toIon(actual) + error("Expected: ${assertion.expectedResult}\nActual: $ion\nMode: ${case.compileOptions.typingMode}") + } + } + is Assertion.EvaluationFailure -> { + val ion = executor.toIon(actual) + error("Expected error to be thrown but none was thrown.\n${case.name}\nActual result: $ion") + } + } + } catch (e: Exception) { + when (case.assertion) { + is Assertion.EvaluationSuccess -> error("Expected success but exception thrown: $e") + is Assertion.EvaluationFailure -> {} // skip + } + } + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/executor/LegacyExecutor.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/executor/LegacyExecutor.kt new file mode 100644 index 000000000..cf966186c --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/executor/LegacyExecutor.kt @@ -0,0 +1,46 @@ +package org.partiql.runner.test.executor + +import com.amazon.ion.IonStruct +import com.amazon.ion.IonValue +import org.partiql.lang.CompilerPipeline +import org.partiql.lang.eval.CompileOptions +import org.partiql.lang.eval.EvaluationSession +import org.partiql.lang.eval.ExprValue +import org.partiql.lang.eval.toIonValue +import org.partiql.runner.ION +import org.partiql.runner.test.TestExecutor +import org.partiql.runner.util.ValueEquals + +/** + * [TestExecutor] which uses the original EvaluatingCompiler APIs. + * + * @property pipeline + * @property session + */ +class LegacyExecutor( + private val pipeline: CompilerPipeline, + private val session: EvaluationSession, +) : TestExecutor { + + private val eq = ValueEquals.legacy + + override fun prepare(statement: String): ExprValue = pipeline.compile(statement).eval(session) + + override fun execute(statement: ExprValue): IonValue = statement.toIonValue(ION) + + override fun fromIon(value: IonValue): IonValue = value + + override fun toIon(value: IonValue): IonValue = value + + override fun compare(actual: IonValue, expect: IonValue): Boolean = eq.equals(actual, expect) + + object Factory : TestExecutor.Factory { + + override fun create(env: IonStruct, options: CompileOptions): TestExecutor { + val pipeline = CompilerPipeline.builder().compileOptions(options).build() + val globals = ExprValue.of(env).bindings + val session = EvaluationSession.build { globals(globals) } + return LegacyExecutor(pipeline, session) + } + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/PartiQLEqualityChecker.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/util/ValueEquals.kt similarity index 75% rename from test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/PartiQLEqualityChecker.kt rename to test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/util/ValueEquals.kt index eb1f4938b..35a07796d 100644 --- a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/PartiQLEqualityChecker.kt +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/util/ValueEquals.kt @@ -1,4 +1,4 @@ -package org.partiql.runner +package org.partiql.runner.util import com.amazon.ion.IonDecimal import com.amazon.ion.IonList @@ -10,6 +10,38 @@ import com.amazon.ion.IonType import com.amazon.ion.IonValue import org.partiql.lang.eval.BAG_ANNOTATION import org.partiql.lang.eval.MISSING_ANNOTATION +import org.partiql.value.PartiQLValue +import org.partiql.value.PartiQLValueExperimental + +/** + * Different methods for asserting value equality. The legacy comparator needed to compared on lowered IonValue but + * we now have PartiQLValue which defines its own `equals` methods. + * + * @param T + */ +interface ValueEquals { + + fun equals(left: T, right: T): Boolean + + companion object { + + @JvmStatic + val legacy: ValueEquals = LegacyValueEquals + + @OptIn(PartiQLValueExperimental::class) + @JvmStatic + val partiql: ValueEquals = PartiQLValueEquals + } +} + +/** + * Value equality using the [PartiQLValue] equality implementation. + */ +@OptIn(PartiQLValueExperimental::class) +private object PartiQLValueEquals : ValueEquals { + + override fun equals(left: PartiQLValue, right: PartiQLValue) = left.equals(right) +} /** * Checks the equality of two PartiQL values defined using its [IonValue] representation. This definition first requires @@ -18,8 +50,9 @@ import org.partiql.lang.eval.MISSING_ANNOTATION * 1. Bag comparison checks ignore ordering of IonLists * 2. Null checks check for `missing` annotation */ -class PartiQLEqualityChecker { - fun areEqual(left: IonValue, right: IonValue): Boolean { +private object LegacyValueEquals : ValueEquals { + + override fun equals(left: IonValue, right: IonValue): Boolean { if (left.type != right.type) { return false } @@ -82,7 +115,7 @@ class PartiQLEqualityChecker { left.size == right.size && left.asSequence() .mapIndexed { index, leftElement -> index to leftElement } - .all { (index, leftElement) -> areEqual(leftElement, right[index]) } + .all { (index, leftElement) -> equals(leftElement, right[index]) } // bags can contain repeated elements, so they are equal if and only if: // * Same size @@ -92,8 +125,8 @@ class PartiQLEqualityChecker { left.size != right.size -> false left.isBag() && right.isBag() -> { left.all { leftEl -> - val leftQtd = left.count { areEqual(leftEl, it) } - val rightQtd = right.count { areEqual(leftEl, it) } + val leftQtd = left.count { equals(leftEl, it) } + val rightQtd = right.count { equals(leftEl, it) } leftQtd == rightQtd }