diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f10176795..c8fc29c1c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,6 @@ name: "Build and Report Generation" on: push: - branches : [main] paths: - '**' - '!docs/**' diff --git a/.github/workflows/conformance-report.yml b/.github/workflows/conformance-report.yml index d8de58c38..e4125f5c7 100644 --- a/.github/workflows/conformance-report.yml +++ b/.github/workflows/conformance-report.yml @@ -2,9 +2,13 @@ name: Conformance Test Report Generation on: [push, pull_request] env: - PATH_TO_TEST_RUNNER: test/partiql-tests-runner + LEGACY_VERSION: v0.14.8 CONFORMANCE_REPORT_NAME: conformance_test_results.ion + PATH_TO_TEST_RUNNER: test/partiql-tests-runner + CONFORMANCE_REPORT_RELATIVE_PATH: build/conformance-test-report COMPARISON_REPORT_NAME: comparison_report.md + COMPARISON_REPORT_NAME_WITH_LIMIT: comparison_report_limited.md + COMMENT_SIZE_LIMIT: 10 jobs: conformance-report: @@ -13,6 +17,7 @@ jobs: steps: - uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} submodules: recursive - name: Use Java 17 uses: actions/setup-java@v3 @@ -27,12 +32,12 @@ jobs: # Run the conformance tests and save to an Ion file. - name: gradle test of the conformance tests (can fail) and save to Ion file continue-on-error: true - run: gradle :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport + run: gradle :test:partiql-tests-runner:generateTestReport # Upload conformance report for future viewing and comparison with future runs. - - name: Upload `conformance_test_results.ion` + - name: Upload `conformance-test-report` folder uses: actions/upload-artifact@v3 with: - path: ${{ env.PATH_TO_TEST_RUNNER }}/${{ env.CONFORMANCE_REPORT_NAME }} + path: ${{ env.PATH_TO_TEST_RUNNER }}/${{ env.CONFORMANCE_REPORT_RELATIVE_PATH }} # Cache the conformance report for `conformance-report-comparison` job (pull_request event only) - name: Cache conformance report and build if: github.event_name == 'pull_request' @@ -86,23 +91,76 @@ jobs: continue-on-error: true run: | cd ${{ github.event.pull_request.base.sha }} - gradle :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport + gradle :test:partiql-tests-runner:generateTestReport - name: (If download of target branch conformance report fails) Move conformance test report of target branch to ./artifact directory if: ${{ steps.download-report.outcome == 'failure' }} continue-on-error: true run: | mkdir -p $GITHUB_WORKSPACE/artifact - cp -r $GITHUB_WORKSPACE/${{ github.event.pull_request.base.sha }}/$PATH_TO_TEST_RUNNER/$CONFORMANCE_REPORT_NAME $GITHUB_WORKSPACE/artifact/$CONFORMANCE_REPORT_NAME - # Run conformance report comparison. Generates `comparison_report.md` - - name: Run conformance report comparison. Generates `comparison_report.md` + cp -r $GITHUB_WORKSPACE/${{ github.event.pull_request.base.sha }}/$PATH_TO_TEST_RUNNER/$CONFORMANCE_REPORT_RELATIVE_PATH $GITHUB_WORKSPACE/artifact/$CONFORMANCE_REPORT_RELATIVE_PATH + - name: "Artifact CROSS-ENGINE Report (creates comparison_report.md)" continue-on-error: true run: | - ARGS="$GITHUB_WORKSPACE/artifact/$CONFORMANCE_REPORT_NAME $CONFORMANCE_REPORT_NAME ${{ github.event.pull_request.base.sha }} $GITHUB_SHA $COMPARISON_REPORT_NAME" + T="CROSS-ENGINE-REPORT" + BF="$GITHUB_WORKSPACE/$PATH_TO_TEST_RUNNER/src/main/resources/config/legacy/$CONFORMANCE_REPORT_NAME" + BL="LEGACY" + BT="$LEGACY_VERSION" + TF="$GITHUB_WORKSPACE/$PATH_TO_TEST_RUNNER/$CONFORMANCE_REPORT_RELATIVE_PATH/eval/$CONFORMANCE_REPORT_NAME" + TL="EVAL" + TT="$GITHUB_SHA" + O="$COMPARISON_REPORT_NAME" + ARGS="-t $T -bf $BF -bl $BL -bt $BT -tf $TF -tl $TL -tt $TT -o $O" + gradle :test:partiql-tests-runner:run --args="$ARGS" + - name: "Artifact CROSS-COMMIT Report (appends to comparison_report.md)" + continue-on-error: true + run: | + T="CROSS-COMMIT-REPORT" + BF="$GITHUB_WORKSPACE/artifact/eval/$CONFORMANCE_REPORT_NAME" + BL="EVAL" + BT="${{ github.event.pull_request.base.sha }}" + TF="$GITHUB_WORKSPACE/$PATH_TO_TEST_RUNNER/$CONFORMANCE_REPORT_RELATIVE_PATH/eval/$CONFORMANCE_REPORT_NAME" + TL="EVAL" + TT="$GITHUB_SHA" + O="$COMPARISON_REPORT_NAME" + ARGS="-t $T -bf $BF -bl $BL -bt $BT -tf $TF -tl $TL -tt $TT -o $O" gradle :test:partiql-tests-runner:run --args="$ARGS" # Print conformance report to GitHub actions workflow summary page - name: Print markdown in run continue-on-error: true run: cat $PATH_TO_TEST_RUNNER/$COMPARISON_REPORT_NAME >> $GITHUB_STEP_SUMMARY + # Upload the full comparison report to CI artifact + - name: Upload `comparison_report.md` + uses: actions/upload-artifact@v3 + with: + path: ${{ env.PATH_TO_TEST_RUNNER }}/comparison_report.md + - name: "Comment CROSS-ENGINE Report (creates comparison_report_limited.md)" + continue-on-error: true + run: | + T="CROSS-ENGINE-REPORT" + BF="$GITHUB_WORKSPACE/$PATH_TO_TEST_RUNNER/src/main/resources/config/legacy/$CONFORMANCE_REPORT_NAME" + BL="LEGACY" + BT="$LEGACY_VERSION" + TF="$GITHUB_WORKSPACE/$PATH_TO_TEST_RUNNER/$CONFORMANCE_REPORT_RELATIVE_PATH/eval/$CONFORMANCE_REPORT_NAME" + TL="EVAL" + TT="$GITHUB_SHA" + O="$COMPARISON_REPORT_NAME_WITH_LIMIT" + L="$COMMENT_SIZE_LIMIT" + ARGS="-t $T -bf $BF -bl $BL -bt $BT -tf $TF -tl $TL -tt $TT -o $O -l $L" + gradle :test:partiql-tests-runner:run --args="$ARGS" + - name: "Comment CROSS-COMMIT Report (appends to comparison_report_limited.md)" + continue-on-error: true + run: | + T="CROSS-COMMIT-REPORT" + BF="$GITHUB_WORKSPACE/artifact/eval/$CONFORMANCE_REPORT_NAME" + BL="EVAL" + BT="${{ github.event.pull_request.base.sha }}" + TF="$GITHUB_WORKSPACE/$PATH_TO_TEST_RUNNER/$CONFORMANCE_REPORT_RELATIVE_PATH/eval/$CONFORMANCE_REPORT_NAME" + TL="EVAL" + TT="$GITHUB_SHA" + O="$COMPARISON_REPORT_NAME_WITH_LIMIT" + L="$COMMENT_SIZE_LIMIT" + ARGS="-t $T -bf $BF -bl $BL -bt $BT -tf $TF -tl $TL -tt $TT -o $O -l $L" + gradle :test:partiql-tests-runner:run --args="$ARGS" # Find comment w/ conformance comparison if previous comment published - name: Find Comment uses: peter-evans/find-comment@v2 @@ -111,7 +169,7 @@ jobs: with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' - body-includes: Conformance + body-includes: CROSS-ENGINE-REPORT # Create or update (if previous comment exists) with markdown version of comparison report - name: Create or update comment continue-on-error: true @@ -119,5 +177,5 @@ jobs: with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} - body-file: ${{ env.PATH_TO_TEST_RUNNER }}/${{ env.COMPARISON_REPORT_NAME }} + body-file: ${{ env.PATH_TO_TEST_RUNNER }}/${{ env.COMPARISON_REPORT_NAME_WITH_LIMIT }} edit-mode: replace diff --git a/.gitignore b/.gitignore index e68379eed..288d59c80 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ build ~$* .gradle out/ - +partiql-parser/src/main/gen # Created by https://www.toptal.com/developers/gitignore/api/vim,git,java,emacs,kotlin,eclipse,intellij+all,macos # Edit at https://www.toptal.com/developers/gitignore?templates=vim,git,java,emacs,kotlin,eclipse,intellij+all,macos diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b7fadbdb..cb7072584 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,41 +23,86 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Thank you to all who have contributed! --> -## [Unreleased] +## [1.0.0-rc.2] + +### Added + +### Changed + +### Deprecated + +### Fixed + +### Removed + +### Security + +### Contributors +Thank you to all who have contributed! + +## [1.0.0-rc.1] + +### Added + +### Changed + +### Deprecated + +### Fixed + +### Removed + +### Security + +### Contributors +Thank you to all who have contributed! + +## [0.14.8] + +### Added + +### Changed + +### Deprecated + +### Fixed +- Case When Branch inference will preserve type constraint for String Type and Decimal Type, if no coercion is required. +### Removed + +### Security + +### Contributors +Thank you to all who have contributed! + +## [0.14.7] + +### Fixed +- `partiql-lang`'s `PartiQLParserBuilder.standard()` will use the ANTLR dependency from `partiql-parser` to +prevent `NoSuchMethodError`s + +## [0.14.6] ### Added - Adds `PartiQLValueTextWriter` implementation of date, time, and timestamp values +- Shades ANTLR dependency to avoid dependency conflicts. ### Changed -- **Behavioral change**: The planner now does NOT support the NullType and MissingType variants of StaticType. The logic -is that the null and missing values are part of *all* data types. Therefore, one must assume that the types returned by -the planner allow for NULL and MISSING values. Similarly, the testFixtures Ion-encoded test resources -representing the catalog do not use "null" or "missing". - **Behavioral change**: The `INTEGER/INT` type is now an alias to the `INT4` type. Previously the INTEGER type was unconstrained which is not SQL-conformant and is causing issues in integrating with other systems. This release makes INTEGER an alias for INT4 which is the internal type name. In a later release, we will make INTEGER the default 32-bit integer with INT/INT4/INTEGER4 being aliases per other systems. This change only applies to org.partiql.parser.PartiQLParser, not the org.partiql.lang.syntax.PartiQLParser. -- The deprecated SqlLayout and SqlDialect (which had stack overflow issues) has been replaced by the optimized version. -The API is slightly different (append vs prepend) and generic method names have been replaced with more descriptive -names (transform and print). +- **Breaking change**: partiql-plan: adds a set quantifier field to SQL set operators `UNION`, `INTERSECT`, and `EXCEPT` +- partiql-plan: adds a dedicated Rex node for PartiQL bag operators `UNION`, `INTERSECT`, and `EXCEPT` +- partiql-planner: Adds typing support for set operators +- partiql-parser: parses non-SFW expressions to be PartiQL `OUTER` bag operators +- partiql-ast: fixes missing parens from `bag_op` when printing using `SqlDialect` ### Deprecated -- We have deprecated `org.partiql.type.NullType` and `org.partiql.type.MissingType`. Please see the corresponding -information in the "Changed" section. In relation to the deprecation of the above, the following APIs have also -been deprecated: - - `org.partiql.type.StaticType.MISSING` - - `org.partiql.type.StaticType.NULL` - - `org.partiql.type.StaticType.NULL_OR_MISSING` - - `org.partiql.type.StaticType.asNullable()` - - `org.partiql.type.StaticType.isNullable()` - - `org.partiql.type.StaticType.isMissable()` - - `org.partiql.type.StaticType.asOptional()` - - `org.partiql.type.AnyOfType()` - - `org.partiql.value.PartiQLValueType.NULL` - - `org.partiql.value.PartiQLValueType.MISSING` ### Fixed +- Fixed classpath conflict for IsStaticTypeMeta +- Fixes ANTLR parser grammar file naming. ### Removed @@ -65,6 +110,11 @@ been deprecated: ### Contributors Thank you to all who have contributed! +- @ + +- @rchowell +- @alancai98 +- @johnedquinn ## [0.14.5] @@ -98,11 +148,19 @@ Thank you to all who have contributed! - @alancai98 - @johnedquinn +## [1.0.0-perf.1] - 2024-03-04 + +This is a pre-release containing: +- A new, experimental evaluator under `org.partiql.eval`. +- Several breaking changes under `org.partiql.plan` and `org.partiql.types` and `org.partiql.spi`. + +Please note that these changes are subject to future breaking changes without warning. + ## [0.14.4] ### Added -- Added constrained decimal as valid parameter type to functions that take in numeric parameters. -- Added async version of physical plan evaluator `PartiQLCompilerAsync`. +- Added constrained decimal as valid parameter type to functions that take in numeric parameters. +- Added async version of physical plan evaluator `PartiQLCompilerAsync`. - The following related async APIs have been added: - `org.partiql.lang.compiler` -- `PartiQLCompilerAsync`, `PartiQLCompilerAsyncBuilder`, `PartiQLCompilerAsyncDefault`, `PartiQLCompilerPipelineAsync` - `org.partiql.lang.eval` -- `PartiQLStatementAsync` @@ -113,7 +171,7 @@ Thank you to all who have contributed! - JMH benchmarks added to partiql-lang: `PartiQLCompilerPipelineBenchmark` and `PartiQLCompilerPipelineAsyncBenchmark` ### Changed -- Function resolution logic: Now the function resolver would match all possible candidate(based on if the argument can be coerced to the Signature parameter type). If there are multiple match it will first attempt to pick the one requires the least cast, then pick the function with the highest precedence. +- Function resolution logic: Now the function resolver would match all possible candidate(based on if the argument can be coerced to the Signature parameter type). If there are multiple match it will first attempt to pick the one requires the least cast, then pick the function with the highest precedence. - partiql-cli -- experimental version of CLI now uses the async physical plan evaluator ### Deprecated @@ -146,6 +204,8 @@ Thank you to all who have contributed! ### Contributors Thank you to all who have contributed! - @alancai98 +- @johnedquinn +- @RCHowell - @yliuuuu ## [0.14.2] - 2024-01-25 @@ -197,8 +257,8 @@ Thank you to all who have contributed! - Adds top-level IR node creation functions. - Adds `componentN` functions (destructuring) to IR nodes via Kotlin data classes - Adds public `tag` field to IR nodes for associating metadata -- Adds AST Normalization Pass. -- Adds PartiQLPlanner Interface, which is responsible for translate an AST to a Plan. +- Adds AST Normalization Pass. +- Adds PartiQLPlanner Interface, which is responsible for translate an AST to a Plan. - **EXPERIMENTAL** Evaluation of `EXCLUDE` in the `EvaluatingCompiler` - This is currently marked as experimental until the RFC is approved https://github.com/partiql/partiql-lang/issues/27 - This will be added to the `PhysicalPlanCompiler` in an upcoming release @@ -206,13 +266,13 @@ Thank you to all who have contributed! ### Changed - StaticTypeInferencer and PlanTyper will not raise an error when an expression is inferred to `NULL` or `unionOf(NULL, MISSING)`. In these cases the StaticTypeInferencer and PlanTyper will still raise the Problem Code `ExpressionAlwaysReturnsNullOrMissing` but the severity of the problem has been changed to warning. In the case an expression always returns `MISSING`, problem code `ExpressionAlwaysReturnsMissing` will be raised, which will have problem severity of error. -- **Breaking** The default integer literal type is now 32-bit; if the literal can not fit in a 32-bit integer, it overflows to 64-bit. -- **BREAKING** `PartiQLValueType` now distinguishes between Arbitrary Precision Decimal and Fixed Precision Decimal. -- **BREAKING** Function Signature Changes. Now Function signature has two subclasses, `Scalar` and `Aggregation`. +- **Breaking** The default integer literal type is now 32-bit; if the literal can not fit in a 32-bit integer, it overflows to 64-bit. +- **BREAKING** `PartiQLValueType` now distinguishes between Arbitrary Precision Decimal and Fixed Precision Decimal. +- **BREAKING** Function Signature Changes. Now Function signature has two subclasses, `Scalar` and `Aggregation`. - **BREAKING** Plugin Changes. Only return one Connector.Factory, use Kotlin fields. JVM signature remains the same. -- **BREAKING** In the produced plan: +- **BREAKING** In the produced plan: - The new plan is fully resolved and typed. - - Operators will be converted to function call. + - Operators will be converted to function call. - Changes the return type of `filter_distinct` to a list if input collection is list - Changes the `PartiQLValue` collections to implement Iterable rather than Sequence, allowing for multiple consumption. - **BREAKING** Moves PartiQLParserBuilder.standard().build() to be PartiQLParser.default(). @@ -230,7 +290,7 @@ Thank you to all who have contributed! ### Removed - **Breaking** Removed IR factory in favor of static top-level functions. Change `Ast.foo()` to `foo()` -- **Breaking** Removed `org.partiql.lang.planner.transforms.AstToPlan`. Use `org.partiql.planner.PartiQLPlanner`. +- **Breaking** Removed `org.partiql.lang.planner.transforms.AstToPlan`. Use `org.partiql.planner.PartiQLPlanner`. - **Breaking** Removed `org.partiql.lang.planner.transforms.PartiQLSchemaInferencer`. In order to achieve the same functionality, one would need to use the `org.partiql.planner.PartiQLPlanner`. - To get the inferred type of the query result, one can do: `(plan.statement as Statement.Query).root.type` @@ -309,7 +369,7 @@ Thank you to all who have contributed! - Parsing of label patterns within node and edge graph patterns now supports disjunction `|`, conjunction `&`, negation `!`, and grouping. - Adds default `equals` and `hashCode` methods for each generated abstract class of Sprout. This affects the generated -classes in `:partiql-ast` and `:partiql-plan`. + classes in `:partiql-ast` and `:partiql-plan`. - Adds README to `partiql-types` package. - Initializes PartiQL's Code Coverage library - Adds support for BRANCH and BRANCH-CONDITION Coverage @@ -351,12 +411,12 @@ classes in `:partiql-ast` and `:partiql-plan`. - Introduces `isNullCall` and `isNullable` properties to FunctionSignature. - Removed `Nullable...Value` implementations of PartiQLValue and made the standard implementations nullable. - Using PartiQLValueType requires optin; this was a miss from an earlier commit. -- Modified timestamp static type to model precision and time zone. +- Modified timestamp static type to model precision and time zone. ### Deprecated -- **Breaking**: Deprecates the `Arguments`, `RequiredArgs`, `RequiredWithOptional`, and `RequiredWithVariadic` classes, - along with the `callWithOptional()`, `callWithVariadic()`, and the overloaded `call()` methods in the `ExprFunction` class, - marking them with a Deprecation Level of ERROR. Now, it's recommended to use +- **Breaking**: Deprecates the `Arguments`, `RequiredArgs`, `RequiredWithOptional`, and `RequiredWithVariadic` classes, + along with the `callWithOptional()`, `callWithVariadic()`, and the overloaded `call()` methods in the `ExprFunction` class, + marking them with a Deprecation Level of ERROR. Now, it's recommended to use `call(session: EvaluationSession, args: List)` and `callWithRequired()` instead. - **Breaking**: Deprecates `optionalParameter` and `variadicParameter` in the `FunctionSignature` with a Deprecation Level of ERROR. Please use multiple implementations of ExprFunction and use the LIST ExprValue to @@ -392,7 +452,7 @@ Thank you to all who have contributed! - Moves PartiqlAst, PartiqlLogical, PartiqlLogicalResolved, and PartiqlPhysical (along with the transforms) to a new project, `partiql-ast`. These are still imported into `partiql-lang` with the `api` annotation. Therefore, no action is required to consume the migrated classes. However, this now gives consumers of the AST, Experimental Plans, - Visitors, and VisitorTransforms the option of importing them directly using: `org.partiql:partiql-ast:${VERSION}`. + Visitors, and VisitorTransforms the option of importing them directly using: `org.partiql:partiql-ast:${VERSION}`. The file `partiql.ion` is still published in the `partiql-lang-kotlin` JAR. - Moves internal class org.partiql.lang.syntax.PartiQLParser to org.partiql.lang.syntax.impl.PartiQLPigParser as we refactor for explicit API. - Moves ANTLR grammar to `partiql-parser` package. The files `PartiQL.g4` and `PartiQLTokens.g4` are still published in the `partiql-lang-kotlin` JAR. @@ -477,15 +537,15 @@ Thank you to all who have contributed! ### Added -- Adds an initial implementation of GPML (Graph Pattern Matching Language), following - PartiQL [RFC-0025](https://github.com/partiql/partiql-docs/blob/main/RFCs/0025-graph-data-model.md) +- Adds an initial implementation of GPML (Graph Pattern Matching Language), following + PartiQL [RFC-0025](https://github.com/partiql/partiql-docs/blob/main/RFCs/0025-graph-data-model.md) and [RFC-0033](https://github.com/partiql/partiql-docs/blob/main/RFCs/0033-graph-query.md). This initial implementation includes: - - A file format for external graphs, defined as a schema in ISL (Ion Schema Language), + - A file format for external graphs, defined as a schema in ISL (Ion Schema Language), as well as an in-memory graph data model and a reader for loading external graphs into it. - - CLI shell commands `!add_graph` and `!add_graph_from_file` for bringing - externally-defined graphs into the evaluation environment. - - Evaluation of straight-path patterns with simple label matching and + - CLI shell commands `!add_graph` and `!add_graph_from_file` for bringing + externally-defined graphs into the evaluation environment. + - Evaluation of straight-path patterns with simple label matching and all directed/undirected edge patterns. - Adds new `TupleConstraint` variant, `Ordered`, to represent ordering in `StructType`. See the KDoc for more information. @@ -595,7 +655,7 @@ breaking changes if migrating from v0.9.2. The breaking changes accidentally int ### Added - Adds ability to pipe queries to the CLI. - Adds ability to run PartiQL files as executables by adding support for shebangs. -- Adds experimental syntax for CREATE TABLE, towards addressing +- Adds experimental syntax for CREATE TABLE, towards addressing [#36](https://github.com/partiql/partiql-docs/issues/36) of specifying PartiQL DDL. ### Changed @@ -1095,7 +1155,13 @@ breaking changes if migrating from v0.9.2. The breaking changes accidentally int ### Added Initial alpha release of PartiQL. -[Unreleased]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.14.4...HEAD +[Unreleased]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.14.8...HEAD +[1.0.0-rc.2]: https://github.com/partiql/partiql-lang-kotlin/compare/v1.0.0-rc.1...v1.0.0-rc.2 +[1.0.0-rc.1]: https://github.com/partiql/partiql-lang-kotlin/compare/v1.0.0-perf.1...v1.0.0-rc.1 +[0.14.8]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.14.7...v0.14.8 +[0.14.7]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.14.6...v0.14.7 +[0.14.6]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.14.5...v0.14.6 +[0.14.5]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.14.4...v0.14.5 [0.14.4]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.14.3...v0.14.4 [0.14.3]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.14.2...v0.14.3 [0.14.2]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.14.1...v0.14.2 diff --git a/README.md b/README.md index 6818fc481..927cf54e0 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ This project is published to [Maven Central](https://search.maven.org/artifact/o | Group ID | Artifact ID | Recommended Version | |---------------|-----------------------|---------------------| -| `org.partiql` | `partiql-lang-kotlin` | `0.14.5` | +| `org.partiql` | `partiql-lang-kotlin` | `1.0.0-rc.2` | For Maven builds, add the following to your `pom.xml`: diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 000000000..f748a1246 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,28 @@ +import io.github.gradlenexus.publishplugin.NexusPublishExtension +import java.time.Duration + +plugins { + id("io.github.gradle-nexus.publish-plugin") +} + +// We use gradle-nexus's publish-plugin to publish all of our published artifacts to Maven using OSSRH. +// Documentation for this plugin, see https://github.com/gradle-nexus/publish-plugin/blob/v2.0.0/README.md +// This plugin must be applied at the root project, so we include the following block around the nexus publish +// extension. +rootProject.run { + plugins.apply("io.github.gradle-nexus.publish-plugin") + extensions.getByType(NexusPublishExtension::class.java).run { + this.repositories { + sonatype { + nexusUrl.set(uri("https://aws.oss.sonatype.org/service/local/")) + username.set(properties["ossrhUsername"].toString()) + password.set(properties["ossrhPassword"].toString()) + } + } + + // these are not strictly required. The default timeouts are set to 1 minute. But Sonatype can be really slow. + // If you get the error "java.net.SocketTimeoutException: timeout", these lines will help. + connectTimeout.set(Duration.ofMinutes(3)) + clientTimeout.set(Duration.ofMinutes(3)) + } +} diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 5e57cc0d9..e37a84cb6 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -23,20 +23,22 @@ repositories { } object Versions { + const val binaryCompatibilityValidator = "0.14.0" const val detekt = "1.20.0-RC2" - const val dokka = "1.6.10" - const val kotlin = "1.6.20" + const val dokka = "1.9.20" + const val kotlin = "1.9.20" const val ktlintGradle = "10.2.1" - const val pig = "0.6.1" + const val nexusPublish = "2.0.0" const val shadow = "8.1.1" } object Plugins { + const val binaryCompatibilityValidator = "org.jetbrains.kotlinx:binary-compatibility-validator:${Versions.binaryCompatibilityValidator}" const val detekt = "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:${Versions.detekt}" const val dokka = "org.jetbrains.dokka:dokka-gradle-plugin:${Versions.dokka}" const val kotlinGradle = "org.jetbrains.kotlin:kotlin-gradle-plugin:${Versions.kotlin}" const val ktlintGradle = "org.jlleitschuh.gradle:ktlint-gradle:${Versions.ktlintGradle}" - const val pig = "org.partiql:pig-gradle-plugin:${Versions.pig}" + const val nexusPublish = "io.github.gradle-nexus:publish-plugin:${Versions.nexusPublish}" const val shadow = "com.github.johnrengelman:shadow:${Versions.shadow}" } @@ -45,7 +47,8 @@ dependencies { implementation(Plugins.dokka) implementation(Plugins.kotlinGradle) implementation(Plugins.ktlintGradle) - implementation(Plugins.pig) + implementation(Plugins.nexusPublish) + implementation(Plugins.binaryCompatibilityValidator) implementation(Plugins.shadow) } diff --git a/buildSrc/src/main/kotlin/org/partiql/gradle/plugin/publish/PublishPlugin.kt b/buildSrc/src/main/kotlin/org/partiql/gradle/plugin/publish/PublishPlugin.kt index f36e3c777..b68dd7860 100644 --- a/buildSrc/src/main/kotlin/org/partiql/gradle/plugin/publish/PublishPlugin.kt +++ b/buildSrc/src/main/kotlin/org/partiql/gradle/plugin/publish/PublishPlugin.kt @@ -29,13 +29,13 @@ import org.gradle.jvm.tasks.Jar import org.gradle.kotlin.dsl.create import org.gradle.kotlin.dsl.get import org.gradle.kotlin.dsl.getByName -import org.gradle.kotlin.dsl.provideDelegate import org.gradle.plugins.signing.SigningExtension import org.gradle.plugins.signing.SigningPlugin import org.jetbrains.dokka.gradle.DokkaPlugin import org.jetbrains.dokka.gradle.DokkaTask import org.jetbrains.kotlin.gradle.dsl.ExplicitApiMode import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension +import kotlinx.validation.BinaryCompatibilityValidatorPlugin import java.io.File /** @@ -53,6 +53,12 @@ abstract class PublishPlugin : Plugin { pluginManager.apply(MavenPublishPlugin::class.java) pluginManager.apply(SigningPlugin::class.java) pluginManager.apply(DokkaPlugin::class.java) + // Use https://github.com/Kotlin/binary-compatibility-validator to maintain list of public binary APIs (defaults + // to /api/.api). When changes are made to public APIs (e.g. modifying a public class, + // adding a public function, etc.), the gradle `apiCheck` task will fail. To fix this error, run the `apiDump` task + // to update these .api files and commit the changes. + // See https://github.com/Kotlin/binary-compatibility-validator#optional-parameters for additional configuration. + pluginManager.apply(BinaryCompatibilityValidatorPlugin::class.java) pluginManager.apply(ShadowPlugin::class.java) extensions.getByType(KotlinJvmProjectExtension::class.java).explicitApi = ExplicitApiMode.Strict val ext = extensions.create("publish", PublishExtension::class.java) @@ -122,7 +128,7 @@ abstract class PublishPlugin : Plugin { licenses { license { name.set("The Apache License, Version 2.0") - url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") + url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") } } developers { @@ -160,17 +166,6 @@ abstract class PublishPlugin : Plugin { } } } - repositories { - maven { - url = uri("https://aws.oss.sonatype.org/service/local/staging/deploy/maven2") - credentials { - val ossrhUsername: String by rootProject - val ossrhPassword: String by rootProject - username = ossrhUsername - password = ossrhPassword - } - } - } } // Sign only if publishing to Maven Central diff --git a/buildSrc/src/main/kotlin/partiql.versions.kt b/buildSrc/src/main/kotlin/partiql.versions.kt index 75eab9e8e..85147f3f9 100644 --- a/buildSrc/src/main/kotlin/partiql.versions.kt +++ b/buildSrc/src/main/kotlin/partiql.versions.kt @@ -18,9 +18,9 @@ object Versions { // Language - const val kotlin = "1.6.20" - const val kotlinLanguage = "1.6" - const val kotlinApi = "1.6" + const val kotlin = "1.9.20" + const val kotlinLanguage = "1.9" + const val kotlinApi = "1.9" const val jvmTarget = "1.8" // Dependencies @@ -28,27 +28,22 @@ object Versions { const val awsSdk = "1.12.344" const val csv = "1.8" const val dotlin = "1.0.2" - const val gson = "2.10.1" const val guava = "31.1-jre" const val ionElement = "1.0.0" - const val ionJava = "1.11.1" const val ionSchema = "1.2.1" const val jansi = "2.4.0" const val jgenhtml = "1.6" const val jline = "3.21.0" - const val jmhGradlePlugin = "0.7.2" - const val jmhCore = "1.37" - const val jmhGeneratorAnnprocess = "1.37" - const val jmhGeneratorBytecode = "1.37" const val joda = "2.12.1" const val kotlinPoet = "1.11.0" const val kotlinxCollections = "0.3.5" const val picoCli = "4.7.0" const val kasechange = "1.3.0" - const val pig = "0.6.2" - const val kotlinxCoroutines = "1.6.0" - const val kotlinxCoroutinesJdk8 = "1.6.0" + const val kotlinLombok = "1.9.20" + const val kotlinxCoroutines = "1.8.1" + const val kotlinxCoroutinesJdk8 = "1.8.1" const val ktlint = "0.42.1" // we're on an old version of ktlint. TODO upgrade https://github.com/partiql/partiql-lang-kotlin/issues/1418 + const val lombok = "1.18.34" // Testing const val assertj = "3.11.0" @@ -58,8 +53,6 @@ object Versions { const val junit4 = "4.12" const val junit4Params = "1.1.1" const val mockito = "4.5.0" - const val mockk = "1.11.0" - const val kotlinxCoroutinesTest = "1.6.0" } object Deps { @@ -75,9 +68,7 @@ object Deps { const val awsSdkS3 = "com.amazonaws:aws-java-sdk-s3:${Versions.awsSdk}" const val csv = "org.apache.commons:commons-csv:${Versions.csv}" const val dotlin = "io.github.rchowell:dotlin:${Versions.dotlin}" - const val gson = "com.google.code.gson:gson:${Versions.gson}" const val guava = "com.google.guava:guava:${Versions.guava}" - const val ionJava = "com.amazon.ion:ion-java:${Versions.ionJava}" const val ionElement = "com.amazon.ion:ion-element:${Versions.ionElement}" const val ionSchema = "com.amazon.ion:ion-schema-kotlin:${Versions.ionSchema}" const val jansi = "org.fusesource.jansi:jansi:${Versions.jansi}" @@ -88,11 +79,10 @@ object Deps { const val kotlinPoet = "com.squareup:kotlinpoet:${Versions.kotlinPoet}" const val kotlinxCollections = "org.jetbrains.kotlinx:kotlinx-collections-immutable:${Versions.kotlinxCollections}" const val picoCli = "info.picocli:picocli:${Versions.picoCli}" - const val pig = "org.partiql:partiql-ir-generator:${Versions.pig}" - const val pigRuntime = "org.partiql:partiql-ir-generator-runtime:${Versions.pig}" const val kotlinxCoroutines = "org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinxCoroutines}" const val kotlinxCoroutinesJdk8 = "org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:${Versions.kotlinxCoroutinesJdk8}" const val ktlint = "com.pinterest.ktlint:ktlint-core:${Versions.ktlint}" + const val lombok = "org.projectlombok:lombok:${Versions.lombok}" // Testing const val assertj = "org.assertj:assertj-core:${Versions.assertj}" @@ -105,8 +95,6 @@ object Deps { const val kotlinTest = "org.jetbrains.kotlin:kotlin-test:${Versions.kotlin}" const val kotlinTestJunit = "org.jetbrains.kotlin:kotlin-test-junit5:${Versions.kotlin}" const val mockito = "org.mockito:mockito-junit-jupiter:${Versions.mockito}" - const val mockk = "io.mockk:mockk:${Versions.mockk}" - const val kotlinxCoroutinesTest = "org.jetbrains.kotlinx:kotlinx-coroutines-test:${Versions.kotlinxCoroutinesTest}" const val ktlintTest = "com.pinterest.ktlint:ktlint-test:${Versions.ktlint}" } @@ -121,7 +109,7 @@ object Plugins { const val application = "org.gradle.application" const val detekt = "io.gitlab.arturbosch.detekt" const val dokka = "org.jetbrains.dokka" - const val jmh = "me.champeau.jmh" const val library = "org.gradle.java-library" + const val kotlinLombok = "org.jetbrains.kotlin.plugin.lombok" const val testFixtures = "org.gradle.java-test-fixtures" } \ No newline at end of file diff --git a/docs/wiki/_Sidebar.md b/docs/wiki/_Sidebar.md index 6661873d9..7630ed802 100644 --- a/docs/wiki/_Sidebar.md +++ b/docs/wiki/_Sidebar.md @@ -38,3 +38,18 @@ * [Abstract Syntax Tree](https://github.com/partiql/partiql-lang-kotlin/wiki/Abstract-Syntax-Tree) * [Architecture Design](https://github.com/partiql/partiql-lang-kotlin/wiki/Architecture-Design) * [Code Style Guidelines](https://github.com/partiql/partiql-lang-kotlin/wiki/CODE-STYLE) +--- +* V1 Documentation + * Basics + * [What is PartiQL?](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO) + * PartiQL CLI + * [Installation](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO) + * [Usage Guide](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO) + * Developer Usage Guides + * [PartiQL Library Use-Cases](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO) + * [Understanding PartiQL's Components](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO) + * [Using the Parser](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO) + * [Using the Planner](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO) + * [Using the Compiler](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO) + * [Executing Compiled Queries](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO) + * [Handling Errors](https://github.com/partiql/partiql-lang-kotlin/wiki/ErrorHandling) diff --git a/docs/wiki/v1/ErrorHandling.md b/docs/wiki/v1/ErrorHandling.md new file mode 100644 index 000000000..244fdde84 --- /dev/null +++ b/docs/wiki/v1/ErrorHandling.md @@ -0,0 +1,231 @@ +# Usage Guide: Error Handling + +## Introduction + +The PartiQL library provides a robust error reporting mechanism, and this usage guide aims to show +how you can leverage the exposed APIs. + +## Who is this for? + +This usage guide is aimed at developers who use any one of [PartiQL's components](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO) for their +application. If you are looking for how to change how errors are reported in the CLI, please run: `partiql --help`. + +To elaborate on why this usage guide may be useful to you, the developer, let us assume that your +company provides a CLI to enable your customers to execute PartiQL queries. When a user is typing a query and +references a table that doesn't exist, your CLI might want to highlight that error and halt processing of the +query to save on computational costs. Or, your CLI might want to highlight the error but continue processing the query to accumulate +errors to better enable the developer to see all of their mistakes at once. In any case, the PartiQL +library allows developers to register their own error listeners to take control over their customers' experience. + +## Error Listeners + +Each major component (parser, planner, compiler) of the PartiQL Library allows for the registration of an `ErrorListener` +that will receive every warning/error that the particular component emits. The default error listener aborts the +component's execution, by throwing an `PErrorListenerException`, upon encountering the first error. This behavior aims to +protect developers who might have decided to avoid reading this documentation. However, as seen further below, this is +easy to override. + +## Halting a Component's Execution + +In the scenario where you want to halt one of the components when a particular warning/error is emitted, error listeners +have the ability to throw an `PErrorListenerException`. This exception acts as a wrapper over any exception you'd like to +halt with. For example: + +```java +import org.partiql.spi.errors.PError; +import org.partiql.spi.errors.PErrorListener; + +import java.lang.annotation.Native; + +class AbortWhenAlwaysMissing extends PErrorListener { + // This is to be used to halt my application after the component finishes execution + boolean hasErrors = false; + + @Override + void error(@NotNull PError error) throws PErrorListenerException { + System.out.println("e: " + getErrorMessage(error)); + hasErrors = true; + } + + @Override + void warning(@NotNull PError error) throws PErrorListenerException { + if (error.getCode() == Error.ALWAYS_MISSING) { + throw new PErrorListenerException("This system does not allow for expressions that always return missing!"); + } + println("w: " + getErrorMessage(error)); + } + + private fun getErrorMessage(@NotNull PError error) { + // Internal implementation details + } +} +``` + +**NOTE**: If you throw an exception that is not an `PErrorListenerException`, the component that contains your registered +`ErrorListener` will catch the exception and send an error to your `ErrorListener` with a code of +`Error.INTERNAL_ERROR`. This will lead to a duplication of errors (which can be a bad experience for your +customers). + +## Registering Error Listeners + +Each component allows for the registration of a custom error listener upon instantiation. For example, let's say you +intend on registering the `AbortWhenAlwaysMissing` error listener from above: +```java + +public class Foo { + public static void main(String[] args) { + // Error Listener + PErrorListener listener = AbortWhenAlwaysMissing(); + + // User Input + String query = args[0]; + Statement ast = parse(query); + + // Planner Component + PartiQLPlanner planner = PartiQLPlanner.standard(); + Context plannerConfig = Context.of(listener); // Registration here!! + + // Planning and catching the PErrorListenerException + Plan plan; + try { + plan = planner.plan(ast, plannerConfig); + } catch (PErrorListenerException ex) { + throw ex; + } + + // Do more ... + } + + private Statement parse(String query) { + // Calling the PartiQL Parser, handling PErrorListenerExceptions, etc. + } +} +``` + +## Errors and Warnings + +Errors and warnings are both represented by the same data structure, an `Error`. In the case of an error/warning, it is +up to the respective component to correctly send the `Error` to either `ErrorListener.error()` or +`ErrorListener.warning()`. + +The `Error` Java class allows for developers to introspect its properties to determine how to create their own error +messages. See the [Javadocs] for the available methods. + +## Writing Quality Error Messages + +As mentioned above, the `Error` Java class exposes information for database implementers to write high quality error +messages. Specifically, `Error` exposes a method, `int getCode()`, to return the enumerated error code received. All +possible error codes are represented as static final fields in +the [Error Javadocs](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO). + +An error code MAY have additional properties accessible via the `.get(...)` API – please consult the Javadocs for an +error code's property usage. + +Now, here's an example of how you might write a quality error message: +```java +public class ConsolePErrorListener extends PErrorListener { + + boolean hasErrors = false; + + @Override + void error(@NotNull PError error) throws PErrorListenerException { + String message = getMessage(error, "e: "); + System.out.println(message); + hasErrors = true; + } + + @Override + void warning(@NotNull PError error) throws PErrorListenerException { + String message = getMessage(error, "w: "); + System.out.println(message); + } + + static String getMessage(@NotNull PError error, @NotNull String prefix) { + switch (error.getCode()) { + case Error.ALWAYS_MISSING: + SourceLocation location = error.location; + String locationStr = getNullSafeLocation(location); + return prefix + locationStr + " Expression always evaluates to missing."; + case Error.FEATURE_NOT_SUPPORTED: + String name = (String) error.getProperty(Property.FEATURE_NAME); + if (name == null) { + name = "UNKNOWN"; + } + return prefix + "Feature (" + name + ") not yet supported."; + default: + return "Unhandled error code received."; + } + } + + @NotNull + String getNullSafeLocation(@Nullable SourceLocation location) { + // Internal implementation + } +} +``` + +## A Component's Output Structures + +Each of PartiQL's components produce a structure for future use. The parser outputs an AST, the planner outputs a plan, +and the compiler outputs an executable. What happens when any of the components experience an error/warning? + +The answer, as is often in software, depends. Since this error reporting mechanism allows developers to register error +listeners that accumulate all errors, the PartiQL components still continue processing until terminated by an error +listener. That being said, when error listeners receive an error, one must assume that the output of the component +is a dud and is incorrect. Therefore, if the parser has produced errors with a malformed AST, you shouldn't pass +the AST to the planner to continue evaluation. + +However, if warnings have been emitted, the output can still be safely relied upon. For example, let's use the same +error listener we wrote further above: +```java +class Example { + + public Plan planInternal(Statement ast) throws PlanningFailure { + AbortWhenAlwaysMissing listener = AbortWhenAlwaysMissing(); + PartiQLPlanner planner = PartiQLPlanner.standard(); + Context plannerConfig = Context.of(listener); + + Plan plan; + try { + plan = planner.plan(ast, plannerConfig); + } catch (PErrorListenerException ex) { + throw new PlanningFailure(ex); + } + // If an error has been reported to the listener, implementers + // should NOT trust the plan that has been returned. + if (listener.hasErrors) { + throw new PlanningFailure("Errors encountered. Exiting."); + } + return plan; + } +} +``` + +## What about Execution? + +Error listeners are specifically meant to provide control over the reporting of errors for PartiQL's major components (parser, +planner, and compiler). However, for the execution of compiled statements, PartiQL still provides errors (and error codes) +by throwing an `EvaluationException` which exposes a method, `Error getError()`. The `EvaluationException` does not +expose a message, cause, or stacktrace. + +Here is an example of how you can leverage this functionality below: +```java +class MyApplication { + void executeAndPrint(PreparedStatement stmt, Session session) { + Datum lazyData; + try { + lazyData = stmt.execute(session); + // Iterate through the lazyData and print to the console. + } catch (EvaluationException e) { + System.out.println(ConsoleErrorListener.getMessage(e.getError(), "e: ")); + } + } +} +``` + +## Reference Implementations + +The PartiQL CLI offers multiple ways to process warnings/errors. See the flags `-Werror`, `-w`, +`--max-errors`, and more when you run `partiql --help`. See the CLI Usage Guide +[here](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO). The implementation details can be found in the +[CLI subproject](https://github.com/partiql/partiql-lang-kotlin/wiki/TODO). diff --git a/docs/wiki/v1/TODO.md b/docs/wiki/v1/TODO.md new file mode 100644 index 000000000..303146283 --- /dev/null +++ b/docs/wiki/v1/TODO.md @@ -0,0 +1,6 @@ +# TODO + +This document has not yet been written. + +If you have a pressing need for the documentation you were looking for, please reach out by filing +a GitHub Issue. Sorry for the inconvenience! diff --git a/docs/wiki/v1/compiler.md b/docs/wiki/v1/compiler.md new file mode 100644 index 000000000..c86da18b0 --- /dev/null +++ b/docs/wiki/v1/compiler.md @@ -0,0 +1,86 @@ +# PartiQL Compiler Guide + +This document is a guide to using the PartiQL compiler. + +## Overview + +The compiler is responsible for converting logical plans (an _operator_ tree) +to physical plans (an _expr_ tree) by applying _strategies_. A _strategy_ is a class +that has a _pattern_ and an _apply_ method. The pattern determines when to +invoke the _apply_ method which converts the matched operators (logical) to expressions (physical). + +## Pattern Matching + +A pattern is a tree composed of nodes with one of the types, + +``` +TYPE: T – match if class T, check children. +ANY: ? - match any node, check children. +ALL: * - match all nodes in the subtree. +``` + +For backwards compatibility, a pattern will ignore unmatched children. +If this is not the desired behavior, a pattern can be marked as "strict" +which will match children exactly and error on extraneous children. + +### Example + +Let's combine a limit and offset into a single relational expression; the logical tree looks like this. + +``` +RelLimit Pattern.match(..) + \ + RelOffset Pattern.match(..) +``` + +We use the builders to create a pattern. + +``` +Pattern.match(RelLimit::class) + .child(Pattern.match(RelOffset::class)) + .build() +``` + +In practice, the compiler will be walking the tree so we must deal with the inputs which have been recursively compiled. +Because these nodes have been compiled, they are part of the "physical" or "Expr" domain. To illustrate, I've enclosed +the compiled children nodes with `< >` so `ExprValue -> ` and `ExprRelation -> `. + +Recall the definition of a `RelLimit` and a `RelOffset` + +``` +* RelLimit(input: Rel, limit: Rex) +* RelOffset(input: Rel, offset: Rex) +``` + +I have labelled these children in the illustration so that you can see where the end up in the match. + +``` +... + \ + RelLimit + / \ +x: RelOffset + / \ + y: z: +``` + +The compiler will look for this pattern in the operator tree, and produce a match like so, + +``` +Match { + matched: [ + RelLimit, + RelOffset, + ], + children: [ + [x:], + [y:, z: ] + ], +} +``` + +The matched items are the flattened (in-order traversal) matched nodes from the pattern, while the children +is a nested list of corresponding compiled children. + +This match structure is sent to the Strategy which gives the implementor all the information they need to know +to continue folding the tree. diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts index 749d49222..bef0075cc 100644 --- a/examples/build.gradle.kts +++ b/examples/build.gradle.kts @@ -23,7 +23,7 @@ application { } dependencies { - implementation(project(":partiql-lang")) + implementation("org.partiql:partiql-lang-kotlin:0.14.8") implementation(Deps.kotlinxCoroutines) implementation(Deps.kotlinxCoroutinesJdk8) implementation(Deps.awsSdkS3) diff --git a/gradle.properties b/gradle.properties index e6bdc6d7b..68096da59 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=org.partiql -version=0.14.6-SNAPSHOT +version=1.0.0-rc.3-SNAPSHOT ossrhUsername=EMPTY ossrhPassword=EMPTY diff --git a/lib/isl/README.md b/lib/isl/README.md deleted file mode 100644 index 9c874d867..000000000 --- a/lib/isl/README.md +++ /dev/null @@ -1,96 +0,0 @@ -## PartiQL ISL Kotlin (Testing Utility) - -**NOTE**: PartiQL ISL is no longer published to Maven. The generated classes should only be used for -internal testing. - -PartiQL ISL Kotlin provides an in-memory [Ion Schema Language](https://amzn.github.io/ion-schema/docs/spec.html) (ISL) object -model. The open-source Kotlin implementation of ISL, [ion-schema-kotlin](https://github.com/amzn/ion-schema-kotlin) allows -for validation of schemas, but does not provide ways for programmatic manipulation of ISL schemas. The implementation -uses [partiql-ir-generator](https://github.com/partiql/partiql-ir-generator) (PIG), a domain modeling and code generating -tool for tree-like data structures, to generate class definitions and APIs for creating/manipulating -ISL entities. PIG also provides visitors, tree walkers, visitor transforms, as well as (de)serialization code to and from Ion. -Some possible use cases for this API could include rewriting schemas through visitor transforms, unifying/merging -multiple schemas, and inferring schemas. - -## Using PIG ISL Domain and Parser - -### PIG ISL Domain -PartiQL ISL Kotlin uses PIG's DSL to model ISL in `isl.ion`. PIG uses that file to generate `isl-model.kt` in which -users can create and manipulate the `IonSchemaModel` class. Users can create instances of `IonSchemaModel` using the following pattern: - -```Kotlin -// Defining an ISL type_definition -IonSchemaModel.build { - typeDefinition("foo", constraintList()) -} -``` - -### Converting an ISL document to `IonSchemaModel` -`parseSchema` is the primary API to convert from ISL to the object domain model, `IonSchemaModel` -```Kotlin -/** - * Transforms an ISL document into an [IonSchemaModel.Schema], which is a PIG-generated in-memory object representing - * ISL entities. - * - * @param elements an [IonElement] representation of an ISL document to be transformed to [IonSchemaModel.Schema] - * @return [elements] transformed into an [IonSchemaModel.Schema] - */ -fun parseSchema(elements: List): IonSchemaModel.Schema -``` - -Users can load schemas that can be converted to `IonElement`. [ion-element-kotlin](https://github.com/amzn/ion-element-kotlin/blob/master/src/com/amazon/ionelement/api/IonElementLoader.kt) -provides several functions to create `IonElement` instances from different sources. The following is an example using `loadAllElements(ionText: String)`: -```Kotlin -val elements: List = loadAllElements("type::{ name: foo }") -val parsedSchemaModel: IonSchemaModel.Model = parseSchema(elements) -``` - -Alternatively, users can parse `ion-schema-kotlin` [Schema](https://github.com/amzn/ion-schema-kotlin/blob/master/src/com/amazon/ionschema/Schema.kt#L36) -by first converting to `IonElement`: -```Kotlin -val schema = ... // some ion-schema-kotlin Schema object -val schemaElements = schema.isl.map { it.toIonElement() } -val parsedSchemaModel: IonSchemaModel.Model = parseSchema(schemaElements) -``` - -### Converting an `IonSchemaModel` to an ISL document -`toIsl` is the primary API to convert from an `IonSchemaModel` to ISL: -```Kotlin -/** - * Transforms a PIG-generated [IonSchemaModel.Schema] into an [IonElement] representation of an ISL document. - * - * @receiver [IonSchemaModel.Schema] to be transformed to an ISL document - * @return transformed ISL document represented as a List<[AnyElement]> - */ -fun IonSchemaModel.Schema.toIsl(): List -``` - -We can convert an `IonSchemaModel.Schema` back to an `IonElement` representation of ISL using the `toIsl` extension function: -```Kotlin -val schemaModel = IonSchemaModel.build { - schema(typeStatement(typeDefinition("foo", constraintList()))) -} -val islElements: List = schemaModel.toIsl() -``` - -## API Status -PartiQL ISL Kotlin is currently usable, but the `isl.ion` domain definition and the code generated by PIG is under development and subject to change. - -## Building -Clone the repo and from the root directory run the following: - -``` -$./gradlew build -``` - -This will use PIG to generate the object model file, which can be found at `src/org/partiql/ionschema/model/isl-model.kt`. -The command will also run the ISL parser's unit tests. - -## Security - -See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. - -## License - -This project is licensed under the Apache-2.0 License. - diff --git a/lib/isl/build.gradle.kts b/lib/isl/build.gradle.kts deleted file mode 100644 index fd83cfc6e..000000000 --- a/lib/isl/build.gradle.kts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -plugins { - id(Plugins.conventions) - id(Plugins.dokka) - id(Plugins.library) - id(Plugins.pig) -} - -dependencies { - api(Deps.ionElement) - api(Deps.ionJava) - api(Deps.ionSchema) - api(Deps.pigRuntime) -} - -// Disabled for ISL project -kotlin { - explicitApi = null -} - -pig { - namespace = "org.partiql.ionschema.model" -} - -tasks.dokkaHtml { - dependsOn(tasks.withType(org.partiql.pig.gradle.PigTask::class)) -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintDiscoverer.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintDiscoverer.kt deleted file mode 100644 index 6dbdc2c04..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintDiscoverer.kt +++ /dev/null @@ -1,223 +0,0 @@ -package org.partiql.ionschema.discovery - -import com.amazon.ion.IonBlob -import com.amazon.ion.IonBool -import com.amazon.ion.IonClob -import com.amazon.ion.IonDecimal -import com.amazon.ion.IonFloat -import com.amazon.ion.IonInt -import com.amazon.ion.IonList -import com.amazon.ion.IonNull -import com.amazon.ion.IonSexp -import com.amazon.ion.IonString -import com.amazon.ion.IonStruct -import com.amazon.ion.IonSymbol -import com.amazon.ion.IonTimestamp -import com.amazon.ion.IonType -import com.amazon.ion.IonValue -import com.amazon.ionelement.api.ionInt -import org.partiql.ionschema.model.IonSchemaModel -import java.math.BigInteger - -// Additional constraint discovery constants -internal const val MIN_INT2 = Short.MIN_VALUE.toLong() -internal const val MAX_INT2 = Short.MAX_VALUE.toLong() -internal const val MIN_INT4 = Int.MIN_VALUE.toLong() -internal const val MAX_INT4 = Int.MAX_VALUE.toLong() -internal const val MIN_INT8 = Long.MIN_VALUE -internal const val MAX_INT8 = Long.MAX_VALUE -internal val INT2_RANGE = BigInteger.valueOf(MIN_INT2)..BigInteger.valueOf(MAX_INT2) -internal val INT4_RANGE = BigInteger.valueOf(MIN_INT4)..BigInteger.valueOf(MAX_INT4) -internal val INT8_RANGE = BigInteger.valueOf(MIN_INT8)..BigInteger.valueOf(MAX_INT8) -internal val INT2_RANGE_CONSTRAINT = IonSchemaModel.build { validValues(rangeOfValidValues(numRange(numberRange(inclusive(ionInt(MIN_INT2)), inclusive(ionInt(MAX_INT2)))))) } -internal val INT4_RANGE_CONSTRAINT = IonSchemaModel.build { validValues(rangeOfValidValues(numRange(numberRange(inclusive(ionInt(MIN_INT4)), inclusive(ionInt(MAX_INT4)))))) } -internal val INT8_RANGE_CONSTRAINT = IonSchemaModel.build { validValues(rangeOfValidValues(numRange(numberRange(inclusive(ionInt(MIN_INT8)), inclusive(ionInt(MAX_INT8)))))) } - -/** - * Defines how additional constraints are to be discovered. This is intended to be called by a [ConstraintInferer] for - * each of the [IonType]s. - */ -internal interface ConstraintDiscoverer { - fun discover(value: IonValue): IonSchemaModel.ConstraintList -} - -/** - * An implementation of [ConstraintDiscoverer] that supports all [IonType]s except for DATAGRAM. All base - * implementations return an empty [IonSchemaModel.ConstraintList] and do not depend on each other (i.e. sequence - * and struct constraint discoverers do not call the scalar constraint discoverers). - * - * Since this is intended to be used by the [TypeAndConstraintInferer] after inferring the - * [IonSchemaModel.Constraint.TypeConstraint], typed nulls collapse to `null` and will not have any additional - * constraints discovered. - */ -internal open class TypeConstraintDiscoverer : ConstraintDiscoverer { - override fun discover(value: IonValue): IonSchemaModel.ConstraintList = - when (value) { - is IonBool -> constraintDiscovererBool(value) - is IonInt -> constraintDiscovererInt(value) - is IonFloat -> constraintDiscovererFloat(value) - is IonDecimal -> constraintDiscovererDecimal(value) - is IonTimestamp -> constraintDiscovererTimestamp(value) - is IonSymbol -> constraintDiscovererSymbol(value) - is IonString -> constraintDiscovererString(value) - is IonClob -> constraintDiscovererClob(value) - is IonBlob -> constraintDiscovererBlob(value) - is IonNull -> constraintDiscovererNull(value) - is IonSexp -> constraintDiscovererSexp(value) - is IonList -> constraintDiscovererList(value) - is IonStruct -> constraintDiscovererStruct(value) - else -> error("Given type is not supported for conversion") - } - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonBool]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererBool(value: IonBool): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonInt]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererInt(value: IonInt): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonFloat]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererFloat(value: IonFloat): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonDecimal]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererDecimal(value: IonDecimal): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonTimestamp]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererTimestamp(value: IonTimestamp): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonSymbol]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererSymbol(value: IonSymbol): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonString]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererString(value: IonString): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonClob]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererClob(value: IonClob): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonBlob]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererBlob(value: IonBlob): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonNull]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererNull(value: IonNull): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonSexp]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererSexp(value: IonSexp): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonList]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererList(value: IonList): IonSchemaModel.ConstraintList = emptyConstraintList - - /** - * Returns a [IonSchemaModel.ConstraintList] with additional discovered constraints for [IonStruct]s. - * - * This implementation returns an empty constraint list. - */ - open fun constraintDiscovererStruct(value: IonStruct): IonSchemaModel.ConstraintList = emptyConstraintList -} - -/** - * A [ConstraintDiscoverer] that infers additional constraints for the following [IonType]s: - * [IonInt]- valid_values - * [IonDecimal]- scale and precision - * [IonString]- codepoint_length - * - * Currently, this class is not open and not intended to be extended. Creating a new [ConstraintDiscoverer] with - * some specific discovered constraints can be done through overriding [TypeConstraintDiscoverer]'s extensible - * `constraintDiscoverer...` functions. - */ -internal class StandardConstraintDiscoverer : TypeConstraintDiscoverer() { - override fun constraintDiscovererInt(value: IonInt) = INT_VALID_VALUES_DISCOVERER(value) - override fun constraintDiscovererDecimal(value: IonDecimal) = DECIMAL_SCALE_AND_PRECISION_DISCOVERER(value) - override fun constraintDiscovererString(value: IonString) = STRING_CODEPOINT_LENGTH_DISCOVERER(value) -} - -/** - * Given an [IonInt], returns a constraint list with [IonSchemaModel.Constraint.ValidValues] range depending on its - * value. If the int is between: - * [Short.MIN_VALUE] to [Short.MAX_VALUE]: range(Short.MIN_VALUE, Short.MAX_VALUE) - * [Int.MIN_VALUE] to [Int.MAX_VALUE]: range(Int.MIN_VALUE, Int.MAX_VALUE) - * [Long.MIN_VALUE] to [Long.MAX_VALUE]: range(Long.MIN_VALUE, Long.MAX_VALUE) - * else: no valid_values range included - */ -internal val INT_VALID_VALUES_DISCOVERER = { value: IonInt -> - IonSchemaModel.build { - when (value.bigIntegerValue()) { - in INT2_RANGE -> constraintList(INT2_RANGE_CONSTRAINT) - in INT4_RANGE -> constraintList(INT4_RANGE_CONSTRAINT) - in INT8_RANGE -> constraintList(INT8_RANGE_CONSTRAINT) - else -> constraintList() // unconstrained int has no constraint added - } - } -} - -/** - * Given an [IonDecimal], returns a constraint list with the decimal's [IonSchemaModel.Constraint.Scale] and - * [IonSchemaModel.Constraint.Precision]. - */ -internal val DECIMAL_SCALE_AND_PRECISION_DISCOVERER = { value: IonDecimal -> - val decimal = value.decimalValue() - val scale = decimal.scale().toLong() - val precision = decimal.precision().toLong() - - IonSchemaModel.build { - constraintList( - scale(equalsNumber(ionInt(scale))), - precision(equalsNumber(ionInt(precision))) - ) - } -} - -/** - * Given an [IonString], returns a constraint list with the string's [IonSchemaModel.Constraint.CodepointLength]. - */ -internal val STRING_CODEPOINT_LENGTH_DISCOVERER = { value: IonString -> - val s = value.stringValue() - val len = s.length.toLong() - IonSchemaModel.build { constraintList(codepointLength(equalsNumber(ionInt(len)))) } -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintInferer.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintInferer.kt deleted file mode 100644 index c9d9f52b1..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintInferer.kt +++ /dev/null @@ -1,206 +0,0 @@ -package org.partiql.ionschema.discovery - -import com.amazon.ion.IonBlob -import com.amazon.ion.IonBool -import com.amazon.ion.IonClob -import com.amazon.ion.IonDecimal -import com.amazon.ion.IonFloat -import com.amazon.ion.IonInt -import com.amazon.ion.IonList -import com.amazon.ion.IonNull -import com.amazon.ion.IonSequence -import com.amazon.ion.IonSexp -import com.amazon.ion.IonString -import com.amazon.ion.IonStruct -import com.amazon.ion.IonSymbol -import com.amazon.ion.IonTimestamp -import com.amazon.ion.IonType -import com.amazon.ion.IonValue -import com.amazon.ionelement.api.ionBool -import com.amazon.ionschema.Type -import org.partiql.ionschema.model.IonSchemaModel - -/** - * Infers [IonSchemaModel.Constraint]s for a given [IonValue]. - * - * Implementations will need to define which set of [IonType]s to infer constraints for. This will usually be for - * all [IonType]s except DATAGRAM. In a typical use case, [inferConstraints] will be called for all scalar [IonType]s - * and will be called recursively for all fields of an [IonStruct] and elements of an [IonSequence] (which may require - * a [ConstraintUnifier] to return a unified [IonSchemaModel.Constraint.Element]). - */ -internal interface ConstraintInferer { - fun inferConstraints(value: IonValue): IonSchemaModel.ConstraintList -} - -/** - * Infers [IonSchemaModel.Constraint.TypeConstraint] and additional constraints discovered using the - * [constraintDiscoverer] for all [IonType]s except for DATAGRAM. For [IonStruct]s, also infers - * [IonSchemaModel.Constraint.Fields] and adds [IonSchemaModel.Constraint.ClosedContent]. For [IonSequence]s, infers - * [IonSchemaModel.Constraint.Element]. - * - * If an [IonValue] is valid for one of the [importedTypes] (i.e. value does not violate any of the imported type's - * constraints), then the type constraint will use the imported type's name. - * - * Typed null [IonValue]s will collapse to null (i.e. type: nullable::$null) and lose their type information. - * - * @param constraintUnifier unifies constraints for [IonSequence]s' elements - * @param constraintDiscoverer discovers additional constraints (other than [IonSchemaModel.Constraint.TypeConstraint]) - * @param importedTypes are additional [Type]s that can be inferred for a given [IonValue]. - */ -internal class TypeAndConstraintInferer( - val constraintUnifier: ConstraintUnifier, - val constraintDiscoverer: ConstraintDiscoverer = StandardConstraintDiscoverer(), - private val importedTypes: List = emptyList() -) : ConstraintInferer { - private val nullNamedType = IonSchemaModel.build { namedType("\$null", nullable = ionBool(true)) } - private val nullNamedTypeConstraintList = IonSchemaModel.build { constraintList(typeConstraint(nullNamedType)) } - private val notNullable = ionBool(false) - - /** - * For each [value], returns an [IonSchemaModel.ConstraintList] with the inferred type constraint and additional - * discovered constraints using [constraintDiscoverer]. For [IonSequence]s, also infers the - * [IonSchemaModel.Constraint.Element] constraint, unifying using [constraintUnifier] if the sequences' elements - * have conflicting inferred constraints. For [IonStruct], infers the [IonSchemaModel.Constraint.Fields] and adds - * the [IonSchemaModel.Constraint.ClosedContent] constraint. - */ - override fun inferConstraints(value: IonValue): IonSchemaModel.ConstraintList { - return when (value) { - is IonBool -> constraintsFromScalar(value, TypeConstraint.BOOL.typeName) - is IonInt -> constraintsFromScalar(value, TypeConstraint.INT.typeName) - is IonFloat -> constraintsFromScalar(value, TypeConstraint.FLOAT.typeName) - is IonDecimal -> constraintsFromScalar(value, TypeConstraint.DECIMAL.typeName) - is IonTimestamp -> constraintsFromScalar(value, TypeConstraint.TIMESTAMP.typeName) - is IonSymbol -> constraintsFromScalar(value, TypeConstraint.SYMBOL.typeName) - is IonString -> constraintsFromScalar(value, TypeConstraint.STRING.typeName) - is IonClob -> constraintsFromScalar(value, TypeConstraint.CLOB.typeName) - is IonBlob -> constraintsFromScalar(value, TypeConstraint.BLOB.typeName) - is IonNull -> constraintsFromScalar(value, TypeConstraint.NULL.typeName) - is IonSexp -> constraintsFromSequence(value, TypeConstraint.SEXP.typeName) - is IonList -> constraintsFromSequence(value, TypeConstraint.LIST.typeName) - is IonStruct -> constraintsFromStruct(value) - else -> error("Given $value is not supported for constraint inference") - } - } - - /** - * Returns the first type name that [this] [IonValue] meets all the type constraints for among the list of - * additional imported types. If [this] [IonValue] does not meet the type constraints for any of the additional - * imported types, will return [typeConstraintName]. - */ - private fun IonValue.getSpecificType(typeConstraintName: String): String { - if (this.typeAnnotations.isNotEmpty()) { - importedTypes.forEach { - if (it.isValid(this)) { - return it.name - } - } - } - return typeConstraintName - } - - /** - * Given a scalar type (i.e. non-sequence, non-struct), returns an [IonSchemaModel.ConstraintList] with the - * type constraint [typeConstraintName] and additional discovered constraints. Typed nulls collapse to the null - * type constraint. - */ - private fun constraintsFromScalar(value: IonValue, typeConstraintName: String): IonSchemaModel.ConstraintList { - val realTypeName = value.getSpecificType(typeConstraintName) - - if (value.isNullValue && realTypeName == typeConstraintName) { - // null and typed nulls for scalar types collapse to null - return nullNamedTypeConstraintList - } - - val constraints = mutableListOf(IonSchemaModel.build { typeConstraintOf(realTypeName) }) - val additionalConstraints = constraintDiscoverer.discover(value) - constraints.addAll(additionalConstraints.items) - - return IonSchemaModel.build { constraintList(constraints) } - } - - /** - * Given an [IonSequence], returns an [IonSchemaModel.ConstraintList] with the sequence's type constraint, - * additional discovered constraints, and [IonSchemaModel.Constraint.Element]. Typed nulls collapse to the null - * type constraint. - */ - private fun constraintsFromSequence(value: IonSequence, typeConstraintName: String): IonSchemaModel.ConstraintList { - val sequenceTypeName = value.getSpecificType(typeConstraintName) - - if (value.isNullValue) { - // null.list and null.sexp collapse to null - return nullNamedTypeConstraintList - } - - val constraints = mutableListOf(IonSchemaModel.build { typeConstraintOf(sequenceTypeName) }) - val additionalConstraints = constraintDiscoverer.discover(value) - constraints.addAll(additionalConstraints.items) - - if (value.isEmpty) { - return IonSchemaModel.build { constraintList(constraints) } - } - - val elementConstraintList = value.map { inferConstraints(it) } - .asSequence() - .unifiedConstraintList(constraintUnifier) - - if (elementConstraintList.isAny()) { - return IonSchemaModel.build { constraintList(constraints) } - } - - constraints.add(IonSchemaModel.build { element(type = inlineType(typeDefinition(constraints = elementConstraintList), notNullable)) }) - return IonSchemaModel.build { constraintList(constraints) } - } - - /** - * Given an [IonStruct], returns an [IonSchemaModel.ConstraintList] with the - * 1. type constraint of [TypeConstraint.STRUCT] name (unless an imported type is inferred) - * 2. [IonSchemaModel.Constraint.Fields] constraint (unless an imported type is inferred) - * 3. additional discovered constraints (unless an imported type is inferred) - * 4. [IonSchemaModel.Constraint.ClosedContent] (unless an imported type is inferred). - * - * Typed null collapses to the null type constraint. - */ - private fun constraintsFromStruct(value: IonStruct): IonSchemaModel.ConstraintList { - val fields = value.associateBy(keySelector = { it.fieldName }, valueTransform = { inferConstraints(it) }) - val realTypeName = value.getSpecificType(TypeConstraint.STRUCT.typeName) - - if (realTypeName != TypeConstraint.STRUCT.typeName) { - return IonSchemaModel.build { constraintList(typeConstraintOf(realTypeName)) } - } - - if (value.isNullValue) { - // null.struct collapses to null - return nullNamedTypeConstraintList - } - - val structConstraints = mutableListOf( - IonSchemaModel.build { typeConstraint(namedType(TypeConstraint.STRUCT.typeName, notNullable)) }, - IonSchemaModel.build { closedContent() } - ) - - if (fields.isNotEmpty()) { - structConstraints.add( - IonSchemaModel.build { - fields( - fields.map { - field( - name = it.key, - type = inlineType( - type = typeDefinition( - name = null, - constraints = it.value - ), - nullable = notNullable - ) - ) - } - ) - } - ) - } - val additionalConstraints = constraintDiscoverer.discover(value) - structConstraints.addAll(additionalConstraints.items) - - return IonSchemaModel.build { constraintList(structConstraints) } - } -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintUnifier.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintUnifier.kt deleted file mode 100644 index 6cd4dc01b..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintUnifier.kt +++ /dev/null @@ -1,323 +0,0 @@ -package org.partiql.ionschema.discovery - -import com.amazon.ionelement.api.ionBool -import org.partiql.ionschema.model.IonSchemaModel - -private val notNullable = ionBool(false) - -/** - * Defines the strategy to unify two constraint lists with different [IonSchemaModel.Constraint.TypeConstraint]s. - * - * When [UNION], conflicting type constraints will be included in a union (i.e. [IonSchemaModel.Constraint.AnyOf]. - * e.g. type: int - * with type: decimal - * -> any_of(int, decimal) - * - * When [ANY], conflicting type constraints will result in ANY (i.e. no type constraint) - * e.g. type: int - * with type: decimal - * -> any - */ -internal enum class ConflictStrategy { - UNION, - ANY -} - -/** - * Defines the behavior when unifying two different struct constraint lists. - * - * When [INTERSECTION], only the [IonSchemaModel.Field]s that are in both structs are included in the output - * e.g. { a: int, b: string } - * with { a: int, c: decimal } - * -> { a: int } - * - * When [UNION], all the [IonSchemaModel.Field]s that are in the structs are included in the output. Any fields that - * have different constraint lists are unified according to the passed ConstraintUnifier - * e.g. { a: int, b: string } - * with { a: int, c: decimal } - * -> { a: int, b: string, c: decimal } - * - * e.g. { a: int } - * with { a: decimal } - * -> { a: unify(int, decimal) } - * - * (Tentative definition) - * When [INTERSECTION_AS_REQUIRED], all the [IonSchemaModel.Field]s that are in the structs are included in the output, - * but any fields that appear in both are marked as [IonSchemaModel.Optionality.Required]. - */ -internal enum class StructBehavior { - INTERSECTION { - override fun unifyStructs( - unifier: ConstraintUnifier, - structA: IonSchemaModel.ConstraintList, - structB: IonSchemaModel.ConstraintList - ): IonSchemaModel.ConstraintList { - TODO("Not yet implemented") - } - }, - UNION { - override fun unifyStructs( - unifier: ConstraintUnifier, - structA: IonSchemaModel.ConstraintList, - structB: IonSchemaModel.ConstraintList - ): IonSchemaModel.ConstraintList { - if (structA.isEmptyStruct()) { - return structB.addClosedContentConstraint() - } else if (structB.isEmptyStruct()) { - return structA.addClosedContentConstraint() - } - - val aFields = structA.getFieldsConstraint().fields.toSet() - val bFields = structB.getFieldsConstraint() - val unifiedFields = aFields.toMutableSet() - - bFields.fields.forEach { bField -> - val aField = aFields.find { it.name == bField.name } - if (aField != null) { - if (aField != bField) { - val unifiedField = IonSchemaModel.build { - field_(aField.name, inlineType(typeDefinition(null, unifier.unify(aField.type.toConstraintList(), bField.type.toConstraintList())), notNullable)) - } - unifiedFields.remove(aField) - unifiedFields.add(unifiedField) - } - // Otherwise, has same name and value type - } else { - // `structA` doesn't have `structB`'s field - unifiedFields.add(bField) - } - } - return IonSchemaModel.build { - constraintList(structA.getTypeConstraint(), closedContent(), fields(unifiedFields.toList())) - } - } - }, - INTERSECTION_AS_REQUIRED { - override fun unifyStructs( - unifier: ConstraintUnifier, - structA: IonSchemaModel.ConstraintList, - structB: IonSchemaModel.ConstraintList - ): IonSchemaModel.ConstraintList { - TODO("Not yet implemented") - } - }; - - abstract fun unifyStructs( - unifier: ConstraintUnifier, - structA: IonSchemaModel.ConstraintList, - structB: IonSchemaModel.ConstraintList - ): IonSchemaModel.ConstraintList -} - -/** - * Interface for unifying two [IonSchemaModel.ConstraintList]s. A [ConstraintUnifier] is intended to be used by - * [SchemaInferencerFromExample] when either - * 1. unifying the types and/or other constraints of a sequence of examples or - * 2. unifying the [IonSchemaModel.Constraint.Element] type for sequence types. - */ -internal interface ConstraintUnifier { - fun unify(aConstraints: IonSchemaModel.ConstraintList, bConstraints: IonSchemaModel.ConstraintList): IonSchemaModel.ConstraintList - - companion object { - fun builder(): Builder = Builder() - } - - class Builder { - private var sequenceTypes = listOf(TypeConstraint.SEXP.typeName, TypeConstraint.LIST.typeName) - private var conflictStrategy = ConflictStrategy.UNION - private var structBehavior = StructBehavior.UNION - private var discoveredConstraintUnifier: DiscoveredConstraintUnifier = MultipleTypedDCU() - - /** - * Defines the sequence type names (i.e. SEXP, LIST, and other imported types). Defaults to - * [TypeConstraint.SEXP] and [TypeConstraint.LIST]. - */ - fun sequenceTypes(types: List): Builder = this.apply { sequenceTypes = types } - - /** - * Defines the type conflict unification strategy. Defaults to [ConflictStrategy.UNION]. - */ - fun conflictStrategy(strategy: ConflictStrategy): Builder = this.apply { conflictStrategy = strategy } - - /** - * Defines the struct field unification behavior. Defaults to [StructBehavior.UNION]. - */ - fun structBehavior(behavior: StructBehavior): Builder = this.apply { structBehavior = behavior } - - /** - * Defines how additional discovered constraints are to be unified. Defaults to use [MultipleTypedDCU] (which - * defaults to using the [standardTypedDiscoveredConstraintUnifiers]). - */ - fun discoveredConstraintUnifier(unifier: DiscoveredConstraintUnifier): Builder = this.apply { discoveredConstraintUnifier = unifier } - - fun build(): ConstraintUnifier { - return ConstraintUnifierImpl(sequenceTypes, conflictStrategy, structBehavior, discoveredConstraintUnifier) - } - } -} - -private class ConstraintUnifierImpl( - private val sequenceTypes: List, - val conflictStrategy: ConflictStrategy, - val structBehavior: StructBehavior, - val discoveredConstraintUnifier: DiscoveredConstraintUnifier -) : ConstraintUnifier { - /** - * Unifies [aConstraints] with [bConstraints]. - */ - override fun unify(aConstraints: IonSchemaModel.ConstraintList, bConstraints: IonSchemaModel.ConstraintList): IonSchemaModel.ConstraintList { - if (aConstraints == bConstraints) { - return aConstraints - } - - when (conflictStrategy) { - ConflictStrategy.UNION -> when { - hasUnion(aConstraints) && hasUnion(bConstraints) -> { - // `aConstraints` and `bConstraints` are unions - val bTypes = bConstraints.getAnyOfConstraint().types.map { it.toConstraintList() } - return bTypes.fold(aConstraints) { union, bConstraintList -> - unifyUnionWithNonUnion(union = union.getAnyOfConstraint(), nonUnion = bConstraintList) - } - } - hasUnion(aConstraints) -> { - // only `aConstraints` is a union - return unifyUnionWithNonUnion(union = aConstraints.getAnyOfConstraint(), nonUnion = bConstraints) - } - hasUnion(bConstraints) -> { - // only `bConstraints` is a union - return unifyUnionWithNonUnion(union = bConstraints.getAnyOfConstraint(), nonUnion = aConstraints) - } - else -> { - // `aConstraints` and `bConstraints` are not unions - val aTypeName = aConstraints.getTypeConstraint().type.getTypename() - val bTypeName = bConstraints.getTypeConstraint().type.getTypename() - - return if (aTypeName == bTypeName) { - IonSchemaModel.build { unifyNonUnionTypes(aConstraints, bConstraints) } - } else { - // typenames of `aConstraints` and `bConstraints` are different so create union - IonSchemaModel.build { - constraintList(anyOf(aConstraints.toTypeReference(), bConstraints.toTypeReference())) - } - } - } - } - ConflictStrategy.ANY -> TODO("Not yet implemented") - } - } - - /** - * Unifies [sequenceA] sequence constraint list with [sequenceB]. - * - * In this implementation, on constraint conflict of sequence elements, the [IonSchemaModel.Constraint.Element] - * will be unified using [unify]. Thus, the resulting unification can result in heterogeneous elements in the - * sequence (e.g. type: list, element: { any_of { int, decimal }). - * - * TODO: decide if this should be a configurable - */ - private fun unifySequences(sequenceA: IonSchemaModel.ConstraintList, sequenceB: IonSchemaModel.ConstraintList): IonSchemaModel.ConstraintList { - // items.size == 1 -> no element type so is empty sequence - if (sequenceA.items.size == 1) { - return sequenceB - } else if (sequenceB.items.size == 1) { - return sequenceA - } - - // Type is either a sexp, list, or other sequence type - val aTypeName = sequenceA.getTypeConstraint().type.getTypename() - - val aElement = sequenceA.getElementConstraint().type as IonSchemaModel.TypeReference.InlineType - val bElement = sequenceB.getElementConstraint().type as IonSchemaModel.TypeReference.InlineType - - val elementTypeConstraints = unify(aElement.type.constraints, bElement.type.constraints) - if (elementTypeConstraints.isAny()) { - return IonSchemaModel.build { - constraintList(typeConstraint(namedType(aTypeName, notNullable))) - } - } - - return IonSchemaModel.build { - constraintList( - typeConstraint(namedType(aTypeName, notNullable)), - element(inlineType(typeDefinition(name = null, constraints = elementTypeConstraints), notNullable)) - ) - } - } - - /** - * Unifies [union] (union of types), with [nonUnion] (a single type). If [nonUnion]'s type is not in [union], then - * adds [nonUnion] to [union]. Otherwise, unify [nonUnion] with it's corresponding type in [union]. - */ - private fun unifyUnionWithNonUnion(union: IonSchemaModel.Constraint.AnyOf, nonUnion: IonSchemaModel.ConstraintList): IonSchemaModel.ConstraintList { - val anyOfConstraintList = union.types.toMutableList() - - val nonUnionType = nonUnion.getTypeConstraint().type - val nonUnionTypeName = nonUnionType.getTypename() - - val matchingTypeIndex = anyOfConstraintList.indexOfFirst { - it.getTypename() == nonUnionTypeName - } - - when (matchingTypeIndex) { - -1 -> { - // no matching typename, so append to union - anyOfConstraintList.add(nonUnion.toTypeReference()) - } - else -> { - // there's a matching typename - val matchingType = anyOfConstraintList[matchingTypeIndex] - val matchingTypeConstraints = matchingType.toConstraintList() - - if (matchingTypeConstraints != nonUnion) { - // typenames equal but need to resolve other constraint conflicts - anyOfConstraintList[matchingTypeIndex] = unifyNonUnionTypes(matchingTypeConstraints, nonUnion).toTypeReference() - } - // else constraint already in union - } - } - return IonSchemaModel.build { constraintList(anyOf(anyOfConstraintList)) } - } - - /** - * Unifies [a] and [b] non-union types. Requires - * 1. [a] != [b] - * 2. [a] and [b] have the same typename - */ - private fun unifyNonUnionTypes(a: IonSchemaModel.ConstraintList, b: IonSchemaModel.ConstraintList): IonSchemaModel.ConstraintList { - val constraints = when { - a.isScalarType() && b.isScalarType() -> listOf(a.getTypeConstraint()) - a.isStructType() && b.isStructType() -> structBehavior.unifyStructs(this@ConstraintUnifierImpl, a, b).items - a.isSequenceType() && b.isSequenceType() -> unifySequences(a, b).items - else -> error("$a and $b typenames do not match") - } - val discoveredConstraints = discoveredConstraintUnifier(a, b).items - return IonSchemaModel.build { constraintList(constraints + discoveredConstraints) } - } - - /** - * Returns true if and only if [this] [IonSchemaModel.ConstraintList]'s type constraint is a sequence type. - * Sequence types include `list`, `sexp`, and any imported sequence types. - */ - private fun IonSchemaModel.ConstraintList.isSequenceType(): Boolean { - val constraintType = this.getTypeConstraint() - val name = constraintType.type.getTypename() - return sequenceTypes.contains(name) - } - - /** - * Returns true if and only if [this] [IonSchemaModel.ConstraintList]'s type constraint is struct. - */ - private fun IonSchemaModel.ConstraintList.isStructType(): Boolean { - val constraintType = this.getTypeConstraint() - val name = constraintType.type.getTypename() - return name == TypeConstraint.STRUCT.typeName - } - - /** - * Returns true if and only if [this] [IonSchemaModel.ConstraintList]'s type constraint is not one of sexp, list, - * defined sequence types, or struct. - */ - private fun IonSchemaModel.ConstraintList.isScalarType(): Boolean { - return !this.isSequenceType() && !this.isStructType() - } -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintUtils.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintUtils.kt deleted file mode 100644 index 30ca804cb..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ConstraintUtils.kt +++ /dev/null @@ -1,216 +0,0 @@ -package org.partiql.ionschema.discovery - -import com.amazon.ionelement.api.ionBool -import com.amazon.ionelement.api.ionInt -import org.partiql.ionschema.model.IonSchemaModel -import kotlin.math.max -import kotlin.math.min - -internal val emptyConstraintList = IonSchemaModel.build { constraintList() } - -private fun IonSchemaModel.ConstraintList.getConstraint(constraint: Class): IonSchemaModel.Constraint = - this.items.find { it.javaClass == constraint } - ?: throw IllegalStateException("Given constraint list $this does not have any $constraint") - -/** - * Returns the first [IonSchemaModel.Constraint.TypeConstraint] in [this] constraint list. Throws an - * [IllegalStateException] if [this] does not contain any [IonSchemaModel.Constraint.TypeConstraint] constraints. - */ -internal fun IonSchemaModel.ConstraintList.getTypeConstraint(): IonSchemaModel.Constraint.TypeConstraint = - this.getConstraint(IonSchemaModel.Constraint.TypeConstraint::class.java) as IonSchemaModel.Constraint.TypeConstraint - -/** - * Returns the first [IonSchemaModel.Constraint.AnyOf] in [this] constraint list. Throws an - * [IllegalStateException] if [this] does not contain any [IonSchemaModel.Constraint.AnyOf] constraints. - */ -internal fun IonSchemaModel.ConstraintList.getAnyOfConstraint(): IonSchemaModel.Constraint.AnyOf = - this.getConstraint(IonSchemaModel.Constraint.AnyOf::class.java) as IonSchemaModel.Constraint.AnyOf - -/** - * Returns the first [IonSchemaModel.Constraint.Element] in [this] constraint list. Throws an - * [IllegalStateException] if [this] does not contain any [IonSchemaModel.Constraint.Element] constraints. - */ -internal fun IonSchemaModel.ConstraintList.getElementConstraint(): IonSchemaModel.Constraint.Element = - this.getConstraint(IonSchemaModel.Constraint.Element::class.java) as IonSchemaModel.Constraint.Element - -/** - * Returns the first [IonSchemaModel.Constraint.Fields] in [this] constraint list. Throws an - * [IllegalStateException] if [this] does not contain any [IonSchemaModel.Constraint.Fields] constraints. - */ -internal fun IonSchemaModel.ConstraintList.getFieldsConstraint(): IonSchemaModel.Constraint.Fields = - this.getConstraint(IonSchemaModel.Constraint.Fields::class.java) as IonSchemaModel.Constraint.Fields - -/** - * Returns the first [IonSchemaModel.Constraint.ValidValues] in [this] constraint list. Throws an - * [IllegalStateException] if [this] does not contain any [IonSchemaModel.Constraint.ValidValues] constraints. - */ -internal fun IonSchemaModel.ConstraintList.getValidValuesConstraint(): IonSchemaModel.Constraint.ValidValues = - this.getConstraint(IonSchemaModel.Constraint.ValidValues::class.java) as IonSchemaModel.Constraint.ValidValues - -/** - * Returns the first [IonSchemaModel.Constraint.Scale] in [this] constraint list. Throws an - * [IllegalStateException] if [this] does not contain any [IonSchemaModel.Constraint.Scale] constraints. - */ -internal fun IonSchemaModel.ConstraintList.getScaleConstraint(): IonSchemaModel.Constraint.Scale = - this.getConstraint(IonSchemaModel.Constraint.Scale::class.java) as IonSchemaModel.Constraint.Scale - -/** - * Returns the first [IonSchemaModel.Constraint.Precision] in [this] constraint list. Throws an - * [IllegalStateException] if [this] does not contain any [IonSchemaModel.Constraint.Precision] constraints. - */ -internal fun IonSchemaModel.ConstraintList.getPrecisionConstraint(): IonSchemaModel.Constraint.Precision = - this.getConstraint(IonSchemaModel.Constraint.Precision::class.java) as IonSchemaModel.Constraint.Precision - -/** - * Returns the first [IonSchemaModel.Constraint.CodepointLength] in [this] constraint list. Throws an - * [IllegalStateException] if [this] does not contain any [IonSchemaModel.Constraint.CodepointLength] constraints. - */ -internal fun IonSchemaModel.ConstraintList.getCodepointLengthConstraint(): IonSchemaModel.Constraint.CodepointLength = - this.getConstraint(IonSchemaModel.Constraint.CodepointLength::class.java) as IonSchemaModel.Constraint.CodepointLength - -/** Returns true if and only if [this] constraint list contains a constraint of type [constraintType] */ -internal fun IonSchemaModel.ConstraintList.containsConstraint(constraintType: Class): Boolean = - this.items.any { it.javaClass == constraintType } - -/** - * Returns true if and only if [this] constraint list has no constraints. - */ -internal fun IonSchemaModel.ConstraintList.isAny(): Boolean = this.items.isEmpty() - -/** - * Returns the first typename of [this] [IonSchemaModel.TypeReference]. [this] must be either an - * [IonSchemaModel.TypeReference.NamedType] or an [IonSchemaModel.TypeReference.InlineType]. - */ -internal fun IonSchemaModel.TypeReference.getTypename(): String = - when (this) { - is IonSchemaModel.TypeReference.InlineType -> this.firstNamedType().name.text - is IonSchemaModel.TypeReference.NamedType -> this.name.text - else -> error("Only InlineType and NamedType are supported") - } - -/** - * Returns the first named type in [this] [IonSchemaModel.TypeReference.InlineType]'s constraints. - */ -private fun IonSchemaModel.TypeReference.InlineType.firstNamedType(): IonSchemaModel.TypeReference.NamedType = - this.type.constraints.getTypeConstraint().type as IonSchemaModel.TypeReference.NamedType - -/** - * Returns [this] [IonSchemaModel.TypeReference] as an [IonSchemaModel.ConstraintList]. - */ -internal fun IonSchemaModel.TypeReference.toConstraintList(): IonSchemaModel.ConstraintList = - when (this) { - is IonSchemaModel.TypeReference.InlineType -> this.type.constraints - is IonSchemaModel.TypeReference.NamedType -> IonSchemaModel.build { constraintList(typeConstraint(this@toConstraintList)) } - else -> error("Only InlineType and NamedType are supported") - } - -/** - * Returns [this] [IonSchemaModel.ConstraintList] as an [IonSchemaModel.TypeReference]. - */ -internal fun IonSchemaModel.ConstraintList.toTypeReference(): IonSchemaModel.TypeReference { - val thisTypeConstraint = this.getTypeConstraint() - return when (this.items.size) { - 1 -> thisTypeConstraint.type - else -> { - IonSchemaModel.build { - inlineType(typeDefinition(constraints = this@toTypeReference), ionBool(false)) - } - } - } -} - -/** - * Returns true if and only if `this` struct is empty (i.e. has no [IonSchemaModel.Constraint.Fields] constraint). - */ -internal fun IonSchemaModel.ConstraintList.isEmptyStruct(): Boolean = - !this.containsConstraint(IonSchemaModel.Constraint.Fields::class.java) - -/** - * Returns true if and only if [constraintList] contains the [IonSchemaModel.Constraint.AnyOf] constraint. - */ -internal fun hasUnion(constraintList: IonSchemaModel.ConstraintList): Boolean = - constraintList.containsConstraint(IonSchemaModel.Constraint.AnyOf::class.java) - -/** - * Returns the first [IonSchemaModel.SchemaStatement.TypeStatement] from [this] [IonSchemaModel]'s statements. Throws - * an [IllegalStateException] if [this] has no [IonSchemaModel.SchemaStatement.TypeStatement]. - */ -internal fun IonSchemaModel.Schema.getFirstTypeStatement(): IonSchemaModel.SchemaStatement.TypeStatement { - val statements = this.statements - val typeStatement = statements.find { it is IonSchemaModel.SchemaStatement.TypeStatement } - ?: throw IllegalStateException("Given schema $this has no TypeStatement") - return typeStatement as IonSchemaModel.SchemaStatement.TypeStatement -} - -/** - * Unifies [this] sequence of [IonSchemaModel.ConstraintList]s to a unified [IonSchemaModel.ConstraintList] using - * [unifier]. - */ -internal fun Sequence.unifiedConstraintList(unifier: ConstraintUnifier): IonSchemaModel.ConstraintList { - return this.reduce { acc, typeConstraint -> - unifier.unify(acc, typeConstraint) - } -} - -/** - * Unifies the two [IonSchemaModel.NumberRule]s. If both number rules are equivalent and are - * [IonSchemaModel.NumberRule.EqualsNumber], returns [numberRuleA] as an [IonSchemaModel.NumberRule.EqualsNumber]. - * Otherwise, returns an [IonSchemaModel.NumberRule.EqualsRange] of the combined number rules. - */ -internal fun unifyNumberRuleConstraints(numberRuleA: IonSchemaModel.NumberRule, numberRuleB: IonSchemaModel.NumberRule): IonSchemaModel.NumberRule { - val aMin: Long - val aMax: Long - val bMin: Long - val bMax: Long - - when (numberRuleA) { - is IonSchemaModel.NumberRule.EqualsNumber -> { - aMin = numberRuleA.value.longValue - aMax = numberRuleA.value.longValue - } - is IonSchemaModel.NumberRule.EqualsRange -> { - aMin = (numberRuleA.range.min as IonSchemaModel.NumberExtent.Inclusive).value.longValue - aMax = (numberRuleA.range.max as IonSchemaModel.NumberExtent.Inclusive).value.longValue - } - } - when (numberRuleB) { - is IonSchemaModel.NumberRule.EqualsNumber -> { - bMin = numberRuleB.value.longValue - bMax = numberRuleB.value.longValue - } - is IonSchemaModel.NumberRule.EqualsRange -> { - bMin = (numberRuleB.range.min as IonSchemaModel.NumberExtent.Inclusive).value.longValue - bMax = (numberRuleB.range.max as IonSchemaModel.NumberExtent.Inclusive).value.longValue - } - } - - val newMin = min(aMin, bMin) - val newMax = max(aMax, bMax) - - return if (newMin == newMax) { - numberRuleA - } else { - IonSchemaModel.build { equalsRange(numberRange(inclusive(ionInt(newMin)), inclusive(ionInt(newMax)))) } - } -} - -/** - * Returns [this] [IonSchemaModel.ConstraintList] with the [IonSchemaModel.Constraint.ClosedContent] constraint added. - */ -internal fun IonSchemaModel.ConstraintList.addClosedContentConstraint(): IonSchemaModel.ConstraintList = - when (this.containsConstraint(IonSchemaModel.Constraint.ClosedContent::class.java)) { - true -> this - else -> { - val constraints = this.items.toMutableList() - constraints.add(IonSchemaModel.build { closedContent() }) - IonSchemaModel.build { - constraintList(constraints) - } - } - } - -/** - * Returns a [IonSchemaModel.Constraint.TypeConstraint] with [typeName] as a non-null named type. - */ -internal fun typeConstraintOf(typeName: String): IonSchemaModel.Constraint.TypeConstraint = - IonSchemaModel.build { typeConstraint(namedType(name = typeName, nullable = ionBool(false))) } diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/DiscoveredConstraintUnifier.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/DiscoveredConstraintUnifier.kt deleted file mode 100644 index 0f4b5cb94..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/DiscoveredConstraintUnifier.kt +++ /dev/null @@ -1,158 +0,0 @@ -package org.partiql.ionschema.discovery - -import org.partiql.ionschema.model.IonSchemaModel - -/** - * For two conflicting [IonSchemaModel.ConstraintList]s with the same type constraint, unifies the constraint - * lists' additional discovered constraints (i.e. not one of: - * - [IonSchemaModel.Constraint.TypeConstraint] - * - [IonSchemaModel.Constraint.Fields] for structs - * - [IonSchemaModel.Constraint.ClosedContent] for structs - * - [IonSchemaModel.Constraint.Element] for sequences). - * - * This is intended to be called by a [ConstraintUnifier] when unifying - * - discovered constraints only ([MultipleTypedDCU]) - * - discovered with definite constraints ([AppendAdditionalConstraints]) - */ -internal fun interface DiscoveredConstraintUnifier { - operator fun invoke(a: IonSchemaModel.ConstraintList, b: IonSchemaModel.ConstraintList): IonSchemaModel.ConstraintList -} - -/** - * Represents a [DiscoveredConstraintUnifier] where each [IonSchemaModel.ConstraintList] to unify has a - * [IonSchemaModel.Constraint.TypeConstraint] with [typeName]. This is intended to be used when creating - * [MultipleTypedDCU]. - */ -internal data class SingleTypedDCU(val typeName: String, val unifyFunc: DiscoveredConstraintUnifier) - -/** - * For two conflicting constraint lists, `a` and `b`, unifies discovered constraints based on [constraintUnifiers]. - * If `a`/`b`'s type name matches one of the [constraintUnifiers]' [SingleTypedDCU.typeName]s, then `a` and `b` are - * unified with that corresponding unifier. Otherwise, an empty constraint list is returned. - * - * @exception IllegalArgumentException if any of [constraintUnifiers] have the same - * [SingleTypedDCU.typeName]. - */ -internal class MultipleTypedDCU( - private val constraintUnifiers: List = standardTypedDiscoveredConstraintUnifiers -) : DiscoveredConstraintUnifier { - private val discoveredConstraintUnifierMapping = initializeMapping() - - private fun initializeMapping(): Map { - val mapping = mutableMapOf() - constraintUnifiers.forEach { - if (mapping.containsKey(it.typeName)) { - throw IllegalArgumentException("${it.typeName} is a repeated type name for MultipleTypedDCU") - } - mapping[it.typeName] = it.unifyFunc - } - return mapping - } - - override fun invoke(a: IonSchemaModel.ConstraintList, b: IonSchemaModel.ConstraintList): IonSchemaModel.ConstraintList { - val typeName = a.getTypeConstraint().type.getTypename() - return when (val unifier = discoveredConstraintUnifierMapping[typeName]) { - null -> emptyConstraintList - else -> unifier(a, b) - } - } -} - -/** - * For two conflicting constraint lists, `a` and `b`, appends `b`'s constraints not found in `a`. Any constraints that - * are found in `a` and `b` will return `a`'s constraint. - */ -internal class AppendAdditionalConstraints : DiscoveredConstraintUnifier { - private fun IonSchemaModel.Constraint.isDiscoveredConstraint(): Boolean { - return this !is IonSchemaModel.Constraint.TypeConstraint && - this !is IonSchemaModel.Constraint.ClosedContent && - this !is IonSchemaModel.Constraint.Fields && - this !is IonSchemaModel.Constraint.Element - } - - override fun invoke(a: IonSchemaModel.ConstraintList, b: IonSchemaModel.ConstraintList): IonSchemaModel.ConstraintList { - val constraints = mutableListOf() - - val aConstraints = a.items.filter { it.isDiscoveredConstraint() } - val bConstraints = b.items.filter { it.isDiscoveredConstraint() } - - // keep all of `a`'s discovered constraints - constraints.addAll(aConstraints) - // add `b`'s constraints that are not in `a` - bConstraints.forEach { bConstraint -> - if (constraints.all { it.javaClass != bConstraint.javaClass }) { - constraints.add(bConstraint) - } - } - return IonSchemaModel.build { constraintList(constraints) } - } -} - -/** - * For [TypeConstraint.INT], merges the [IonSchemaModel.Constraint.ValidValues] ranges. If either do not have the - * valid_values constraint, an empty constraint list is returned. - */ -internal val INT_VALID_VALUES_UNIFIER = SingleTypedDCU(TypeConstraint.INT.typeName) { a, b -> - val constraintList = mutableListOf() - - val aHasValidValuesConstraint = a.containsConstraint(IonSchemaModel.Constraint.ValidValues::class.java) - val bHasValidValuesConstraint = b.containsConstraint(IonSchemaModel.Constraint.ValidValues::class.java) - - // constraints are both not unconstrained - if (aHasValidValuesConstraint && bHasValidValuesConstraint) { - val aValidValuesConstraint = a.getValidValuesConstraint() - val bValidValuesConstraint = b.getValidValuesConstraint() - - if (aValidValuesConstraint == INT8_RANGE_CONSTRAINT || bValidValuesConstraint == INT8_RANGE_CONSTRAINT) { - constraintList.add(INT8_RANGE_CONSTRAINT) - } - // else, constraints differ, so one must be int2 and the other int4. Thus, int4 is returned - else { - constraintList.add(INT4_RANGE_CONSTRAINT) - } - } - IonSchemaModel.build { constraintList(constraintList) } -} - -/** - * For [TypeConstraint.DECIMAL], merges the [IonSchemaModel.Constraint.Scale] and [IonSchemaModel.Constraint.Precision] - * ranges. - * - * @exception IllegalStateException if either of the constraint lists do not have scale or precision constraints. - */ -internal val DECIMAL_SCALE_AND_PRECISION_UNIFIER = SingleTypedDCU(TypeConstraint.DECIMAL.typeName) { a, b -> - val constraintList = mutableListOf() - - val aScale = a.getScaleConstraint().rule - val bScale = b.getScaleConstraint().rule - constraintList.add(IonSchemaModel.build { scale(unifyNumberRuleConstraints(aScale, bScale)) }) - - val aPrecision = a.getPrecisionConstraint().rule - val bPrecision = b.getPrecisionConstraint().rule - constraintList.add(IonSchemaModel.build { precision(unifyNumberRuleConstraints(aPrecision, bPrecision)) }) - IonSchemaModel.build { constraintList(constraintList) } -} - -/** - * For [TypeConstraint.STRING], merges the [IonSchemaModel.Constraint.CodepointLength] ranges. - * - * @exception IllegalStateException if either of the constraint lists do not have the codepoint_length constraint. - */ -internal val STRING_CODEPOINT_LENGTH_UNIFIER = SingleTypedDCU(TypeConstraint.STRING.typeName) { a, b -> - val aLength = a.getCodepointLengthConstraint().rule - val bLength = b.getCodepointLengthConstraint().rule - IonSchemaModel.build { constraintList(codepointLength(unifyNumberRuleConstraints(aLength, bLength))) } -} - -/** - * List of [SingleTypedDCU]s, composed of: - * [INT_VALID_VALUES_UNIFIER]- unifies INT's valid_values constraint, - * [DECIMAL_SCALE_AND_PRECISION_UNIFIER]- unifies DECIMAL's scale and precision constraints, - * [STRING_CODEPOINT_LENGTH_UNIFIER]- unifies STRING's codepoint_length constraint - */ -internal val standardTypedDiscoveredConstraintUnifiers = - listOf( - INT_VALID_VALUES_UNIFIER, - DECIMAL_SCALE_AND_PRECISION_UNIFIER, - STRING_CODEPOINT_LENGTH_UNIFIER - ) diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/IonExampleParser.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/IonExampleParser.kt deleted file mode 100644 index b9aaf5bb3..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/IonExampleParser.kt +++ /dev/null @@ -1,18 +0,0 @@ -package org.partiql.ionschema.discovery - -import com.amazon.ion.IonReader -import com.amazon.ion.IonSystem -import com.amazon.ion.IonValue - -/** - * Basic parser for ion data. - */ -class IonExampleParser(val ion: IonSystem) { - /** - * Returns the next [IonValue] or null if there are no move values to read. - */ - fun parseExample(reader: IonReader): IonValue? { - reader.next() ?: return null - return ion.newValue(reader) - } -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/NormalizeDecimalPrecisionsToUpToRange.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/NormalizeDecimalPrecisionsToUpToRange.kt deleted file mode 100644 index 8e5a4b1f3..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/NormalizeDecimalPrecisionsToUpToRange.kt +++ /dev/null @@ -1,31 +0,0 @@ -package org.partiql.ionschema.discovery - -import com.amazon.ionelement.api.ionInt -import org.partiql.ionschema.model.IonSchemaModel - -/** - * Normalizes decimal precisions ([IonSchemaModel.Constraint.Precision]) to an "upto" range. For exact precision p, - * returns an inclusive range from 1 to p. For a ranged precision with an inclusive max, returns an inclusive range - * from 1 to max. - * - * Because the dataguide's default [ConstraintDiscoverer] for decimals infers exact precisions and ranges, some use - * cases would just like to infer a range from 1 (inclusive) to the max, inclusive precision (i.e. "upto" inclusive - * range). This [IonSchemaModel.VisitorTransform] normalizes such decimal precisions to an inclusive "upto" range. - */ -class NormalizeDecimalPrecisionsToUpToRange : IonSchemaModel.VisitorTransform() { - override fun transformConstraintPrecision(node: IonSchemaModel.Constraint.Precision): IonSchemaModel.Constraint { - val transformedPrecisionRule = when (val nodeNumberRule = node.rule) { - is IonSchemaModel.NumberRule.EqualsNumber -> IonSchemaModel.build { - equalsRange(numberRange(inclusive(ionInt(1)), inclusive(nodeNumberRule.value))) - } - is IonSchemaModel.NumberRule.EqualsRange -> IonSchemaModel.build { - val maxValue = when (val nodeNumberMax = nodeNumberRule.range.max) { - is IonSchemaModel.NumberExtent.Inclusive -> nodeNumberMax.value - else -> error("Unsupported number range for normalization") - } - equalsRange(numberRange(inclusive(ionInt(1)), inclusive(maxValue))) - } - } - return IonSchemaModel.build { precision(transformedPrecisionRule) } - } -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/NormalizeNullableVisitorTransform.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/NormalizeNullableVisitorTransform.kt deleted file mode 100644 index 9e01e5ccf..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/NormalizeNullableVisitorTransform.kt +++ /dev/null @@ -1,76 +0,0 @@ -package org.partiql.ionschema.discovery - -import com.amazon.ionelement.api.ionBool -import org.partiql.ionschema.model.IonSchemaModel - -/** - * This VisitorTransform normalizes [IonSchemaModel.Constraint.AnyOf] constraints that have the null type to use - * the `nullable` annotation. E.g. - * - * any_of(null, symbol, bool) -> any_of(nullable:: symbol, nullable::bool) - * any_of(null, symbol) -> nullable::symbol - */ -class NormalizeNullableVisitorTransform : IonSchemaModel.VisitorTransform() { - private val nullNamedType = IonSchemaModel.build { namedType("\$null", ionBool(true)) } - - /** - * Returns whether [this] [IonSchemaModel.TypeReference] is nullable. - */ - private fun IonSchemaModel.TypeReference.isNullable(): Boolean = - when (this) { - is IonSchemaModel.TypeReference.InlineType -> this.nullable.booleanValue - is IonSchemaModel.TypeReference.NamedType -> this.nullable.booleanValue - is IonSchemaModel.TypeReference.ImportedType -> this.nullable.booleanValue - } - - /** - * Returns [this] [IonSchemaModel.TypeReference]'s first [IonSchemaModel.TypeReference.NamedType] as nullable. - */ - private fun IonSchemaModel.TypeReference.toNullable(): IonSchemaModel.TypeReference { - val thisTypeRef = this - return IonSchemaModel.build { - when (thisTypeRef) { - is IonSchemaModel.TypeReference.NamedType -> namedType(thisTypeRef.getTypename(), ionBool(true)) - is IonSchemaModel.TypeReference.InlineType -> - inlineType(typeDefinition(thisTypeRef.type.name?.text, thisTypeRef.type.constraints.toNullable()), ionBool(false)) - else -> error("Only InlineType and NamedType are supported") - } - } - } - - /** - * Returns [this] [IonSchemaModel.ConstraintList] with its [IonSchemaModel.Constraint.TypeConstraint] as a - * nullable [IonSchemaModel.TypeReference]. - */ - private fun IonSchemaModel.ConstraintList.toNullable(): IonSchemaModel.ConstraintList { - val thisTypeConstraint = this.getTypeConstraint() - val nonTypeConstraints: List = this.items.filter { it !is IonSchemaModel.Constraint.TypeConstraint } - val nullableType = thisTypeConstraint.type.toNullable() - - val allConstraints = listOf(IonSchemaModel.build { typeConstraint(nullableType) }) + nonTypeConstraints - return IonSchemaModel.build { constraintList(allConstraints) } - } - - /** - * Transforms the given [IonSchemaModel.ConstraintList], [node]. If [node] has an `any_of` constraint which - * contains the null type, returns `any_of` with the `nullable` annotation for every other type. If [node]'s - * `any_of` constraint has just one type T and null, returns the nullable form of T. - */ - override fun transformConstraintList_items(node: IonSchemaModel.ConstraintList): List { - if (hasUnion(node)) { - val anyOfTypes = node.getAnyOfConstraint().types - if (anyOfTypes.any { it.isNullable() }) { - val newAnyOf = anyOfTypes.filter { it != nullNamedType } - .map { transformTypeReference(it.toNullable()) } - - return if (newAnyOf.size == 1) { - newAnyOf.first().toConstraintList().items - } else { - listOf(IonSchemaModel.build { anyOf(newAnyOf) }) - } - } - // else no null type in `any_of` - } - return super.transformConstraintList_items(node) - } -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ResourceAuthority.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ResourceAuthority.kt deleted file mode 100644 index 582fe8782..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/ResourceAuthority.kt +++ /dev/null @@ -1,39 +0,0 @@ -package org.partiql.ionschema.discovery - -import com.amazon.ion.IonSystem -import com.amazon.ion.IonValue -import com.amazon.ionschema.Authority -import com.amazon.ionschema.IonSchemaSystem -import com.amazon.ionschema.util.CloseableIterator -import java.io.InputStream - -class ResourceAuthority( - val rootPackage: String, - val classLoader: ClassLoader, - val ion: IonSystem -) : Authority { - override fun iteratorFor(iss: IonSchemaSystem, id: String): CloseableIterator { - val resourceName = "$rootPackage/$id" - var str: InputStream? = classLoader.getResourceAsStream(resourceName) - ?: error("Failed to load schema with resource name '$resourceName'") - - return object : CloseableIterator { - - private var stream = str - private var reader = ion.newReader(stream).also { it.next() } - private var iter = ion.iterate(reader) - - override fun hasNext() = iter.hasNext() - override fun next() = iter.next() - override fun close() { - try { - reader?.close() - stream?.close() - } finally { - reader = null - stream = null - } - } - } - } -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/SchemaInferencerFromExample.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/SchemaInferencerFromExample.kt deleted file mode 100644 index da472397c..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/SchemaInferencerFromExample.kt +++ /dev/null @@ -1,16 +0,0 @@ -package org.partiql.ionschema.discovery - -import com.amazon.ion.IonReader -import org.partiql.ionschema.model.IonSchemaModel - -/** - * Infers a basic schema from a sequence of example data. - */ -interface SchemaInferencerFromExample { - /** - * Infers an [IonSchemaModel.Schema] from an [IonReader] using [maxExampleCount] examples. - * - * If a non-null [definiteISL] is provided, the discovered schema will also be unified with the definite schema. - */ - fun inferFromExamples(reader: IonReader, maxExampleCount: Int, definiteISL: IonSchemaModel.Schema? = null): IonSchemaModel.Schema -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/SchemaInferencerFromExampleImpl.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/SchemaInferencerFromExampleImpl.kt deleted file mode 100644 index 8626f5146..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/SchemaInferencerFromExampleImpl.kt +++ /dev/null @@ -1,123 +0,0 @@ -package org.partiql.ionschema.discovery - -import com.amazon.ion.IonReader -import com.amazon.ion.IonSequence -import com.amazon.ion.IonStruct -import com.amazon.ion.IonText -import com.amazon.ion.IonValue -import com.amazon.ion.system.IonSystemBuilder -import com.amazon.ionschema.IonSchemaSystem -import com.amazon.ionschema.Type -import org.partiql.ionschema.model.IonSchemaModel - -/** - * Implementation for [SchemaInferencerFromExample]. Requires a [typeName] for the generated schema's top level type - * name. Also requires an [IonSchemaSystem] and [schemaIds] to load additional schema types that will be used in the - * generated schema. The passed [schemaIds] will also be used for the generated - * [IonSchemaModel.SchemaStatement.HeaderStatement]'s [IonSchemaModel.ImportList]. - */ -class SchemaInferencerFromExampleImpl(val typeName: String, iss: IonSchemaSystem, val schemaIds: List) : - SchemaInferencerFromExample { - private val importedTypes = schemaIds.loadImportedTypes(iss) - private val sequenceTypes = importedTypes.loadSequenceTypes() - private val islAnyConstraints = IonSchemaModel.build { constraintList() } - private val islAnySchema = IonSchemaModel.build { - schema( - headerStatement(openFieldList(), importList(schemaIds.map { import(it) })), - typeStatement(typeDefinition(typeName, islAnyConstraints)), - footerStatement(openFieldList()) - ) - } - - override fun inferFromExamples(reader: IonReader, maxExampleCount: Int, definiteISL: IonSchemaModel.Schema?): IonSchemaModel.Schema { - val parser = IonExampleParser(IonSystemBuilder.standard().build()) - if (maxExampleCount < 1) { - return islAnySchema - } - - val firstExample = parser.parseExample(reader) ?: return islAnySchema - val examples = mutableListOf(firstExample) - - var example = parser.parseExample(reader) - var numExamplesLeft = maxExampleCount - 1 - while (example != null && numExamplesLeft > 0) { - examples.add(example) - example = parser.parseExample(reader) - numExamplesLeft-- - } - - val dataguideConstraintUnifier = ConstraintUnifier.builder() - .sequenceTypes(sequenceTypes) - .discoveredConstraintUnifier(MultipleTypedDCU(constraintUnifiers = standardTypedDiscoveredConstraintUnifiers)) - .build() - - val dataguideInferer = TypeAndConstraintInferer( - constraintUnifier = dataguideConstraintUnifier, - constraintDiscoverer = StandardConstraintDiscoverer(), - importedTypes = importedTypes - ) - - val discoveredWithDefiniteUnifier = ConstraintUnifier.builder() - .sequenceTypes(sequenceTypes) - .discoveredConstraintUnifier(AppendAdditionalConstraints()) - .build() - - val unifiedTypeConstraint = examples.asSequence() - .map { dataguideInferer.inferConstraints(it) } - .unifiedConstraintList(dataguideInferer.constraintUnifier) - .let { NormalizeNullableVisitorTransform().transformConstraintList(it) } - .let { discoveredConstraints -> - when (definiteISL) { - null -> discoveredConstraints - else -> { - val definiteSchemaTypeStatement = definiteISL.getFirstTypeStatement() - val definiteISLTopTypeName = definiteSchemaTypeStatement.typeDef.name?.text - if (typeName != definiteISLTopTypeName) { - error( - """Top level type name differs. - Expected: $typeName - Actual: $definiteISLTopTypeName - """.trimIndent() - ) - } - discoveredWithDefiniteUnifier.unify(discoveredConstraints, definiteSchemaTypeStatement.typeDef.constraints) - } - } - } - - return IonSchemaModel.build { - schema( - headerStatement(openFieldList(), importList(schemaIds.map { import(it) })), - typeStatement(typeDefinition(name = typeName, constraints = unifiedTypeConstraint)), - footerStatement(openFieldList()) - ) - } - } - - /** - * Returns a list of all the [Type]s in [this] list of schema identifiers. - */ - private fun List.loadImportedTypes(iss: IonSchemaSystem): List { - val schemas = this.map { schemaId -> iss.loadSchema(schemaId) } - return schemas.flatMap { schema -> schema.getTypes().asSequence().toList() } - } - - /** - * Returns the names of all the [IonSequence]s (i.e. list, sexp) and imported sequence types. - */ - private fun List.loadSequenceTypes(): List { - return this.fold(mutableListOf(TypeConstraint.LIST.typeName, TypeConstraint.SEXP.typeName)) { acc, t -> - val typeAsStruct = t.isl as IonStruct - val definedType = typeAsStruct.get("type").stringValueOrNull() - if (definedType == TypeConstraint.LIST.typeName || definedType == TypeConstraint.SEXP.typeName) { - acc.add(t.name) - } - acc - } - } - - private fun IonValue.stringValueOrNull(): String? = when (this) { - is IonText -> stringValue() - else -> null - } -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/TypeConstraint.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/TypeConstraint.kt deleted file mode 100644 index df702521e..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/discovery/TypeConstraint.kt +++ /dev/null @@ -1,27 +0,0 @@ -package org.partiql.ionschema.discovery - -import org.partiql.ionschema.model.IonSchemaModel - -/** - * Enum representing the core [IonSchemaModel.TypeReference.NamedType]s along with their corresponding [typeName]s. - */ -enum class TypeConstraint(val typeName: String) { - // scalar types - BOOL("bool"), - INT("int"), - FLOAT("float"), - DECIMAL("decimal"), - TIMESTAMP("timestamp"), - SYMBOL("symbol"), - STRING("string"), - CLOB("clob"), - BLOB("blob"), - NULL("\$null"), - - // sequence types - SEXP("sexp"), - LIST("list"), - - // struct type - STRUCT("struct") -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/model/ToIsl.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/model/ToIsl.kt deleted file mode 100644 index ab0431d93..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/model/ToIsl.kt +++ /dev/null @@ -1,246 +0,0 @@ -package org.partiql.ionschema.model - -import com.amazon.ionelement.api.AnyElement -import com.amazon.ionelement.api.IonElement -import com.amazon.ionelement.api.StructElement -import com.amazon.ionelement.api.StructField -import com.amazon.ionelement.api.field -import com.amazon.ionelement.api.ionListOf -import com.amazon.ionelement.api.ionString -import com.amazon.ionelement.api.ionStructOf -import com.amazon.ionelement.api.ionSymbol - -private val MAX = ionSymbol("max") -private val MIN = ionSymbol("min") - -/** - * Transforms a PIG-generated [IonSchemaModel.Schema] into an [IonElement] representation of an ISL document. - * - * @receiver [IonSchemaModel.Schema] to be transformed to an ISL document - * @return transformed ISL document represented as a List<[AnyElement]> - */ -fun IonSchemaModel.Schema.toIsl(): List = - this.statements.map { stmt -> - when (stmt) { - is IonSchemaModel.SchemaStatement.HeaderStatement -> { - val fields = listOfNotNull( - stmt.imports?.let { imports -> - field("imports", ionListOf(imports.items.map { i -> i.toIsl() })) - }, - *stmt.openContent.toStructFields().toTypedArray() - ) - - ionStructOf(fields, annotations = listOf("schema_header")).asAnyElement() - } - is IonSchemaModel.SchemaStatement.FooterStatement -> - ionStructOf( - stmt.openContent.toStructFields(), - annotations = listOf("schema_footer") - ) - .asAnyElement() - is IonSchemaModel.SchemaStatement.TypeStatement -> stmt.typeDef.toIsl(isInline = false).asAnyElement() - is IonSchemaModel.SchemaStatement.ContentStatement -> stmt.value - } - } - -private fun IonSchemaModel.Import.toIsl(): AnyElement = - ionStructOf( - listOfNotNull( - field("id", ionSymbol(this.id.text)), - this.typeName?.let { field("type", ionSymbol(it.text)) }, - this.alias?.let { field("as", ionSymbol(it.text)) } - ) - ).asAnyElement() - -private fun IonSchemaModel.OpenFieldList.toStructFields(): List = - this.contents.map { field(it.name.text, it.value) } - -/** - * Transforms a PIG-generated [IonSchemaModel.TypeDefinition] into a [StructElement] representing the ISL type - * definition. - * - * @receiver [IonSchemaModel.TypeDefinition] that will be transformed into an ISL type definition - * @param isInline indicates if [this] is an inline type. If false, adds the "type" annotation to the returned ISL type - * definition - * @return transformed ISL document represented as a [StructElement] - */ -fun IonSchemaModel.TypeDefinition.toIsl(isInline: Boolean): StructElement { - val name = name?.text - val nameField = name?.let { listOf(field("name", ionSymbol(it))) } ?: listOf() - val typeStruct = ionStructOf(nameField + this.constraints.items.map { it.toIsl() }) - - return if (!isInline) { - typeStruct.withAnnotations("type") - } else { - typeStruct - } -} - -private fun IonSchemaModel.Constraint.toIsl(): StructField { - return when (this) { - is IonSchemaModel.Constraint.CodepointLength -> field("codepoint_length", this.rule.toIsl()) - is IonSchemaModel.Constraint.ByteLength -> field("byte_length", this.rule.toIsl()) - is IonSchemaModel.Constraint.ContainerLength -> field("container_length", this.rule.toIsl()) - is IonSchemaModel.Constraint.ClosedContent -> field("content", ionSymbol("closed")) - is IonSchemaModel.Constraint.Element -> field("element", this.type.toIsl()) - is IonSchemaModel.Constraint.Fields -> field("fields", ionStructOf(this.fields.map { field(it.name.text, it.type.toIsl()) })) - is IonSchemaModel.Constraint.Precision -> field("precision", this.rule.toIsl()) - is IonSchemaModel.Constraint.Scale -> field("scale", this.rule.toIsl()) - is IonSchemaModel.Constraint.TypeConstraint -> field("type", this.type.toIsl()) - is IonSchemaModel.Constraint.Occurs -> field("occurs", this.spec.toIsl()) - is IonSchemaModel.Constraint.ValidValues -> field("valid_values", this.spec.toIsl()) - is IonSchemaModel.Constraint.Regex -> field("regex", this.toIsl()) - is IonSchemaModel.Constraint.Contains -> field("contains", ionListOf(this.values)) - is IonSchemaModel.Constraint.Not -> field("not", this.type.toIsl()) - is IonSchemaModel.Constraint.OneOf -> field("one_of", this.types.toIsl()) - is IonSchemaModel.Constraint.AllOf -> field("all_of", this.types.toIsl()) - is IonSchemaModel.Constraint.AnyOf -> field("any_of", this.types.toIsl()) - is IonSchemaModel.Constraint.OrderedElements -> field("ordered_elements", this.types.toIsl()) - is IonSchemaModel.Constraint.Annotations -> field("annotations", this.toIsl()) - is IonSchemaModel.Constraint.TimestampPrecision -> field("timestamp_precision", this.precision.toIsl()) - is IonSchemaModel.Constraint.TimestampOffset -> field("timestamp_offset", ionListOf(this.offsetPatterns.map { ionString(it.text) })) - is IonSchemaModel.Constraint.Utf8ByteLength -> field("utf8_byte_length", this.rule.toIsl()) - is IonSchemaModel.Constraint.ArbitraryConstraint -> field(this.name.text, this.value) - } -} - -private fun IonSchemaModel.TsPrecision.toIsl(): IonElement = - when (this) { - is IonSchemaModel.TsPrecision.EqualsTsPrecisionRange -> this.toIsl() - is IonSchemaModel.TsPrecision.EqualsTsPrecisionValue -> this.value.toIsl() - } - -private fun IonSchemaModel.TsPrecision.EqualsTsPrecisionRange.toIsl(): IonElement = - ionListOf(this.range.min.toIsl(), this.range.max.toIsl(), annotations = listOf("range")) - -private fun IonSchemaModel.TsPrecisionExtent.toIsl(): IonElement = - when (this) { - is IonSchemaModel.TsPrecisionExtent.MinTsp -> ionSymbol("min") - is IonSchemaModel.TsPrecisionExtent.MaxTsp -> ionSymbol("max") - is IonSchemaModel.TsPrecisionExtent.InclusiveTsp -> this.precision.toIsl() - is IonSchemaModel.TsPrecisionExtent.ExclusiveTsp -> this.precision.toIsl(listOf("exclusive")) - } - -private fun IonSchemaModel.TsPrecisionValue.toIsl(annotations: List = emptyList()) = - when (this) { - is IonSchemaModel.TsPrecisionValue.Year -> ionSymbol("year", annotations) - is IonSchemaModel.TsPrecisionValue.Month -> ionSymbol("month", annotations) - is IonSchemaModel.TsPrecisionValue.Day -> ionSymbol("day", annotations) - is IonSchemaModel.TsPrecisionValue.Minute -> ionSymbol("minute", annotations) - is IonSchemaModel.TsPrecisionValue.Second -> ionSymbol("second", annotations) - is IonSchemaModel.TsPrecisionValue.Millisecond -> ionSymbol("millisecond", annotations) - is IonSchemaModel.TsPrecisionValue.Microsecond -> ionSymbol("microsecond", annotations) - is IonSchemaModel.TsPrecisionValue.Nanosecond -> ionSymbol("nanosecond", annotations) - } - -private fun List.toIsl(): IonElement = - ionListOf(this.map { it.toIsl() }) - -private fun IonSchemaModel.Constraint.Regex.toIsl() = - ionString( - this.pattern.text, - annotations = listOfNotNull( - "i".takeIf { this.caseInsensitive.booleanValue }, - "m".takeIf { this.multiline.booleanValue } - ) - ) - -private fun IonSchemaModel.Constraint.Annotations.toIsl(): IonElement { - val optionality = when (this.defaultOptionality) { - is IonSchemaModel.Optionality.Required -> "required" - is IonSchemaModel.Optionality.Optional -> "optional" - else -> null - } - val ordered = when { - this.isOrdered.booleanValue -> "ordered" - else -> null - } - - return ionListOf(this.annos.items.map { it.toIsl() }, listOfNotNull(optionality, ordered)) -} - -private fun IonSchemaModel.Annotation.toIsl(): IonElement { - val optionality = when (this.optionality) { - is IonSchemaModel.Optionality.Required -> "required" - is IonSchemaModel.Optionality.Optional -> "optional" - else -> null - } - return ionSymbol(this.text.text, listOfNotNull(optionality)) -} - -private fun IonSchemaModel.ValidValuesSpec.toIsl(): IonElement = - when (this) { - is IonSchemaModel.ValidValuesSpec.OneOfValidValues -> ionListOf(this.values) - is IonSchemaModel.ValidValuesSpec.RangeOfValidValues -> this.range.toIsl() - } - -private fun IonSchemaModel.OccursSpec.toIsl(): IonElement = - when (this) { - is IonSchemaModel.OccursSpec.OccursRule -> this.rule.toIsl() - is IonSchemaModel.OccursSpec.OccursOptional -> ionSymbol("optional") - is IonSchemaModel.OccursSpec.OccursRequired -> ionSymbol("required") - } - -private fun IonSchemaModel.TypeReference.toIsl(): IonElement { - fun nullableAnnos(nullable: Boolean) = - when { - nullable -> listOf("nullable") - else -> listOf() - } - - return when (this) { - is IonSchemaModel.TypeReference.NamedType -> ionSymbol(this.name.text, annotations = nullableAnnos(this.nullable.booleanValue)) - // TODO: since the type definition inside may contain a nullable type constraint, this feels redundant? - // is that just a quirk of ISL or is it something we need to fix. - is IonSchemaModel.TypeReference.InlineType -> this.type.toIsl(isInline = true).withAnnotations(nullableAnnos(this.nullable.booleanValue)) - is IonSchemaModel.TypeReference.ImportedType -> this.toIsl() - } -} - -private fun IonSchemaModel.ValuesRange.toIsl(): IonElement { - return when (this) { - is IonSchemaModel.ValuesRange.NumRange -> this.range.toIsl() - is IonSchemaModel.ValuesRange.TimestampRange -> this.range.toIsl() - } -} - -private fun IonSchemaModel.TsValueRange.toIsl(): IonElement = - ionListOf(this.min.toIsl(), this.max.toIsl(), annotations = listOf("range")) - -private fun IonSchemaModel.TsValueExtent.toIsl(): IonElement = - when (this) { - is IonSchemaModel.TsValueExtent.MinTsValue -> MIN - is IonSchemaModel.TsValueExtent.MaxTsValue -> MAX - is IonSchemaModel.TsValueExtent.InclusiveTsValue -> this.value - is IonSchemaModel.TsValueExtent.ExclusiveTsValue -> this.value.withAnnotations("exclusive") - } - -private fun IonSchemaModel.TypeReference.ImportedType.toIsl(): IonElement { - val nullable = when { - this.nullable.booleanValue -> "nullable" - else -> null - } - val alias = when { - this.alias != null -> listOf(field("as", ionSymbol(this.alias.text))) - else -> listOf() - } - - return ionStructOf(listOf(field("id", ionString(this.id.text)), field("type", ionSymbol(this.type.text))) + alias, annotations = listOfNotNull(nullable)) -} - -private fun IonSchemaModel.NumberRule.toIsl(): IonElement = - when (this) { - is IonSchemaModel.NumberRule.EqualsNumber -> value - is IonSchemaModel.NumberRule.EqualsRange -> this.range.toIsl() - } - -private fun IonSchemaModel.NumberRange.toIsl(): IonElement = - ionListOf(this.min.toIsl(), this.max.toIsl(), annotations = listOf("range")) - -private fun IonSchemaModel.NumberExtent.toIsl(): IonElement = - when (this) { - is IonSchemaModel.NumberExtent.Min -> MIN - is IonSchemaModel.NumberExtent.Max -> MAX - is IonSchemaModel.NumberExtent.Inclusive -> this.value - is IonSchemaModel.NumberExtent.Exclusive -> this.value.withAnnotations("exclusive") - } diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/Error.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/Error.kt deleted file mode 100644 index 2f32614a3..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/Error.kt +++ /dev/null @@ -1,116 +0,0 @@ -package org.partiql.ionschema.parser - -import com.amazon.ionelement.api.ElementType -import com.amazon.ionelement.api.IonElement - -sealed class Error(private val messageFormatter: () -> String) { - val message = messageFormatter() - - data class IonElementConstraintException(val msg: String) : - Error({ "Parse error: $msg" }) - - data class OpenContentOnTypeTemporarilyBlocked(val fieldName: String) : - Error({ "Unrecognized type field '$fieldName' (this is temporary until the parser supports all constraints)" }) - - object ImportMissingTypeFieldWhenAsSpecified : - Error({ "The import's `type` field is required if the `as` field is specified." }) - - data class ValueOfClosedFieldNotContentSymbol(val foundValue: IonElement) : - Error({ "Expected `content`, found `$foundValue`" }) - - data class UnexpectedAnnotation(val annotation: String) : - Error({ "Unexpected annotation: '$annotation'" }) - - data class UnexpectedAnnotationCount(val expectedCount: IntRange, val actualCount: Int) : - Error({ "Expected $expectedCount annotation but $actualCount was/were found" }) - - data class AnnotationNotAllowedHere(val annotation: String) : - Error({ "Annotation '$annotation' is not allowed here" }) - - data class UnexpectedListSize(val expectedCount: IntRange, val actualCount: Int) : - Error({ "Expected $expectedCount list elements but $actualCount was/were found" }) - - object EmptyListNotAllowedHere : - Error({ "Expected nonempty list but empty list was found" }) - - object InvalidNumericExtent : - Error({ "Invalid numeric range; expected 'min', 'max' or [exclusive::]" }) - - object InvalidTimestampExtent : - Error({ "Invalid timestamp range; expected 'min', 'max' or [exclusive::]" }) - - object InvalidValidValuesRangeExtent : - Error({ "Invalid range; expected one of the ends of range to be valid number or timestamp" }) - - object InvalidRange : - Error({ "Invalid range specification" }) - - data class DuplicateField(val fieldName: String) : - Error({ "Duplicate struct field: '$fieldName'" }) - - data class RequiredFieldMissing(val fieldName: String) : - Error({ "Required field '$fieldName' missing" }) - - data class UnexpectedField(val fieldName: String) : - Error({ "Unexpected field '$fieldName'" }) - - object TypeReferenceMustBeSymbolOrStruct : - Error({ "Type references must be a symbol or a struct" }) - - object HeaderMustAppearBeforeTypes : - Error({ "schema_header::{} must appear before any type::{}" }) - - object TypeNotAllowedAfterFooter : - Error({ "type::{} is not allowed after schema_footer::{}" }) - - object MoreThanOneHeaderFound : - Error({ "More than one schema_header::{} is not allowed" }) - - object MoreThanOneFooterFound : - Error({ "More than one schema_footer::{} is not allowed" }) - - object HeaderPresentButNoFooter : - Error({ "A schema_header::{} was included but a schema_footer::{} was not" }) - - object FooterMustAppearAfterHeader : - Error({ "A schema_footer::{} must appear after the schema_header::{}" }) - - object IncorrectRegexPropertyOrder : - Error({ "Incorrect regex property order (expected 'i::m::')" }) - - data class UnexpectedType(val type: ElementType, val expectedTypes: List) : - Error({ "Expected a value of type(s): [${expectedTypes.joinToString()}]; instead found a value of type $type" }) - - data class InvalidOccursSpec(val found: IonElement) : - Error({ "Expected 'optional', 'required' or int range instead of '$found'" }) - - data class InvalidValidValuesSpec(val invalidSpec: IonElement) : - Error({ "Invalid valid_values specification: '$invalidSpec'" }) - - data class InvalidAnnotationsForAnnotationsConstraint(val annotation: String) : - Error({ "Invalid annotations for 'annotations' constraint. Expected 'required', 'optional' or 'ordered' but found $annotation" }) - - data class DuplicateAnnotationsNotAllowed(val annotation: String) : - Error({ "Expected unique annotations but found duplicated annotations $annotation" }) - - object CannotIncludeRequiredAndOptional : - Error({ "Cannot include both 'required' and 'optional' annotations at the same time. Expected no annotations or only either of 'required' or 'optional'." }) - - data class InvalidTimeStampPrecision(val found: String) : - Error({ "Expected 'year', 'month', 'day', 'minute', 'second', 'millisecond', 'microsecond' or 'nanosecond' but found '$found'" }) - - data class InvalidTimeStampOffsetPattern(val found: String) : - Error({ "Pattern must be of the form '[+|-]HH:MM' but found '$found'" }) - - data class InvalidTimeStampOffsetValueForHH(val found: String) : - Error({ "'HH' offset in the offset pattern '[+|-]HH:MM' expected in the range [0,23] but found '$found'" }) - - data class InvalidTimeStampOffsetValueForMM(val found: String) : - Error({ "'MM' offset in the offset pattern '[+|-]HH:MM' expected in the range [0,59] but found '$found'" }) - - data class UnexpectedNumberOfFields(val expectedCount: IntRange, val actualCount: Int) : - Error({ "Expected $expectedCount fields but $actualCount was/were found" }) - - data class InvalidFieldsForInlineImport(val found: List) : - Error({ "Expected fields to be 'id', 'type' or 'as' but found $found" }) -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/Exceptions.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/Exceptions.kt deleted file mode 100644 index 1c7d72763..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/Exceptions.kt +++ /dev/null @@ -1,44 +0,0 @@ -package org.partiql.ionschema.parser - -import com.amazon.ionelement.api.ElementType -import com.amazon.ionelement.api.IonElement -import com.amazon.ionelement.api.IonLocation -import com.amazon.ionelement.api.location -import com.amazon.ionschema.IonSchemaException - -class IonSchemaParseException(val location: IonLocation?, val error: Error) : - IonSchemaException(getMessage(location, error.message)) - -private fun getMessage(blame: IonLocation?, message: String): String { - val location = blame ?: "" - return "$location: $message" -} - -internal fun parseError(blame: IonElement, error: Error): Nothing = - throw IonSchemaParseException(blame.metas.location, error) - -internal fun parseError(blame: IonLocation?, error: Error): Nothing = - throw IonSchemaParseException(blame, error) - -data class ModelValidationError( - val component: String, - val actualType: ElementType, - val expectedTypes: List -) { - internal fun makeMessage(): String { - val typesString = expectedTypes.joinToString { it.toString() } - return "Expected $component to be (one of) $typesString instead of $actualType" - } -} - -// model validation errors generally do not have source locations. -internal fun modelValidationError( - component: String, - actualType: ElementType, - expectedTypes: List -): Nothing { - throw IonSchemaModelValidationError(ModelValidationError(component, actualType, expectedTypes)) -} - -class IonSchemaModelValidationError(val error: ModelValidationError) : - IonSchemaException(error.makeMessage()) diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/IonSchemaModelValidator.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/IonSchemaModelValidator.kt deleted file mode 100644 index 57d584d17..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/IonSchemaModelValidator.kt +++ /dev/null @@ -1,83 +0,0 @@ -package org.partiql.ionschema.parser - -import com.amazon.ionelement.api.ElementType -import com.amazon.ionelement.api.IonElement -import org.partiql.ionschema.model.IonSchemaModel - -private object IonSchemaModelValidator : IonSchemaModel.Visitor() { - - /** Constrains `byte_length` values to integers. */ - override fun visitConstraintByteLength(node: IonSchemaModel.Constraint.ByteLength) = validateIntRule(node.rule) - - /** Constrains `codepoint_length` values to integers. */ - override fun visitConstraintCodepointLength(node: IonSchemaModel.Constraint.CodepointLength) = - validateIntRule(node.rule) - - /** Constrains `container_length` values to integers. */ - override fun visitConstraintContainerLength(node: IonSchemaModel.Constraint.ContainerLength) = - validateIntRule(node.rule) - - /** Constrains `precision` values to integers. */ - override fun visitConstraintPrecision(node: IonSchemaModel.Constraint.Precision) = validateIntRule(node.rule) - - /** Constrains `scale` values to integers. */ - override fun visitConstraintScale(node: IonSchemaModel.Constraint.Scale) = validateIntRule(node.rule) - - /** Constrains `occurs` values to integers. */ - override fun visitOccursSpecOccursRule(node: IonSchemaModel.OccursSpec.OccursRule) = validateIntRule(node.rule) - - /** Constrains `imported_type`'s `nullable` annotation to booleans. */ - override fun visitTypeReferenceImportedType(node: IonSchemaModel.TypeReference.ImportedType) = - requireBooleanType(node.nullable, "ImportedType.nullable") - - /** Constrains `inline_type`'s `nullable` annotation to booleans. */ - override fun visitTypeReferenceInlineType(node: IonSchemaModel.TypeReference.InlineType) = - requireBooleanType(node.nullable, "InlineType.nullable") - - /** Constrains `named_type`'s `nullable` annotation to booleans. */ - override fun visitTypeReferenceNamedType(node: IonSchemaModel.TypeReference.NamedType) = - requireBooleanType(node.nullable, "NamedType.nullable") - - /** Constrains `utf8_byte_length` to integers. */ - override fun visitConstraintUtf8ByteLength(node: IonSchemaModel.Constraint.Utf8ByteLength) { - validateIntRule(node.rule) - } - - /** - * Constrains `valid_values: range::...` values to *numbers*. - * The reason we can't just override [visitNumberExtent] and [visitNumberRule] directly is because - * the validation must be applied *differently* to this particular number range. For `valid_values`, - * any number is allowed. - */ - override fun visitValuesRangeNumRange(node: IonSchemaModel.ValuesRange.NumRange) = - validateNumberRange(node.range) - - /** Constrains `regex`'s `case_insensitive` and `multiline` annotations to booleans. */ - override fun visitConstraintRegex(node: IonSchemaModel.Constraint.Regex) { - requireBooleanType(node.caseInsensitive, "Regex.caseInsensitive") - requireBooleanType(node.multiline, "Regex.multiline") - } -} - -private fun requireBooleanType(elem: IonElement, component: String) { - if (elem.type != ElementType.BOOL) { - modelValidationError(component, elem.type, listOf(ElementType.BOOL)) - } -} - -/** - * Validates the given schema. - * - * Constrains the types of Ion values that are allowed in the various elements of type - * `number_range` within the `ion_schema_model` PIG domain. - * - * This is necessary because the `number_range` type is currently forced to allow any - * Ion value due to a number of as-yet unimplemented features that PIG would help - * make modeling this simpler: - * - * https://github.com/partiql/partiql-ir-generator/issues/43 - * https://github.com/partiql/partiql-ir-generator/issues/46 - * https://github.com/partiql/partiql-ir-generator/issues/47 - */ -fun validateSchemaModel(schema: IonSchemaModel.Schema) = - IonSchemaModelValidator.walkSchema(schema) diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/IonSchemaParser.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/IonSchemaParser.kt deleted file mode 100644 index be2f83556..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/IonSchemaParser.kt +++ /dev/null @@ -1,546 +0,0 @@ -package org.partiql.ionschema.parser - -import com.amazon.ionelement.api.AnyElement -import com.amazon.ionelement.api.IonElement -import com.amazon.ionelement.api.IonElementConstraintException -import com.amazon.ionelement.api.ListElement -import com.amazon.ionelement.api.StringElement -import com.amazon.ionelement.api.StructElement -import com.amazon.ionelement.api.SymbolElement -import com.amazon.ionelement.api.TimestampElement -import com.amazon.ionelement.api.ionBool -import com.amazon.ionelement.api.ionSymbol -import org.partiql.ionschema.model.IonSchemaModel -import org.partiql.pig.runtime.SymbolPrimitive - -private val MIN = ionSymbol("min") -private val MAX = ionSymbol("max") -internal val validAnnotationsForAnnotationsConstraint = setOf("ordered", "required", "optional") -internal val validAnnotationsForAnnotationConstraint = setOf("required", "optional") -internal val validAnnotationsForTypeReference = setOf("nullable", "type") -internal val timestampOffsetPatternRegex = Regex("[+-]\\d\\d:\\d\\d") -internal val validImportedTypeFields = listOf("id", "type", "as") - -/** - * Transforms an ISL document into an [IonSchemaModel.Schema], which is a PIG-generated in-memory object representing - * ISL entities. - * - * @param elements an [IonElement] representation of an ISL document to be transformed to [IonSchemaModel.Schema] - * @return [elements] transformed into an [IonSchemaModel.Schema] - */ -fun parseSchema(elements: List): IonSchemaModel.Schema = - try { - IonSchemaModel.build { - // Mutation is icky--a solution that doesn't utilize mutation would be better. - var hasHeader = false - var hasFooter = false - var hasType = false - - val stmts = elements.map { - when { - it.annotations.contains("type") -> { - hasType = true - - if (hasFooter) { - parseError(it, Error.TypeNotAllowedAfterFooter) - } - - typeStatement(parseTypeDefinition(it.asStruct(), isInline = false)) - } - it.annotations.contains("schema_header") -> { - if (hasHeader) { - parseError(it, Error.MoreThanOneHeaderFound) - } - if (hasType) { - parseError(it, Error.HeaderMustAppearBeforeTypes) - } - hasHeader = true - parseHeader(it.asStruct()) - } - it.annotations.contains("schema_footer") -> { - if (hasFooter) { - parseError(it, Error.MoreThanOneFooterFound) - } - if (!hasHeader) { - parseError(it, Error.FooterMustAppearAfterHeader) - } - hasFooter = true - - parseFooter(it.asStruct()) - } - else -> contentStatement(it) - } - } - - if (hasHeader && !hasFooter) { - parseError( - elements.first { it.annotations.contains("schema_header") }, - Error.HeaderPresentButNoFooter - ) - } - - schema(stmts).also { validateSchemaModel(it) } - } - } catch (ex: IonElementConstraintException) { - parseError( - ex.location, - Error.IonElementConstraintException(ex.message ?: "") - ) - } - -private fun parseHeader(elem: StructElement): IonSchemaModel.SchemaStatement.HeaderStatement { - return extractAllFields(elem) { - val importList = extractOptional("imports") { parseImportList(it.asList()) } - val openFields = extractRemainingFields().map { IonSchemaModel.build { openField(it.name, it.value) } } - - IonSchemaModel.build { - headerStatement( - imports = importList, - openContent = openFieldList(openFields) - ) - } - } -} - -private fun parseImportList(listElem: ListElement): IonSchemaModel.ImportList { - val imports: List = listElem.values.map { listItem -> - extractAllFields(listItem.asStruct()) { - val id = extractRequired("id") { it.textValue } - val type = extractOptional("type") { it.symbolValue } - val asAlias = extractOptional("as") { it.symbolValue } - - if (asAlias != null && type == null) { - parseError(listItem, Error.ImportMissingTypeFieldWhenAsSpecified) - } - - IonSchemaModel.build { import(id, type, asAlias) } - } - } - return IonSchemaModel.build { importList(imports) } -} - -private fun parseFooter(elem: StructElement): IonSchemaModel.SchemaStatement.FooterStatement = - IonSchemaModel.build { - footerStatement(openFieldList(elem.fields.map { openField(it.name, it.value) })) - } - -private typealias ConstraintParselet = IonSchemaModel.Builder.(AnyElement) -> IonSchemaModel.Constraint - -private fun constraintParselet(name: String, block: ConstraintParselet): Pair = Pair(name, block) - -private val constraintParselets = mapOf( - constraintParselet("content") { it: AnyElement -> - if (it.symbolValue != "closed") { - parseError(it, Error.ValueOfClosedFieldNotContentSymbol(it)) - } - closedContent() - }, - constraintParselet("type") { it: AnyElement -> - typeConstraint(parseTypeReference(it)) - }, - constraintParselet("codepoint_length") { - codepointLength(parseNumberRule(it)) - }, - constraintParselet("precision") { - precision(parseNumberRule(it)) - }, - constraintParselet("scale") { - scale(parseNumberRule(it)) - }, - constraintParselet("element") { - element(parseTypeReference(it)) - }, - constraintParselet("byte_length") { - byteLength(parseNumberRule(it)) - }, - constraintParselet("container_length") { - containerLength(parseNumberRule(it)) - }, - constraintParselet("regex") { - parseRegexConstraint(it.asString()) - }, - constraintParselet("not") { - not(parseTypeReference(it)) - }, - constraintParselet("all_of") { - allOf(parseTypeReferenceList(it)) - }, - constraintParselet("any_of") { - anyOf(parseTypeReferenceList(it)) - }, - constraintParselet("one_of") { - oneOf(parseTypeReferenceList(it)) - }, - constraintParselet("ordered_elements") { - orderedElements(parseTypeReferenceList(it)) - }, - constraintParselet("contains") { - contains(it.listValues) - }, - constraintParselet("occurs") { it -> - occurs( - when { - it is SymbolElement -> { - when (it.textValue) { - "optional" -> this.occursOptional() - "required" -> this.occursRequired() - else -> parseError(it, Error.InvalidOccursSpec(it)) - } - } - it is ListElement || it.isNumber -> { - occursRule(parseNumberRule(it)) - } - else -> parseError(it, Error.InvalidOccursSpec(it)) - } - ) - }, - constraintParselet("valid_values") { it -> - validValues( - when { - it.annotations.contains("range") -> rangeOfValidValues(parseValuesRange(it)) - it is ListElement -> oneOfValidValues(it.listValues) - else -> parseError(it, Error.InvalidValidValuesSpec(it)) - } - ) - }, - constraintParselet("fields") { elem -> - elem.requireZeroAnnotations() - - val structElem = elem.asStruct() - - IonSchemaModel.build { - fields( - structElem.fields.map { structField -> - field(structField.name, parseTypeReference(structField.value)) - } - ) - } - }, - constraintParselet("annotations") { - parseAnnotations(it) - }, - constraintParselet("timestamp_precision") { - timestampPrecision(parseTimestampPrecision(it)) - }, - constraintParselet("timestamp_offset") { - timestampOffset(parseTimestampOffset(it)) - }, - constraintParselet("utf8_byte_length") { - utf8ByteLength(parseNumberRule(it)) - } -) - -internal fun parseAnnotations(value: AnyElement): IonSchemaModel.Constraint.Annotations { - val annotationsList = value.asList().requireUniqueAnnotations() - - if (annotationsList.annotations.size !in 0..2) { - parseError(annotationsList, Error.UnexpectedAnnotationCount(0..2, annotationsList.annotations.size)) - } - - annotationsList.annotations.forEach { - if (it !in validAnnotationsForAnnotationsConstraint) { - parseError(annotationsList, Error.InvalidAnnotationsForAnnotationsConstraint(it)) - } - } - - if (annotationsList.annotations.containsAll(validAnnotationsForAnnotationConstraint)) { - parseError(annotationsList, Error.CannotIncludeRequiredAndOptional) - } - - val optionality = when { - annotationsList.annotations.contains("required") -> IonSchemaModel.build { required() } - annotationsList.annotations.contains("optional") -> IonSchemaModel.build { optional() } - else -> null - } - - val isOrdered = when { - annotationsList.annotations.contains("ordered") -> ionBool(true) - else -> ionBool(false) - } - - val annos = annotationsList.values.map { - it.allowSingleAnnotation(validAnnotationsForAnnotationConstraint) - val anno = it.annotations.firstOrNull() - IonSchemaModel.build { - annotation( - it.textValue, - when (anno) { - "required" -> IonSchemaModel.build { required() } - "optional" -> IonSchemaModel.build { optional() } - null -> null - else -> parseError(it, Error.AnnotationNotAllowedHere(anno)) - } - ) - } - } - - return IonSchemaModel.build { annotations(isOrdered, annotationList(annos), optionality) } -} - -internal fun parseTimestampPrecision(tsp: AnyElement): IonSchemaModel.TsPrecision = IonSchemaModel.build { - when { - tsp.allowSingleAnnotation("range") -> equalsTsPrecisionRange(parseTsPrecisionRange(tsp)) - else -> equalsTsPrecisionValue(parseTsPrecisionValue(tsp)) - } -} - -internal fun parseValuesRange(valuesRange: AnyElement): IonSchemaModel.ValuesRange { - val listElem = valuesRange - .requireSingleAnnotation("range") - .asList() - .requireSize(2) - - val fromElem = listElem.values[0] - val toElem = listElem.values[1] - - return when { - fromElem.isNumber || toElem.isNumber -> IonSchemaModel.build { numRange(parseNumberRange(valuesRange)) } - fromElem is TimestampElement || toElem is TimestampElement -> parseTimestampValuesRange(valuesRange) - else -> parseError(valuesRange, Error.InvalidValidValuesRangeExtent) - } -} - -internal fun parseTimestampValuesRange(tsValueRange: AnyElement): IonSchemaModel.ValuesRange.TimestampRange { - val listElem = tsValueRange - .requireSingleAnnotation("range") - .asList() - .requireSize(2) - - val fromElem = listElem.values[0] - val toElem = listElem.values[1] - - return IonSchemaModel.build { - timestampRange( - tsValueRange( - parseTimestampValuesExtent(fromElem), - parseTimestampValuesExtent(toElem) - ) - ) - } -} - -internal fun parseTimestampValuesExtent(elem: AnyElement) = - when (elem) { - is SymbolElement -> when (elem) { - MIN -> IonSchemaModel.build { minTsValue() } - MAX -> IonSchemaModel.build { maxTsValue() } - else -> parseError(elem, Error.InvalidTimestampExtent) - } - is TimestampElement -> if (elem.allowSingleAnnotation("exclusive")) { - IonSchemaModel.build { exclusiveTsValue((elem as TimestampElement).withoutAnnotations()) } - } else { - IonSchemaModel.build { inclusiveTsValue(elem) } - } - else -> { - parseError(elem, Error.InvalidTimestampExtent) - } - } - -internal fun parseTsPrecisionRange(tspRange: AnyElement): IonSchemaModel.TsPrecisionRange { - val tspRangeList = tspRange - .asList() - .requireSize(2) - - return IonSchemaModel.build { tsPrecisionRange(min = parseTsPrecisionExtent(tspRangeList.values[0]), max = parseTsPrecisionExtent(tspRangeList.values[1])) } -} - -internal fun parseTsPrecisionExtent(tspExtent: AnyElement): IonSchemaModel.TsPrecisionExtent { - if (tspExtent.textValue == "min") { - return IonSchemaModel.build { minTsp() } - } - if (tspExtent.textValue == "max") { - return IonSchemaModel.build { maxTsp() } - } - - return when { - tspExtent.allowSingleAnnotation("exclusive") -> IonSchemaModel.build { exclusiveTsp(parseTsPrecisionValue(tspExtent)) } - else -> IonSchemaModel.build { inclusiveTsp(parseTsPrecisionValue(tspExtent)) } - } -} - -internal fun parseTsPrecisionValue(precision: AnyElement): IonSchemaModel.TsPrecisionValue = - when (precision.textValue) { - "year" -> IonSchemaModel.build { year() } - "month" -> IonSchemaModel.build { month() } - "day" -> IonSchemaModel.build { day() } - "minute" -> IonSchemaModel.build { minute() } - "second" -> IonSchemaModel.build { second() } - "millisecond" -> IonSchemaModel.build { millisecond() } - "microsecond" -> IonSchemaModel.build { microsecond() } - "nanosecond" -> IonSchemaModel.build { nanosecond() } - else -> parseError(precision, Error.InvalidTimeStampPrecision(precision.textValue)) - } - -internal fun parseTimestampOffset(offset: AnyElement): List { - return offset - .requireZeroAnnotations() - .asList() - .requireNonzeroListSize() - .values.map { parseTimestampOffsetPattern(it.asString()) } -} - -internal fun parseTimestampOffsetPattern(offsetPattern: StringElement): String { - if (!offsetPattern.textValue.matches(timestampOffsetPatternRegex)) { - parseError(offsetPattern, Error.InvalidTimeStampOffsetPattern(offsetPattern.textValue)) - } - - if (offsetPattern.textValue.substring(1, 3).toInt() !in 0..23) { - parseError(offsetPattern, Error.InvalidTimeStampOffsetValueForHH(offsetPattern.textValue)) - } - - if (offsetPattern.textValue.substring(4, 6).toInt() !in 0..59) { - parseError(offsetPattern, Error.InvalidTimeStampOffsetValueForMM(offsetPattern.textValue)) - } - - return offsetPattern.textValue -} - -private fun parseRegexConstraint(regex: StringElement): IonSchemaModel.Constraint { - if (regex.annotations.size !in 0..2) { - parseError(regex, Error.UnexpectedAnnotationCount(0..2, regex.annotations.size)) - } - - val invalidAnno = regex.annotations.firstOrNull() { it != "i" && it != "m" } - if (invalidAnno != null) { - parseError(regex, Error.UnexpectedAnnotation(invalidAnno)) - } - - if (regex.annotations.size == 2 && regex.annotations[0] == "m") { - parseError(regex, Error.IncorrectRegexPropertyOrder) - } - - val hasI = regex.annotations.contains("i") - val hasM = regex.annotations.contains("m") - return IonSchemaModel.build { - regex(regex.textValue, caseInsensitive = ionBool(hasI), multiline = ionBool(hasM)) - } -} - -/** - * Transforms an ISL type definition into an [IonSchemaModel.TypeDefinition]. - * - * @param struct an [IonElement] representation of an ISL type definition to be transformed to - * [IonSchemaModel.TypeDefinition] - * @param isInline indicates if [struct] is an inline type. If false, checks that [struct] has the "type" annotation - * @return [struct] transformed into an [IonSchemaModel.TypeDefinition] - * @throws [IonSchemaParseException] if [isInline] is false and the "type" annotation is not included in [struct] - */ -fun parseTypeDefinition(struct: StructElement, isInline: Boolean): IonSchemaModel.TypeDefinition { - if (!isInline) { - struct.requireSingleAnnotation("type") - } - - return extractAllFields(struct) { - val typeName = extractOptional("name") { it.asSymbol() } - - val constraintElements = extractRemainingFields() - // TODO: support open content. - - IonSchemaModel.build { - val constraints = constraintElements.map { - // TODO: allow open content here, remove the parseError! - val parselet = constraintParselets[it.name] - when (parselet) { - null -> arbitraryConstraint(name = it.name, value = it.value) - else -> parselet(it.value) - } - } - - typeDefinition_( - name = typeName?.toSymbolPrimitive(), - constraints = constraintList(constraints) - ) - } - } -} - -private fun parseTypeReferenceList(elem: AnyElement): List { - val listElem = elem.asList() - return listElem.values.map { parseTypeReference(it) } -} - -internal fun parseTypeReference(elem: AnyElement): IonSchemaModel.TypeReference { - elem.allowAnnotations(validAnnotationsForTypeReference) - val isNullable = ionBool(elem.annotations.contains("nullable")) - return IonSchemaModel.build { - when (elem) { - is SymbolElement -> namedType(elem.symbolValue, isNullable) - is StructElement -> when { - elem.getOptional("id") != null -> parseImportedType(elem) - else -> inlineType(parseTypeDefinition(elem, isInline = true), isNullable) - } - else -> parseError(elem, Error.TypeReferenceMustBeSymbolOrStruct) - } - } -} - -internal fun parseImportedType(elem: StructElement): IonSchemaModel.TypeReference.ImportedType { - if (elem.fields.size !in 2..3) { - parseError(elem, Error.UnexpectedNumberOfFields(2..3, elem.fields.size)) - } - - val invalidFields = elem.fields.map { it.name } - validImportedTypeFields - if (invalidFields.isNotEmpty()) { - parseError(elem, Error.InvalidFieldsForInlineImport(invalidFields)) - } - - return IonSchemaModel.build { - importedType( - id = elem["id"].textValue, - type = elem["type"].textValue, - nullable = if (elem.annotations.contains("nullable")) ionBool(true) else ionBool(false), - alias = elem.getOptional("as")?.textValue - ) - } -} - -internal fun parseNumberRule(elem: AnyElement): IonSchemaModel.NumberRule = - if (elem.isNumber) { - elem.requireZeroAnnotations() - IonSchemaModel.build { - equalsNumber(elem) - } - } else { - IonSchemaModel.build { - equalsRange(parseNumberRange(elem)) - } - } - -private val IonElement.isNumber get() = NUMBER_TYPES.contains(this.type) - -private fun parseNumberRange(elem: AnyElement): IonSchemaModel.NumberRange { - val listElem = elem - .requireSingleAnnotation("range") - .asList() - .requireSize(2) - - val fromElem = listElem.values[0] - val toElem = listElem.values[1] - - return IonSchemaModel.build { - numberRange(parseNumberExtent(fromElem), parseNumberExtent(toElem)) - } -} - -private fun parseNumberExtent(elem: IonElement): IonSchemaModel.NumberExtent = IonSchemaModel.build { - when { - elem is SymbolElement -> - when (elem) { - MIN -> min() - MAX -> max() - else -> parseError(elem, Error.InvalidNumericExtent) - } - elem.isNumber -> - if (elem.allowSingleAnnotation("exclusive")) { - exclusive(elem.withoutAnnotations()) - } else { - inclusive(elem) - } - else -> { - parseError(elem, Error.InvalidNumericExtent) - } - } -} - -private fun SymbolElement.toSymbolPrimitive() = - SymbolPrimitive(this.textValue, this.metas) diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/RangeValidator.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/RangeValidator.kt deleted file mode 100644 index 28bf6cf71..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/RangeValidator.kt +++ /dev/null @@ -1,49 +0,0 @@ -package org.partiql.ionschema.parser - -import com.amazon.ionelement.api.ElementType -import com.amazon.ionelement.api.IonElement -import org.partiql.ionschema.model.IonSchemaModel - -private class RangeValidator( - val allowedTypes: Set -) : IonSchemaModel.Visitor() { - - private fun assertIsNumber(elem: IonElement, component: String) { - if (!allowedTypes.contains(elem.type)) { - modelValidationError(component, elem.type, allowedTypes.toList()) - } - } - - override fun visitNumberExtentInclusive(node: IonSchemaModel.NumberExtent.Inclusive) = - assertIsNumber(node.value, "Inclusive.value") - - override fun visitNumberExtentExclusive(node: IonSchemaModel.NumberExtent.Exclusive) = - assertIsNumber(node.value, "Exclusive.value") - - override fun visitNumberRuleEqualsNumber(node: IonSchemaModel.NumberRule.EqualsNumber) = - assertIsNumber(node.value, "EqualsNumber.value") -} - -internal val NUMBER_TYPES = setOf(ElementType.INT, ElementType.FLOAT, ElementType.DECIMAL) -private val NUMBER_RANGE_VALIDATOR = RangeValidator(NUMBER_TYPES) -private val INT_RANGE_VALIDATOR = RangeValidator(setOf(ElementType.INT)) - -/** - * Validates that the specified [IonSchemaModel.NumberRange] contains only Ion numbers. - * - * Throws an exception of the range contains a value that is not an `int`, `float`, or `decimal`. - * - * This is necessary because at this time PIG does not support generics. - */ -internal fun validateNumberRange(range: IonSchemaModel.NumberRange) = - NUMBER_RANGE_VALIDATOR.walkNumberRange(range) - -/** - * Validates that the specified [IonSchemaModel.NumberRange] contains only Ion integers. - * - * Throws an exception of the range contains a value that is not an `int`. - * - * This is necessary because at this time PIG does not support a `number` type, only `int`. - */ -internal fun validateIntRule(rule: IonSchemaModel.NumberRule) = - INT_RANGE_VALIDATOR.walkNumberRule(rule) diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/RequireFunctions.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/RequireFunctions.kt deleted file mode 100644 index 0bec766fe..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/RequireFunctions.kt +++ /dev/null @@ -1,85 +0,0 @@ -package org.partiql.ionschema.parser - -import com.amazon.ionelement.api.ContainerElement -import com.amazon.ionelement.api.IonElement - -internal fun T.requireSingleAnnotation(anno: String): T { - when (this.annotations.size) { - 1 -> { - if (this.annotations[0] != anno) { - parseError(this, Error.UnexpectedAnnotation(this.annotations[0])) - } - } - else -> parseError(this, Error.UnexpectedAnnotationCount(1..1, this.annotations.size)) - } - return this -} - -internal fun T.requireZeroAnnotations(): T { - if (this.annotations.any()) { - parseError(this, Error.UnexpectedAnnotationCount(0..0, this.annotations.size)) - } - return this -} - -internal fun T.allowSingleAnnotation(anno: String): Boolean = - when (this.annotations.size) { - 0 -> false - 1 -> { - val foundAnno = this.annotations[0] - if (foundAnno != anno) { - parseError(this, Error.AnnotationNotAllowedHere(foundAnno)) - } - true - } - else -> parseError(this, Error.UnexpectedAnnotationCount(0..1, this.annotations.size)) - } - -internal fun T.allowSingleAnnotation(validAnnos: Set): Boolean = - when (this.annotations.size) { - 0 -> false - 1 -> { - val foundAnno = this.annotations[0] - if (foundAnno !in validAnnos) { - parseError(this, Error.AnnotationNotAllowedHere(foundAnno)) - } - true - } - else -> parseError(this, Error.UnexpectedAnnotationCount(0..1, this.annotations.size)) - } - -internal fun T.allowAnnotations(validAnnos: Set): T = - when (this.annotations.size) { - in 0..validAnnos.size -> { - this.annotations.forEach { - if (it !in validAnnos) { - parseError(this, Error.AnnotationNotAllowedHere(it)) - } - } - this - } - else -> parseError(this, Error.UnexpectedAnnotationCount(0..1, this.annotations.size)) - } - -internal fun T.requireUniqueAnnotations(): T { - this.annotations - .groupBy { it }.entries - .firstOrNull { it.value.size > 1 } - ?.let { (key, _) -> - parseError(this, Error.DuplicateAnnotationsNotAllowed(key)) - } - return this -} -internal fun T.requireSize(s: Int): T { - if (this.size != s) { - parseError(this, Error.UnexpectedListSize(s..s, this.size)) - } - return this -} - -internal fun T.requireNonzeroListSize(): T { - if (this.size == 0) { - parseError(this, Error.EmptyListNotAllowedHere) - } - return this -} diff --git a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/StructElementFieldExtractor.kt b/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/StructElementFieldExtractor.kt deleted file mode 100644 index c0f419ca8..000000000 --- a/lib/isl/src/main/kotlin/org/partiql/ionschema/parser/StructElementFieldExtractor.kt +++ /dev/null @@ -1,49 +0,0 @@ -package org.partiql.ionschema.parser - -import com.amazon.ionelement.api.AnyElement -import com.amazon.ionelement.api.StructElement -import com.amazon.ionelement.api.StructField - -// TODO: how should this class handle duplicate required or optional fields? -// If we only call extract* once, the other fields of the same name will still be present -// and will be returned from extractRemainingFields. -internal class StructElementFieldExtractor(val struct: StructElement) { - private val remainingFields = struct.fields.toMutableList() - - val fieldsRemainingCount get() = remainingFields.size - - fun extractOptional(fieldName: String, process: (AnyElement) -> T): T? { - val matchingFields = remainingFields.filter { it.name == fieldName } - return when (matchingFields.size) { - 0 -> null - 1 -> { - val field = matchingFields.first() - process(field.value).also { - remainingFields.remove(field) - } - } - else -> parseError(struct, Error.DuplicateField(fieldName)) - } - } - - fun extractRequired(fieldName: String, process: (AnyElement) -> T): T = - extractOptional(fieldName, process) ?: parseError(struct, Error.RequiredFieldMissing(fieldName)) - - fun extractRemainingFields(): Iterable { - val leftovers = remainingFields.toList() - remainingFields.clear() - return leftovers - } -} - -internal fun extractAllFields(struct: StructElement, block: StructElementFieldExtractor.() -> T): T { - val extractor = StructElementFieldExtractor(struct) - val extracted = extractor.block() - - if (extractor.fieldsRemainingCount > 0) { - val unexpectedField = extractor.extractRemainingFields().first() - parseError(unexpectedField.value, Error.UnexpectedField(unexpectedField.name)) - } - - return extracted -} diff --git a/lib/isl/src/main/pig/isl.ion b/lib/isl/src/main/pig/isl.ion deleted file mode 100644 index 8d3946686..000000000 --- a/lib/isl/src/main/pig/isl.ion +++ /dev/null @@ -1,237 +0,0 @@ - -// TODO: locate and remove unused types. - -// TODO: follow up with Ion team on how ISL will be versioned. Based on that outcome, may need to make changes to the ISL domain - -(define ion_schema_model - (domain - // ::= ... - // |
...