From 3bebc9ebb952d96aebc2b479af11f74786da616d Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Tue, 16 May 2023 14:10:42 +0200 Subject: [PATCH 01/14] Fix parsing. --- src/main/java/org/rumbledb/items/Base64BinaryItem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/rumbledb/items/Base64BinaryItem.java b/src/main/java/org/rumbledb/items/Base64BinaryItem.java index 30ea44784..73bdce1f5 100644 --- a/src/main/java/org/rumbledb/items/Base64BinaryItem.java +++ b/src/main/java/org/rumbledb/items/Base64BinaryItem.java @@ -41,7 +41,7 @@ public Base64BinaryItem() { public Base64BinaryItem(String stringValue) { this.value = parseBase64BinaryString(stringValue); - this.stringValue = StringUtils.chomp(Base64.encodeBase64String(this.value)); + this.stringValue = StringUtils.chomp(Base64.getEncoder().encodeToString(this.value)); } @Override From b73a128ea8c0ce6a73c9b4e9936a3724c8d85924 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 10:53:23 +0100 Subject: [PATCH 02/14] Update to Spark 3.5. --- pom.xml | 14 +++++++------- .../java/org/rumbledb/items/Base64BinaryItem.java | 2 +- src/main/java/org/rumbledb/items/DoubleItem.java | 2 +- src/main/java/org/rumbledb/items/FloatItem.java | 2 +- .../strings/NormalizeSpaceFunctionIterator.java | 2 +- .../org/rumbledb/runtime/typing/CastIterator.java | 2 +- .../spark/ml/ApplyEstimatorRuntimeIterator.java | 2 +- .../spark/ml/ApplyTransformerRuntimeIterator.java | 2 +- src/test/java/iq/base/AnnotationsTestsBase.java | 2 +- .../test_files/RumbleML/RumbleML/MLPipeline1.jq | 2 +- .../test_files/RumbleML/RumbleML/MLPipeline2.jq | 2 +- .../test_files/RumbleML/RumbleML/MLPipeline3.jq | 2 +- 12 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index 55c12b062..991bcd648 100644 --- a/pom.xml +++ b/pom.xml @@ -200,31 +200,31 @@ org.apache.spark spark-core_2.12 - 3.3.2 + 3.5.0 provided org.apache.spark spark-sql_2.12 - 3.3.2 + 3.5.0 provided org.apache.spark spark-mllib_2.12 - 3.3.2 + 3.5.0 provided org.apache.hadoop hadoop-aws - 3.3.2 + 3.3.6 provided org.apache.spark spark-avro_2.12 - 3.3.2 + 3.4.0 org.antlr @@ -250,12 +250,12 @@ org.apache.commons commons-text - 1.6 + 1.10.0 org.apache.commons commons-lang3 - 3.9 + 3.12.0 commons-io diff --git a/src/main/java/org/rumbledb/items/Base64BinaryItem.java b/src/main/java/org/rumbledb/items/Base64BinaryItem.java index 73bdce1f5..5e2b8dc3a 100644 --- a/src/main/java/org/rumbledb/items/Base64BinaryItem.java +++ b/src/main/java/org/rumbledb/items/Base64BinaryItem.java @@ -3,7 +3,7 @@ import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.util.Base64; import org.rumbledb.api.Item; import org.rumbledb.exceptions.ExceptionMetadata; diff --git a/src/main/java/org/rumbledb/items/DoubleItem.java b/src/main/java/org/rumbledb/items/DoubleItem.java index fab38f624..a82018813 100644 --- a/src/main/java/org/rumbledb/items/DoubleItem.java +++ b/src/main/java/org/rumbledb/items/DoubleItem.java @@ -24,7 +24,7 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.rumbledb.api.Item; import org.rumbledb.exceptions.ExceptionMetadata; import org.rumbledb.exceptions.IteratorFlowException; diff --git a/src/main/java/org/rumbledb/items/FloatItem.java b/src/main/java/org/rumbledb/items/FloatItem.java index d9dca106c..62f6a2aae 100644 --- a/src/main/java/org/rumbledb/items/FloatItem.java +++ b/src/main/java/org/rumbledb/items/FloatItem.java @@ -24,7 +24,7 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.rumbledb.api.Item; import org.rumbledb.exceptions.ExceptionMetadata; import org.rumbledb.exceptions.IteratorFlowException; diff --git a/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java b/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java index 9b19d0b89..13b704934 100644 --- a/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java @@ -20,7 +20,7 @@ package org.rumbledb.runtime.functions.strings; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.rumbledb.api.Item; import org.rumbledb.context.DynamicContext; import org.rumbledb.context.Name; diff --git a/src/main/java/org/rumbledb/runtime/typing/CastIterator.java b/src/main/java/org/rumbledb/runtime/typing/CastIterator.java index 082b9fb49..b96022e0d 100644 --- a/src/main/java/org/rumbledb/runtime/typing/CastIterator.java +++ b/src/main/java/org/rumbledb/runtime/typing/CastIterator.java @@ -2,7 +2,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Hex; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.rumbledb.api.Item; import org.rumbledb.context.DynamicContext; import org.rumbledb.context.RuntimeStaticContext; diff --git a/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java b/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java index 05dad6ca1..f6b655db4 100644 --- a/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java +++ b/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java @@ -1,6 +1,6 @@ package sparksoniq.spark.ml; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.spark.ml.Estimator; import org.apache.spark.ml.Transformer; import org.apache.spark.ml.linalg.VectorUDT; diff --git a/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java b/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java index 81e288c59..8c6607f08 100644 --- a/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java +++ b/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java @@ -1,6 +1,6 @@ package sparksoniq.spark.ml; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.spark.ml.Transformer; import org.apache.spark.ml.linalg.VectorUDT; import org.apache.spark.ml.param.ParamMap; diff --git a/src/test/java/iq/base/AnnotationsTestsBase.java b/src/test/java/iq/base/AnnotationsTestsBase.java index 30d4353d1..c3a7dff39 100644 --- a/src/test/java/iq/base/AnnotationsTestsBase.java +++ b/src/test/java/iq/base/AnnotationsTestsBase.java @@ -20,7 +20,7 @@ package iq.base; -import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.junit.Assert; import org.rumbledb.api.Item; import org.rumbledb.api.Rumble; diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq index 4a5b07f1d..297962782 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler") let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq index 5bdd8da1a..57b2c3bc0 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler")(?, { "inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features" }) let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq index 43aefcb26..37e99a394 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) declare type local:mytype as { "id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal" From 9d0bfc4a242551501d6be357a6f6f9b190462028 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 11:04:05 +0100 Subject: [PATCH 03/14] Upgrade to AntLR 4.9.3. --- pom.xml | 2 +- .../rumbledb/parser/JsoniqBaseVisitor.java | 2 +- .../java/org/rumbledb/parser/JsoniqLexer.java | 4 +-- .../org/rumbledb/parser/JsoniqParser.java | 4 +-- .../org/rumbledb/parser/JsoniqVisitor.java | 2 +- .../java/org/rumbledb/parser/XQueryLexer.java | 4 +-- .../org/rumbledb/parser/XQueryParser.java | 29 ++++++------------- .../parser/XQueryParserBaseVisitor.java | 2 +- .../rumbledb/parser/XQueryParserVisitor.java | 2 +- 9 files changed, 20 insertions(+), 31 deletions(-) diff --git a/pom.xml b/pom.xml index 991bcd648..aeb0a23d9 100644 --- a/pom.xml +++ b/pom.xml @@ -229,7 +229,7 @@ org.antlr antlr4-runtime - 4.8 + 4.9.3 org.jline diff --git a/src/main/java/org/rumbledb/parser/JsoniqBaseVisitor.java b/src/main/java/org/rumbledb/parser/JsoniqBaseVisitor.java index 294470559..378b338ba 100644 --- a/src/main/java/org/rumbledb/parser/JsoniqBaseVisitor.java +++ b/src/main/java/org/rumbledb/parser/JsoniqBaseVisitor.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; diff --git a/src/main/java/org/rumbledb/parser/JsoniqLexer.java b/src/main/java/org/rumbledb/parser/JsoniqLexer.java index 888f964a1..10855f70d 100644 --- a/src/main/java/org/rumbledb/parser/JsoniqLexer.java +++ b/src/main/java/org/rumbledb/parser/JsoniqLexer.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; @@ -16,7 +16,7 @@ @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public class JsoniqLexer extends Lexer { - static { RuntimeMetaData.checkVersion("4.8", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.9.3", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/src/main/java/org/rumbledb/parser/JsoniqParser.java b/src/main/java/org/rumbledb/parser/JsoniqParser.java index 299fcb8a3..f68fcbf4c 100644 --- a/src/main/java/org/rumbledb/parser/JsoniqParser.java +++ b/src/main/java/org/rumbledb/parser/JsoniqParser.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; @@ -25,7 +25,7 @@ @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public class JsoniqParser extends Parser { - static { RuntimeMetaData.checkVersion("4.8", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.9.3", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/src/main/java/org/rumbledb/parser/JsoniqVisitor.java b/src/main/java/org/rumbledb/parser/JsoniqVisitor.java index e4a5acf0d..b77448678 100644 --- a/src/main/java/org/rumbledb/parser/JsoniqVisitor.java +++ b/src/main/java/org/rumbledb/parser/JsoniqVisitor.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; diff --git a/src/main/java/org/rumbledb/parser/XQueryLexer.java b/src/main/java/org/rumbledb/parser/XQueryLexer.java index 8429e5ce8..7e03a5a61 100644 --- a/src/main/java/org/rumbledb/parser/XQueryLexer.java +++ b/src/main/java/org/rumbledb/parser/XQueryLexer.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/XQueryLexer.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/XQueryLexer.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; @@ -18,7 +18,7 @@ @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public class XQueryLexer extends Lexer { - static { RuntimeMetaData.checkVersion("4.8", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.9.3", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/src/main/java/org/rumbledb/parser/XQueryParser.java b/src/main/java/org/rumbledb/parser/XQueryParser.java index af62b9768..f737166a8 100644 --- a/src/main/java/org/rumbledb/parser/XQueryParser.java +++ b/src/main/java/org/rumbledb/parser/XQueryParser.java @@ -1,31 +1,20 @@ -// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; -import java.util.ArrayList; -import java.util.List; - -import org.antlr.v4.runtime.NoViableAltException; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.RuntimeMetaData; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.Vocabulary; -import org.antlr.v4.runtime.VocabularyImpl; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNDeserializer; -import org.antlr.v4.runtime.atn.ParserATNSimulator; -import org.antlr.v4.runtime.atn.PredictionContextCache; +import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.tree.ParseTreeVisitor; -import org.antlr.v4.runtime.tree.TerminalNode; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.misc.*; +import org.antlr.v4.runtime.tree.*; +import java.util.List; +import java.util.Iterator; +import java.util.ArrayList; @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public class XQueryParser extends Parser { - static { RuntimeMetaData.checkVersion("4.8", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.9.3", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/src/main/java/org/rumbledb/parser/XQueryParserBaseVisitor.java b/src/main/java/org/rumbledb/parser/XQueryParserBaseVisitor.java index bef7ecec8..f41ea731a 100644 --- a/src/main/java/org/rumbledb/parser/XQueryParserBaseVisitor.java +++ b/src/main/java/org/rumbledb/parser/XQueryParserBaseVisitor.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; diff --git a/src/main/java/org/rumbledb/parser/XQueryParserVisitor.java b/src/main/java/org/rumbledb/parser/XQueryParserVisitor.java index c73098436..d8dafaa78 100644 --- a/src/main/java/org/rumbledb/parser/XQueryParserVisitor.java +++ b/src/main/java/org/rumbledb/parser/XQueryParserVisitor.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; From 570b548b69f8d540c789a2e4271440d920350af9 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 11:18:56 +0100 Subject: [PATCH 04/14] Change jackson version to align with Spark. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aeb0a23d9..536a9ba68 100644 --- a/pom.xml +++ b/pom.xml @@ -285,7 +285,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - 2.13.4 + 2.15.2 From c9931b3b1f5409537bab14a2109cf4aad0700ae1 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 11:21:31 +0100 Subject: [PATCH 05/14] Fix test. --- .../org/rumbledb/parser/XQueryParser.java | 25 +++++++++++++------ .../DataFrames/GroupbyClause5.jq | 3 ++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/rumbledb/parser/XQueryParser.java b/src/main/java/org/rumbledb/parser/XQueryParser.java index f737166a8..5af047f1f 100644 --- a/src/main/java/org/rumbledb/parser/XQueryParser.java +++ b/src/main/java/org/rumbledb/parser/XQueryParser.java @@ -3,14 +3,25 @@ // Java header package org.rumbledb.parser; -import org.antlr.v4.runtime.atn.*; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.misc.*; -import org.antlr.v4.runtime.tree.*; -import java.util.List; -import java.util.Iterator; import java.util.ArrayList; +import java.util.List; + +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.RuntimeMetaData; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.VocabularyImpl; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNDeserializer; +import org.antlr.v4.runtime.atn.ParserATNSimulator; +import org.antlr.v4.runtime.atn.PredictionContextCache; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.tree.ParseTreeVisitor; +import org.antlr.v4.runtime.tree.TerminalNode; @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public class XQueryParser extends Parser { diff --git a/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClause5.jq b/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClause5.jq index 64f860404..18a657d22 100644 --- a/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClause5.jq +++ b/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClause5.jq @@ -1,10 +1,11 @@ -(:JIQS: ShouldRun; Output="([ 1, { "guess" : "Bulgarian", "choices" : [ "Albanian", "Bulgarian", "Russian", "Ukrainian" ] }, { "guess" : "Bulgarian", "choices" : [ "Arabic", "Bulgarian", "Hebrew", "Spanish" ] } ], [ 2, { "guess" : "Arabic", "choices" : [ "Arabic", "Cantonese", "Maltese", "Samoan" ] }, { "guess" : "Arabic", "choices" : [ "Albanian", "Arabic", "Czech", "Dutch" ] } ])" :) +(:JIQS: ShouldRun; Output="([ 1, { "guess" : "Arabic", "choices" : [ "Arabic", "Cantonese", "Maltese", "Samoan" ] }, { "guess" : "Arabic", "choices" : [ "Albanian", "Arabic", "Czech", "Dutch" ] } ], [ 2, { "guess" : "Bulgarian", "choices" : [ "Albanian", "Bulgarian", "Russian", "Ukrainian" ] }, { "guess" : "Bulgarian", "choices" : [ "Arabic", "Bulgarian", "Hebrew", "Spanish" ] } ])" :) for $i in parallelize(( {"guess": "Arabic", "choices": ["Arabic", "Cantonese", "Maltese", "Samoan"]}, {"guess": "Arabic", "choices": ["Albanian", "Arabic", "Czech", "Dutch"]}, {"guess": "Bulgarian", "choices": ["Albanian", "Bulgarian", "Russian", "Ukrainian"]}, {"guess": "Bulgarian", "choices": ["Arabic", "Bulgarian", "Hebrew", "Spanish"]})) group by $guess := $i.guess +order by $guess count $group_index return [$group_index, $i] From 1803a5c5bbb9b9006d6631353782208de7c9d750 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 11:28:33 +0100 Subject: [PATCH 06/14] Fix tests. --- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq index 297962782..d39162a15 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.657406267583814 ], "rawPrediction" : [ -37.806767087841045, 37.806767087841045 ], "probability" : [ 3.808287043115719E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.56770709258576, -37.56770709258576 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709795 ], "rawPrediction" : [ -19.168353441006655, 19.168353441006655 ], "probability" : [ 4.73467170810621E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler") let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq index 57b2c3bc0..1c45b2695 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838137 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.567707092585735, -37.567707092585735 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006627, 19.168353441006627 ], "probability" : [ 4.734671708106345E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler")(?, { "inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features" }) let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq index 37e99a394..98837bce5 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838128 ], "rawPrediction" : [ -37.806767087841024, 37.806767087841024 ], "probability" : [ 3.8082870431158006E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258576, -37.56770709258576 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006637, 19.168353441006637 ], "probability" : [ 4.734671708106294E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) declare type local:mytype as { "id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal" diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq index ab9d09ab9..0d2e943f4 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676778395, 38.071372676778395 ], "probability" : [ 2.9228930727443083E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188425, -36.29582598188425 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917061183, 21.188416917061183 ], "probability" : [ 6.280402132397163E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676780044, 38.071372676780044 ], "probability" : [ 2.9228930727394895E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.295825981882075, -36.295825981882075 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.18841691706346, 21.18841691706346 ], "probability" : [ 6.280402132382861E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq index 679ff7ac1..b43c79b86 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676778395, 38.071372676778395 ], "probability" : [ 2.9228930727443083E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188425, -36.29582598188425 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917061183, 21.188416917061183 ], "probability" : [ 6.280402132397163E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.07137267677621, 38.07137267677621 ], "probability" : [ 2.922893072750705E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188709, -36.29582598188709 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917058074, 21.188416917058074 ], "probability" : [ 6.280402132416686E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; let $vector-assembler := get-transformer("VectorAssembler", {"inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features"}) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq index 69b981b46..b60117e61 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676778395, 38.071372676778395 ], "probability" : [ 2.9228930727443083E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188425, -36.29582598188425 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917061183, 21.188416917061183 ], "probability" : [ 6.280402132397163E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.07137267677527, 38.07137267677527 ], "probability" : [ 2.922893072753446E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.295825981888406, -36.295825981888406 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917056745, 21.188416917056745 ], "probability" : [ 6.280402132425032E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; let $pipeline := get-estimator("Pipeline", { From 1d1d9f667836b335917d37ef3478bec9f16a3cd5 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 12:04:11 +0100 Subject: [PATCH 07/14] Fix tests. --- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq index 1c45b2695..df2dc8f33 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838137 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.567707092585735, -37.567707092585735 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006627, 19.168353441006627 ], "probability" : [ 4.734671708106345E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +\1;95;0c(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838137 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.567707092585735, -37.567707092585735 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006627, 19.168353441006627 ], "probability" : [ 4.734671708106345E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler")(?, { "inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features" }) let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq index b43c79b86..7532e2966 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.07137267677621, 38.07137267677621 ], "probability" : [ 2.922893072750705E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188709, -36.29582598188709 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917058074, 21.188416917058074 ], "probability" : [ 6.280402132416686E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.07137267677909, 38.07137267677909 ], "probability" : [ 2.922893072742273E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188342, -36.29582598188342 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917062117, 21.188416917062117 ], "probability" : [ 6.280402132391294E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; let $vector-assembler := get-transformer("VectorAssembler", {"inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features"}) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq index b60117e61..8e81c5d0b 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.07137267677527, 38.07137267677527 ], "probability" : [ 2.922893072753446E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.295825981888406, -36.295825981888406 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917056745, 21.188416917056745 ], "probability" : [ 6.280402132425032E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676775525, 38.071372676775525 ], "probability" : [ 2.9228930727526986E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.295825981888, -36.295825981888 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.18841691705713, 21.18841691705713 ], "probability" : [ 6.280402132422621E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; let $pipeline := get-estimator("Pipeline", { From 524667d5fc2ca9e85bb31eaac6b34beb2528fb1a Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 12:05:08 +0100 Subject: [PATCH 08/14] Fix tests. --- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq index d39162a15..ecc669443 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.657406267583814 ], "rawPrediction" : [ -37.806767087841045, 37.806767087841045 ], "probability" : [ 3.808287043115719E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.56770709258576, -37.56770709258576 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709795 ], "rawPrediction" : [ -19.168353441006655, 19.168353441006655 ], "probability" : [ 4.73467170810621E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838137 ], "rawPrediction" : [ -37.80676708784381, 37.80676708784381 ], "probability" : [ 3.8082870431051935E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.567707092586666, -37.567707092586666 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441008513, 19.168353441008513 ], "probability" : [ 4.734671708097413E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler") let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq index 0d2e943f4..9e2037bf6 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676780044, 38.071372676780044 ], "probability" : [ 2.9228930727394895E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.295825981882075, -36.295825981882075 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.18841691706346, 21.18841691706346 ], "probability" : [ 6.280402132382861E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676777095, 38.071372676777095 ], "probability" : [ 2.9228930727481084E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.2958259818859, -36.2958259818859 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.18841691705933, 21.18841691705933 ], "probability" : [ 6.280402132408788E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; From c52ff1f1ede0d015773e20042ffaf05b52a4107b Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Tue, 27 Feb 2024 17:07:07 +0100 Subject: [PATCH 09/14] Fix test. --- .../resources/test_files/RumbleML/RumbleML/MLPipeline3.jq | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq index 047744ce8..43aefcb26 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838128 ], "rawPrediction" : [ -37.806767087841024, 37.806767087841024 ], "probability" : [ 3.8082870431158006E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258576, -37.56770709258576 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006637, 19.168353441006637 ], "probability" : [ 4.734671708106294E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) declare type local:mytype as { "id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal" @@ -33,4 +33,4 @@ for $i in $trained_est2( $my-new-test-data, {"featuresCol": "features2", "predictionCol": "ocol1"} ) -return local:round($i) \ No newline at end of file +return $i \ No newline at end of file From 4aef76912649b3147652e5babe048beaef456a77 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Tue, 27 Feb 2024 17:08:06 +0100 Subject: [PATCH 10/14] Fix test. --- .../test_files/RumbleML/RumbleML/MLPipeline3.jq | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq index 43aefcb26..2b1ac52fd 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq @@ -1,4 +1,16 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "ocol1" : 1, "features2" : [ 0.6811443, -1.6574062 ], "rawPrediction" : [ -37.806767, 37.806767 ], "probability" : [ 3.808287E-17, 1 ] }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "ocol1" : 0, "features2" : [ -2.9389858, 0.48441696 ], "rawPrediction" : [ 37.567707, -37.567707 ], "probability" : [ 1, 0 ] }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "ocol1" : 1, "features2" : [ 0.28609395, 1.1822492 ], "rawPrediction" : [ -19.168354, 19.168354 ], "probability" : [ 4.7346718E-9, 1 ] })" :) + +declare function local:round($i as object) as object { + {| + remove-keys($i, ("features2", "rawPrediction", "probability")), + { + "features2" : [ for $v in $i.features2[] return float($v) ], + "rawPrediction" : [ for $v in $i.rawPrediction[] return float($v) ], + "probability" : [ for $v in $i.probability[] return float($v) ] + } + |} +}; + declare type local:mytype as { "id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal" @@ -33,4 +45,4 @@ for $i in $trained_est2( $my-new-test-data, {"featuresCol": "features2", "predictionCol": "ocol1"} ) -return $i \ No newline at end of file +return local:round($i) \ No newline at end of file From c4a0e34c743f3c504fcaf2a8e3111d793c6cc195 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Wed, 10 Jul 2024 10:42:45 +0200 Subject: [PATCH 11/14] Minor update. --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 536a9ba68..53ad33031 100644 --- a/pom.xml +++ b/pom.xml @@ -200,19 +200,19 @@ org.apache.spark spark-core_2.12 - 3.5.0 + 3.5.1 provided org.apache.spark spark-sql_2.12 - 3.5.0 + 3.5.1 provided org.apache.spark spark-mllib_2.12 - 3.5.0 + 3.5.1 provided @@ -224,7 +224,7 @@ org.apache.spark spark-avro_2.12 - 3.4.0 + 3.5.1 org.antlr From 3ce80c1a69170b784496d9c3d858a081d347ae38 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 17 Oct 2024 14:43:11 +0200 Subject: [PATCH 12/14] Update pom.xml --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d66785f1e..43b5e3fec 100644 --- a/pom.xml +++ b/pom.xml @@ -304,8 +304,8 @@ io.delta - delta-core_2.12 - 2.4.0 + delta-spark_2.12 + 3.2.1 From 5a8383c18c42c3a2d7a35f8c9de9e181d8e9f262 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 18 Nov 2024 14:40:02 +0100 Subject: [PATCH 13/14] Delete src/test/resources/test_files/runtime-spark/FileInputs/Delta.jq --- .../resources/test_files/runtime-spark/FileInputs/Delta.jq | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 src/test/resources/test_files/runtime-spark/FileInputs/Delta.jq diff --git a/src/test/resources/test_files/runtime-spark/FileInputs/Delta.jq b/src/test/resources/test_files/runtime-spark/FileInputs/Delta.jq deleted file mode 100644 index 2d0ba36f9..000000000 --- a/src/test/resources/test_files/runtime-spark/FileInputs/Delta.jq +++ /dev/null @@ -1,3 +0,0 @@ -(:JIQS: ShouldRun; Output="{ "bool_array" : [ true ], "float" : 4.2, "float_array" : [ 4.2 ], "int64" : 42, "int64_array" : [ 42 ], "object" : { "bool" : true, "float" : 4.2, "int64" : 42, "object" : { "bool" : true }, "string" : "hello" }, "object_array" : [ { "bool" : true, "float" : 4.2, "int64" : 42, "object" : { "bool" : true }, "string" : "hello" } ], "string" : "hello", "string_array" : [ "hello" ] }" :) -delta-file("../../../queries/sample_json_delta") - From 9875e81118cbf8475989d495c97857ce98eba237 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 18 Nov 2024 14:40:33 +0100 Subject: [PATCH 14/14] Update maven.yml --- .github/workflows/maven.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index cda5ccce9..149545d52 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -136,5 +136,3 @@ jobs: run: mvn install -DskipTests -Dgpg.skip --quiet - name: Compile with Maven run: mvn clean compile assembly:single - - name: DeltaUpdateRuntimeTests - run: mvn -Dtest=DeltaUpdateRuntimeTests test