Update to Spark 3.5.

RumbleDB · Dec 21, 2023 · b73a128 · b73a128
1 parent 3bebc9e
commit b73a128
Show file tree

Hide file tree

Showing 12 changed files with 18 additions and 18 deletions.
diff --git a/pom.xml b/pom.xml
@@ -200,31 +200,31 @@
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_2.12</artifactId>
-            <version>3.3.2</version>
+            <version>3.5.0</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_2.12</artifactId>
-            <version>3.3.2</version>
+            <version>3.5.0</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-mllib_2.12</artifactId>
-            <version>3.3.2</version>
+            <version>3.5.0</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
              <artifactId>hadoop-aws</artifactId>
-            <version>3.3.2</version>
+            <version>3.3.6</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-avro_2.12</artifactId>
-            <version>3.3.2</version>
+            <version>3.4.0</version>
         </dependency>
         <dependency>
             <groupId>org.antlr</groupId>
@@ -250,12 +250,12 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-text</artifactId>
-            <version>1.6</version>
+            <version>1.10.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.9</version>
+            <version>3.12.0</version>
         </dependency>
 		<dependency>
             <groupId>commons-io</groupId>

diff --git a/src/main/java/org/rumbledb/items/Base64BinaryItem.java b/src/main/java/org/rumbledb/items/Base64BinaryItem.java
@@ -3,7 +3,7 @@
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import java.util.Base64;
 import org.rumbledb.api.Item;
 import org.rumbledb.exceptions.ExceptionMetadata;

diff --git a/src/main/java/org/rumbledb/items/DoubleItem.java b/src/main/java/org/rumbledb/items/DoubleItem.java
@@ -24,7 +24,7 @@
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.rumbledb.api.Item;
 import org.rumbledb.exceptions.ExceptionMetadata;
 import org.rumbledb.exceptions.IteratorFlowException;

diff --git a/src/main/java/org/rumbledb/items/FloatItem.java b/src/main/java/org/rumbledb/items/FloatItem.java
@@ -24,7 +24,7 @@
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.rumbledb.api.Item;
 import org.rumbledb.exceptions.ExceptionMetadata;
 import org.rumbledb.exceptions.IteratorFlowException;

diff --git a/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java b/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java
@@ -20,7 +20,7 @@
 
 package org.rumbledb.runtime.functions.strings;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.rumbledb.api.Item;
 import org.rumbledb.context.DynamicContext;
 import org.rumbledb.context.Name;

diff --git a/src/main/java/org/rumbledb/runtime/typing/CastIterator.java b/src/main/java/org/rumbledb/runtime/typing/CastIterator.java
@@ -2,7 +2,7 @@
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.rumbledb.api.Item;
 import org.rumbledb.context.DynamicContext;
 import org.rumbledb.context.RuntimeStaticContext;

diff --git a/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java b/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java
@@ -1,6 +1,6 @@
 package sparksoniq.spark.ml;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.spark.ml.Estimator;
 import org.apache.spark.ml.Transformer;
 import org.apache.spark.ml.linalg.VectorUDT;

diff --git a/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java b/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java
@@ -1,6 +1,6 @@
 package sparksoniq.spark.ml;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.spark.ml.Transformer;
 import org.apache.spark.ml.linalg.VectorUDT;
 import org.apache.spark.ml.param.ParamMap;

diff --git a/src/test/java/iq/base/AnnotationsTestsBase.java b/src/test/java/iq/base/AnnotationsTestsBase.java
@@ -20,7 +20,7 @@
 
 package iq.base;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.junit.Assert;
 import org.rumbledb.api.Item;
 import org.rumbledb.api.Rumble;

diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq
@@ -1,4 +1,4 @@
-(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :)
+(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :)
 let $vector-assembler := get-transformer("VectorAssembler")
 let $training-data := (
     {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1},

diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq
@@ -1,4 +1,4 @@
-(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :)
+(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :)
 let $vector-assembler := get-transformer("VectorAssembler")(?, { "inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features" })
 let $training-data := (
     {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1},

diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq
@@ -1,4 +1,4 @@
-(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :)
+(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :)
 
 declare type local:mytype as {
  "id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"