Spark 4.0.0 (preview1)

.github/workflows/maven.yml

@@ -12,10 +12,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Java 11
+    - name: Set up Java 17
       uses: actions/setup-java@v3
       with:
-        java-version: 11
+        java-version: 17
         distribution: adopt
     - name: Cache Maven packages
       uses: actions/cache@v3
@@ -55,10 +55,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Java 11
+    - name: Set up Java 17
       uses: actions/setup-java@v3
       with:
-        java-version: 11
+        java-version: 17
         distribution: adopt
     - name: Cache Maven packages
       uses: actions/cache@v3
@@ -85,10 +85,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Java 11
+    - name: Set up Java 17
       uses: actions/setup-java@v3
       with:
-        java-version: 11
+        java-version: 17
         distribution: adopt
     - name: Cache Maven packages
       uses: actions/cache@v3

pom.xml

@@ -40,33 +40,34 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.8.1</version>
+                <version>3.13.0</version>
                 <!-- Holy grail of eclipse configuration:  https://stackoverflow.com/a/49599478/10707488   -->
                 <configuration>
                     <compilerId>eclipse</compilerId>
-                    <source>1.8</source>
-                    <target>1.8</target>
-                    <compilerArguments>
-                        <properties>${project.basedir}/org.eclipse.jdt.core.prefs</properties>
-                    </compilerArguments>
+                    <source>11</source>
+                    <target>11</target>
+                    <compilerArgs>
+                        <arg>-properties</arg>
+                        <arg>${project.basedir}/org.eclipse.jdt.core.prefs</arg>
+                    </compilerArgs>
                 </configuration>
                 <dependencies>
                     <dependency>
                         <groupId>org.codehaus.plexus</groupId>
                         <artifactId>plexus-compiler-eclipse</artifactId>
-                        <version>2.8.5</version>
+                        <version>2.13.0</version>
                     </dependency>
                     <dependency>
                         <groupId>org.eclipse.jdt</groupId>
                         <artifactId>ecj</artifactId>
-                        <version>3.20.0</version>
+                        <version>3.36.0</version>
                     </dependency>
                 </dependencies>
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
-                <version>3.1.2</version>
+                <version>3.4.2</version>
                 <executions>
                     <execution>
                         <goals>
@@ -107,7 +108,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <version>3.1.1</version>
+                <version>3.7.0</version>
                 <configuration>
                     <excludePackageNames>org.rumbledb.cli:org.rumbledb.config:sparksoniq.*:sparksoniq</excludePackageNames>
                 </configuration>
@@ -193,6 +194,14 @@
                     </java>
                 </configuration>
             </plugin>
+	    <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+	    <version>3.3.0</version>
+            <configuration>
+                <argLine>--add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED</argLine>
+            </configuration>
+        </plugin>
         </plugins>
     </build>
 
@@ -209,37 +218,37 @@
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-core_2.12</artifactId>
-            <version>3.4.2</version>
+            <artifactId>spark-core_2.13</artifactId>
+            <version>4.0.0-preview2</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-sql_2.12</artifactId>
-            <version>3.4.2</version>
+            <artifactId>spark-sql_2.13</artifactId>
+            <version>4.0.0-preview2</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-mllib_2.12</artifactId>
-            <version>3.4.2</version>
+            <artifactId>spark-mllib_2.13</artifactId>
+            <version>4.0.0-preview2</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
              <artifactId>hadoop-aws</artifactId>
-            <version>3.3.2</version>
+            <version>3.3.6</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-avro_2.12</artifactId>
-            <version>3.4.2</version>
+            <artifactId>spark-avro_2.13</artifactId>
+            <version>4.0.0-preview2</version>
         </dependency>
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
-            <version>4.9.3</version>
+            <version>4.13.1</version>
         </dependency>
         <dependency>
             <groupId>org.jline</groupId>
@@ -249,7 +258,7 @@
         <dependency>
             <groupId>com.esotericsoftware</groupId>
             <artifactId>kryo</artifactId>
-            <version>4.0.2</version>
+            <version>5.6.0</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
@@ -265,7 +274,7 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.12.0</version>
+            <version>3.14.0</version>
         </dependency>
         <dependency>
             <groupId>commons-net</groupId>
@@ -300,12 +309,12 @@
         <dependency>
             <groupId>com.fasterxml.jackson.dataformat</groupId>
             <artifactId>jackson-dataformat-yaml</artifactId>
-            <version>2.13.4</version>
+            <version>2.17.1</version>
         </dependency>
         <dependency>
             <groupId>io.delta</groupId>
-            <artifactId>delta-core_2.12</artifactId>
-            <version>2.4.0</version>
+            <artifactId>delta-spark_2.13</artifactId>
+            <version>4.0.0rc1</version>
         </dependency>
     </dependencies>
 

src/main/java/org/rumbledb/api/Rumble.java

@@ -5,6 +5,8 @@
 import org.rumbledb.context.DynamicContext;
 import org.rumbledb.expressions.module.MainModule;
 import org.rumbledb.runtime.RuntimeIterator;
+import org.rumbledb.runtime.update.PendingUpdateList;
+
 import sparksoniq.spark.SparkSessionManager;
 
 import java.io.IOException;
@@ -72,6 +74,11 @@ public SequenceOfItems runQuery(URI location) throws IOException {
             this.configuration
         );
 
+        if (iterator.isUpdating()) {
+            PendingUpdateList pul = iterator.getPendingUpdateList(dynamicContext);
+            pul.applyUpdates(iterator.getMetadata());
+        }
+
         return new SequenceOfItems(iterator, dynamicContext, this.configuration);
     }
 

src/main/java/org/rumbledb/cli/Main.java

@@ -37,12 +37,16 @@ public class Main {
 
     public static void main(String[] args) throws IOException {
         String javaVersion = System.getProperty("java.version");
-        if (!javaVersion.startsWith("11.") && !javaVersion.startsWith("17.")) {
-            System.err.println("[Error] RumbleDB requires Java 11 or 17.");
+        if (
+            !javaVersion.startsWith("11")
+                && !javaVersion.startsWith("17")
+                && !javaVersion.startsWith("21")
+        ) {
+            System.err.println("[Error] RumbleDB requires Java 11, 17 or 21 (17 being the default Spark 4 version).");
             System.err.println("Your Java version: " + System.getProperty("java.version"));
-            System.err.println("You can download Java 11 or 17 from https://adoptium.net/");
+            System.err.println("You can download Java 11, 17 or 21 from https://adoptium.net/");
             System.err.println(
-                "If you do have Java 11 or 17, but the wrong version appears above, then it means you need to set your JAVA_HOME environment variable properly to point to Java 11 or 17."
+                "If you do have Java 11, 17 or 21, but the wrong version appears above, then it means you need to set your JAVA_HOME environment variable properly to point to Java 11, 17 or 21."
             );
             System.exit(43);
         }

src/main/java/org/rumbledb/items/parsing/ItemParser.java

@@ -46,13 +46,13 @@
 import org.rumbledb.types.BuiltinTypesCatalogue;
 import org.rumbledb.types.FieldDescriptor;
 import org.rumbledb.types.ItemType;
-import scala.collection.mutable.WrappedArray;
+import scala.collection.immutable.ArraySeq;
+import scala.collection.Iterator;
 import sparksoniq.spark.SparkSessionManager;
 
 import java.io.IOException;
 import java.io.Serializable;
 import java.io.StringReader;
-import java.lang.reflect.Array;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.sql.Date;
@@ -589,9 +589,14 @@ private static Item convertValueToItem(
                 }
             } else {
                 @SuppressWarnings("unchecked")
-                Object arrayObject = ((WrappedArray<Object>) o).array();
-                for (int index = 0; index < Array.getLength(arrayObject); index++) {
-                    Object value = Array.get(arrayObject, index);
+                Iterator<Object> iterator = null;
+                if (o instanceof scala.collection.mutable.ArraySeq) {
+                    iterator = ((scala.collection.mutable.ArraySeq<Object>) o).iterator();
+                } else {
+                    iterator = ((ArraySeq<Object>) o).iterator();
+                }
+                while (iterator.hasNext()) {
+                    Object value = iterator.next();
                     members.add(convertValueToItem(value, dataType, metadata, memberType));
                 }
             }

src/main/java/org/rumbledb/parser/JsoniqBaseVisitor.java

@@ -1,4 +1,4 @@
-// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.9.3
+// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.13.1
 
 // Java header
 package org.rumbledb.parser;
@@ -13,6 +13,7 @@
  * @param <T> The return type of the visit operation. Use {@link Void} for
  * operations with no return type.
  */
+@SuppressWarnings("CheckReturnValue")
 public class JsoniqBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements JsoniqVisitor<T> {
 	/**
 	 * {@inheritDoc}