Added spark 3.5 to compile

hablapps · Mar 8, 2024 · a612451 · a612451
1 parent be8f352
commit a612451
Show file tree

Hide file tree

Showing 37 changed files with 68 additions and 26 deletions.
diff --git a/build.sbt b/build.sbt
@@ -4,12 +4,13 @@ import scala.language.postfixOps
 
 val stableVersion = "0.0.7"
 
-val sparkDefaultShortVersion = "3.4"
+val sparkDefaultShortVersion = "3.5"
 val spark30Version           = "3.0.3"
 val spark31Version           = "3.1.3"
 val spark32Version           = "3.2.3"
 val spark33Version           = "3.3.4"
 val spark34Version           = "3.4.2"
+val spark35Version           = "3.5.1"
 
 val versionRegex      = """^(.*)\.(.*)\.(.*)$""".r
 val versionRegexShort = """^(.*)\.(.*)$""".r
@@ -23,7 +24,8 @@ val parserSparkVersion: String => String = {
   case versionRegexShort("3", "2") => spark32Version
   case versionRegexShort("3", "3") => spark33Version
   case versionRegexShort("3", "4") => spark34Version
-  case versionRegex("3", b, c)       => s"3.$b.$c"
+  case versionRegexShort("3", "5") => spark35Version
+  case versionRegex("3", b, c)     => s"3.$b.$c"
 }
 
 val long2ShortVersion: String => String = { case versionRegex(a, b, _) =>
@@ -36,6 +38,7 @@ val scalaVersionSelect: String => List[String] = {
   case versionRegex("3", "2", _) => List(scala212, scala213)
   case versionRegex("3", "3", _) => List(scala212, scala213)
   case versionRegex("3", "4", _) => List(scala212, scala213)
+  case versionRegex("3", "5", _) => List(scala212, scala213)
 }
 
 ThisBuild / organization := "org.hablapps"

diff --git a/...4/scala/doric/syntax/ArrayColumns3x.scala → ...5/scala/doric/syntax/ArrayColumns3x.scala b/...4/scala/doric/syntax/ArrayColumns3x.scala → ...5/scala/doric/syntax/ArrayColumns3x.scala
@@ -3,7 +3,8 @@ package syntax
 
 import cats.implicits._
 import doric.types.CollectionType
-import org.apache.spark.sql.{Column, functions => f, Row}
+
+import org.apache.spark.sql.{Column, Row, functions => f}
 import org.apache.spark.sql.catalyst.expressions._
 
 trait ArrayColumns3x {

diff --git a/.../scala/doric/syntax/CommonColumns3x.scala → .../scala/doric/syntax/CommonColumns3x.scala b/.../scala/doric/syntax/CommonColumns3x.scala → .../scala/doric/syntax/CommonColumns3x.scala
diff --git a/...3_3.4/scala/doric/syntax/DStructs3x.scala → ...4_3.5/scala/doric/syntax/DStructs3x.scala b/...3_3.4/scala/doric/syntax/DStructs3x.scala → ...4_3.5/scala/doric/syntax/DStructs3x.scala
@@ -1,10 +1,11 @@
 package doric
 package syntax
 
+import scala.jdk.CollectionConverters._
+
 import doric.types.SparkType
-import org.apache.spark.sql.{Row, functions => f}
 
-import scala.jdk.CollectionConverters._
+import org.apache.spark.sql.{Row, functions => f}
 
 private[syntax] trait DStructs3x {
 

diff --git a/...3.4/scala/doric/syntax/MapColumns3x.scala → ...3.5/scala/doric/syntax/MapColumns3x.scala b/...3.4/scala/doric/syntax/MapColumns3x.scala → ...3.5/scala/doric/syntax/MapColumns3x.scala
@@ -3,8 +3,9 @@ package syntax
 
 import cats.implicits._
 import doric.types.SparkType
-import org.apache.spark.sql.catalyst.expressions.{MapFilter, MapZipWith, TransformKeys, TransformValues}
+
 import org.apache.spark.sql.{Column, Row, functions => f}
+import org.apache.spark.sql.catalyst.expressions.{MapFilter, MapZipWith, TransformKeys, TransformValues}
 
 trait MapColumns3x {
 

diff --git a/...4/scala/doric/syntax/StringColumn3x.scala → ...5/scala/doric/syntax/StringColumn3x.scala b/...4/scala/doric/syntax/StringColumn3x.scala → ...5/scala/doric/syntax/StringColumn3x.scala
@@ -1,12 +1,13 @@
 package doric
 package syntax
 
+import scala.jdk.CollectionConverters._
+
 import cats.implicits._
+
+import org.apache.spark.sql.{Column, functions => f}
 import org.apache.spark.sql.catalyst.expressions.StringSplit
 import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.{Column, functions => f}
-
-import scala.jdk.CollectionConverters._
 
 trait StringColumn3x {
   implicit class StringOperationsSyntax3x(s: DoricColumn[String]) {

diff --git a/...ationalGroupedDatasetDoricInterface.scala → ...ationalGroupedDatasetDoricInterface.scala b/...ationalGroupedDatasetDoricInterface.scala → ...ationalGroupedDatasetDoricInterface.scala
diff --git a/...a/doric/syntax/AggregationColumns31.scala → ...a/doric/syntax/AggregationColumns31.scala b/...a/doric/syntax/AggregationColumns31.scala → ...a/doric/syntax/AggregationColumns31.scala
diff --git a/...scala/doric/syntax/BooleanColumns31.scala → ...scala/doric/syntax/BooleanColumns31.scala b/...scala/doric/syntax/BooleanColumns31.scala → ...scala/doric/syntax/BooleanColumns31.scala
diff --git a/...scala/doric/syntax/NumericColumns31.scala → ...scala/doric/syntax/NumericColumns31.scala b/...scala/doric/syntax/NumericColumns31.scala → ...scala/doric/syntax/NumericColumns31.scala
diff --git a/.../scala/doric/syntax/StringColumns31.scala → .../scala/doric/syntax/StringColumns31.scala b/.../scala/doric/syntax/StringColumns31.scala → .../scala/doric/syntax/StringColumns31.scala
diff --git a/...cala/doric/sqlExpressions/CustomAgg.scala → ...cala/doric/sqlExpressions/CustomAgg.scala b/...cala/doric/sqlExpressions/CustomAgg.scala → ...cala/doric/sqlExpressions/CustomAgg.scala
diff --git a/...a/doric/syntax/AggregationColumns32.scala → ...a/doric/syntax/AggregationColumns32.scala b/...a/doric/syntax/AggregationColumns32.scala → ...a/doric/syntax/AggregationColumns32.scala
diff --git a/.../scala/doric/syntax/BinaryColumns32.scala → .../scala/doric/syntax/BinaryColumns32.scala b/.../scala/doric/syntax/BinaryColumns32.scala → .../scala/doric/syntax/BinaryColumns32.scala
diff --git a/...scala/doric/syntax/NumericColumns32.scala → ...scala/doric/syntax/NumericColumns32.scala b/...scala/doric/syntax/NumericColumns32.scala → ...scala/doric/syntax/NumericColumns32.scala
diff --git a/...doric/types/SparkTypeLPI_I_Specific.scala → ...doric/types/SparkTypeLPI_I_Specific.scala b/...doric/types/SparkTypeLPI_I_Specific.scala → ...doric/types/SparkTypeLPI_I_Specific.scala
diff --git a/core/src/main/spark_3.5_mount/scala/doric/syntax/All.scala b/core/src/main/spark_3.5_mount/scala/doric/syntax/All.scala
@@ -0,0 +1,31 @@
+package doric.syntax
+
+private[doric] trait All
+    extends ArrayColumns
+    with TypeMatcher
+    with CommonColumns
+    with DStructs
+    with LiteralConversions
+    with MapColumns
+    with NumericColumns
+    with DateColumns
+    with TimestampColumns
+    with BooleanColumns
+    with StringColumns
+    with ControlStructures
+    with AggregationColumns
+    with CNameOps
+    with BinaryColumns
+    with Interpolators
+    with AggregationColumns31
+    with BooleanColumns31
+    with NumericColumns31
+    with NumericColumns32
+    with StringColumns31
+    with BinaryColumns32
+    with ArrayColumns3x
+    with CommonColumns3x
+    with MapColumns3x
+    with StringColumn3x
+    with AggregationColumns32
+    with DStructs3x
diff --git a/core/src/test/scala/doric/sem/ChildColumnNotFound.scala b/core/src/test/scala/doric/sem/ChildColumnNotFound.scala
@@ -9,7 +9,7 @@ object ChildColumnNotFound {
   ): SparkErrorWrapper = {
     SparkErrorWrapper(
       new Throwable(
-        if (!sparkSession.version.startsWith("3.4"))
+        if (!(sparkSession.version.startsWith("3.4") || sparkSession.version.startsWith("3.5")))
           s"No such struct field $expectedCol in ${foundCols.mkString(", ")}"
         else
           s"[FIELD_NOT_FOUND] No such struct field `$expectedCol` in ${foundCols

diff --git a/core/src/test/scala/doric/sem/ColumnNotFound.scala b/core/src/test/scala/doric/sem/ColumnNotFound.scala
@@ -10,7 +10,7 @@ object ColumnNotFound {
 
     SparkErrorWrapper(
       new Throwable(
-        if (!sparkSession.version.startsWith("3.4"))
+        if (!(sparkSession.version.startsWith("3.4") || sparkSession.version.startsWith("3.5")))
           s"""Cannot resolve column name "$expectedCol" among (${foundCols
               .mkString(", ")})"""
         else

diff --git a/core/src/test/scala/doric/sem/TransformOpsSpec.scala b/core/src/test/scala/doric/sem/TransformOpsSpec.scala
@@ -2,11 +2,12 @@ package doric
 package sem
 
 import doric.implicitConversions._
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.types.{DoubleType, IntegerType, StringType, TimestampType}
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.EitherValues
 
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.types.{DoubleType, IntegerType, StringType, TimestampType}
+
 class TransformOpsSpec
     extends DoricTestElements
     with Matchers
@@ -122,13 +123,13 @@ class TransformOpsSpec
           )
       }
       error.getMessage should startWith(
-        if (!spark.version.startsWith("3.4"))
+        if (!(spark.version.startsWith("3.4") || spark.version.startsWith("3.5")))
           "Found duplicate column(s) in given column names:"
         else
           "[COLUMN_ALREADY_EXISTS] The column `a` already exists. Consider to choose another name or rename the existing column."
       )
       error.getMessage should include("`a`")
-      if (!spark.version.startsWith("3.4")) {
+      if (!(spark.version.startsWith("3.4") || spark.version.startsWith("3.5"))) {
         error.getMessage should include("`b`")
       }
     }

diff --git a/core/src/test/scala/doric/syntax/StringColumnsSpec.scala b/core/src/test/scala/doric/syntax/StringColumnsSpec.scala
@@ -322,7 +322,7 @@ class StringColumnsSpec extends DoricTestElements {
     }
 
     it("should raise an error if group > regex group result") {
-      intercept[java.lang.IllegalArgumentException] {
+      intercept[java.lang.RuntimeException] {
         df.withColumn(
           "res",
           colString("col1").regexpExtract("(\\d+)-(\\d+)".lit, 4.lit)
@@ -481,7 +481,7 @@ class StringColumnsSpec extends DoricTestElements {
       df.testColumns2("col1", "")(
         (str, pattern) => colString(str).split(pattern.lit),
         (str, pattern) => f.split(f.col(str), pattern),
-        if (!spark.version.startsWith("3.4"))
+        if (!(spark.version.startsWith("3.4") || spark.version.startsWith("3.5")))
           List(
             Array("h", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d", ""),
             Array("1", "2", "3", "4", "5", ""),

diff --git a/...DoricTestElementsCompanion_Specific.scala → ...DoricTestElementsCompanion_Specific.scala b/...DoricTestElementsCompanion_Specific.scala → ...DoricTestElementsCompanion_Specific.scala
diff --git a/...ala/doric/syntax/ArrayColumns3xSpec.scala → ...ala/doric/syntax/ArrayColumns3xSpec.scala b/...ala/doric/syntax/ArrayColumns3xSpec.scala → ...ala/doric/syntax/ArrayColumns3xSpec.scala
@@ -2,11 +2,12 @@ package doric
 package syntax
 
 import doric.testUtilities.data.User
-import org.apache.spark.sql.catalyst.expressions.ArraySort
-import org.apache.spark.sql.{Column, Row, functions => f}
 import org.scalatest.EitherValues
 import org.scalatest.matchers.should.Matchers
 
+import org.apache.spark.sql.{Column, Row, functions => f}
+import org.apache.spark.sql.catalyst.expressions.ArraySort
+
 class ArrayColumns3xSpec
     extends DoricTestElements
     with EitherValues

diff --git a/...la/doric/syntax/CommonColumns3xSpec.scala → ...la/doric/syntax/CommonColumns3xSpec.scala b/...la/doric/syntax/CommonColumns3xSpec.scala → ...la/doric/syntax/CommonColumns3xSpec.scala
diff --git a/...4/scala/doric/syntax/DStructs3xSpec.scala → ...5/scala/doric/syntax/DStructs3xSpec.scala b/...4/scala/doric/syntax/DStructs3xSpec.scala → ...5/scala/doric/syntax/DStructs3xSpec.scala
@@ -1,10 +1,11 @@
 package doric.syntax
 
-import doric.{DoricTestElements, colStruct}
-import org.apache.spark.sql.{functions => f}
+import scala.jdk.CollectionConverters._
 
+import doric.{colStruct, DoricTestElements}
 import java.sql.Timestamp
-import scala.jdk.CollectionConverters._
+
+import org.apache.spark.sql.{functions => f}
 
 class DStructs3xSpec extends DoricTestElements {
 

diff --git a/...cala/doric/syntax/DateColumns3xSpec.scala → ...cala/doric/syntax/DateColumns3xSpec.scala b/...cala/doric/syntax/DateColumns3xSpec.scala → ...cala/doric/syntax/DateColumns3xSpec.scala
diff --git a/...scala/doric/syntax/MapColumns3xSpec.scala → ...scala/doric/syntax/MapColumns3xSpec.scala b/...scala/doric/syntax/MapColumns3xSpec.scala → ...scala/doric/syntax/MapColumns3xSpec.scala
diff --git a/...la/doric/syntax/StringColumns3xSpec.scala → ...la/doric/syntax/StringColumns3xSpec.scala b/...la/doric/syntax/StringColumns3xSpec.scala → ...la/doric/syntax/StringColumns3xSpec.scala
@@ -1,13 +1,14 @@
 package doric
 package syntax
 
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.{Row, functions => f}
+import scala.jdk.CollectionConverters._
+
+import java.sql.Timestamp
 import org.scalatest.EitherValues
 import org.scalatest.matchers.should.Matchers
 
-import java.sql.Timestamp
-import scala.jdk.CollectionConverters._
+import org.apache.spark.sql.{Row, functions => f}
+import org.apache.spark.sql.types.StructType
 
 class StringColumns3xSpec
     extends DoricTestElements

diff --git a/...doric/syntax/TimestampColumns3xSpec.scala → ...doric/syntax/TimestampColumns3xSpec.scala b/...doric/syntax/TimestampColumns3xSpec.scala → ...doric/syntax/TimestampColumns3xSpec.scala
diff --git a/...ric/syntax/AggregationColumns31Spec.scala → ...ric/syntax/AggregationColumns31Spec.scala b/...ric/syntax/AggregationColumns31Spec.scala → ...ric/syntax/AggregationColumns31Spec.scala
diff --git a/...a/doric/syntax/BooleanColumns31Spec.scala → ...a/doric/syntax/BooleanColumns31Spec.scala b/...a/doric/syntax/BooleanColumns31Spec.scala → ...a/doric/syntax/BooleanColumns31Spec.scala
diff --git a/....4/scala/doric/syntax/Numeric31Spec.scala → ....5/scala/doric/syntax/Numeric31Spec.scala b/....4/scala/doric/syntax/Numeric31Spec.scala → ....5/scala/doric/syntax/Numeric31Spec.scala
diff --git a/...la/doric/syntax/StringColumns31Spec.scala → ...la/doric/syntax/StringColumns31Spec.scala b/...la/doric/syntax/StringColumns31Spec.scala → ...la/doric/syntax/StringColumns31Spec.scala
diff --git a/...ric/syntax/AggregationColumns32Spec.scala → ...ric/syntax/AggregationColumns32Spec.scala b/...ric/syntax/AggregationColumns32Spec.scala → ...ric/syntax/AggregationColumns32Spec.scala
diff --git a/...s/DeserializeSparkTypeSpec_Specific.scala → ...s/DeserializeSparkTypeSpec_Specific.scala b/...s/DeserializeSparkTypeSpec_Specific.scala → ...s/DeserializeSparkTypeSpec_Specific.scala
diff --git a/...c/types/PrimitiveTypesSpec_Specific.scala → ...c/types/PrimitiveTypesSpec_Specific.scala b/...c/types/PrimitiveTypesSpec_Specific.scala → ...c/types/PrimitiveTypesSpec_Specific.scala
diff --git a/...pes/SerializeSparkTypeSpec_Specific.scala → ...pes/SerializeSparkTypeSpec_Specific.scala b/...pes/SerializeSparkTypeSpec_Specific.scala → ...pes/SerializeSparkTypeSpec_Specific.scala