Merge branch 'main' into add-eq-props-projection

.github/workflows/rust.yml

@@ -492,7 +492,11 @@ jobs:
           ./dev/update_config_docs.sh
           git diff --exit-code
 
-  # Verify MSRV for the crates which are directly used by other projects.
+  # Verify MSRV for the crates which are directly used by other projects:
+  # - datafusion
+  # - datafusion-substrait
+  # - datafusion-proto
+  # - datafusion-cli
   msrv:
     name: Verify MSRV (Min Supported Rust Version)
     runs-on: ubuntu-latest

Cargo.toml

@@ -28,7 +28,7 @@ homepage = "https://github.com/apache/arrow-datafusion"
 license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/apache/arrow-datafusion"
-rust-version = "1.70"
+rust-version = "1.72"
 version = "35.0.0"
 
 [workspace.dependencies]
@@ -58,7 +58,7 @@ datafusion-sql = { path = "datafusion/sql", version = "35.0.0" }
 datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "35.0.0" }
 datafusion-substrait = { path = "datafusion/substrait", version = "35.0.0" }
 doc-comment = "0.3"
-env_logger = "0.10"
+env_logger = "0.11"
 futures = "0.3"
 half = "2.2.1"
 indexmap = "2.0.0"

benchmarks/Cargo.toml

@@ -24,7 +24,7 @@ authors = ["Apache Arrow <[email protected]>"]
 homepage = "https://github.com/apache/arrow-datafusion"
 repository = "https://github.com/apache/arrow-datafusion"
 license = "Apache-2.0"
-rust-version = "1.70"
+rust-version = { workspace = true }
 
 [features]
 ci = []

datafusion-cli/Cargo.toml

@@ -25,7 +25,8 @@ keywords = ["arrow", "datafusion", "query", "sql"]
 license = "Apache-2.0"
 homepage = "https://github.com/apache/arrow-datafusion"
 repository = "https://github.com/apache/arrow-datafusion"
-rust-version = "1.70"
+# Specify MSRV here as `cargo msrv` doesn't support workspace version
+rust-version = "1.72"
 readme = "README.md"
 
 [dependencies]

datafusion/common/src/scalar.rs

@@ -1483,7 +1483,9 @@ impl ScalarValue {
             DataType::Interval(IntervalUnit::MonthDayNano) => {
                 build_array_primitive!(IntervalMonthDayNanoArray, IntervalMonthDayNano)
             }
-            DataType::List(_) | DataType::LargeList(_) => build_list_array(scalars)?,
+            DataType::List(_)
+            | DataType::LargeList(_)
+            | DataType::FixedSizeList(_, _) => build_list_array(scalars)?,
             DataType::Struct(fields) => {
                 // Initialize a Vector to store the ScalarValues for each column
                 let mut columns: Vec<Vec<ScalarValue>> =
@@ -1595,7 +1597,6 @@ impl ScalarValue {
             | DataType::Time64(TimeUnit::Second)
             | DataType::Time64(TimeUnit::Millisecond)
             | DataType::Duration(_)
-            | DataType::FixedSizeList(_, _)
             | DataType::Union(_, _)
             | DataType::Map(_, _)
             | DataType::RunEndEncoded(_, _) => {

datafusion/common/src/utils.rs

@@ -440,9 +440,9 @@ pub fn arrays_into_list_array(
 /// ```
 pub fn base_type(data_type: &DataType) -> DataType {
     match data_type {
-        DataType::List(field) | DataType::LargeList(field) => {
-            base_type(field.data_type())
-        }
+        DataType::List(field)
+        | DataType::LargeList(field)
+        | DataType::FixedSizeList(field, _) => base_type(field.data_type()),
         _ => data_type.to_owned(),
     }
 }
@@ -464,31 +464,23 @@ pub fn coerced_type_with_base_type_only(
     base_type: &DataType,
 ) -> DataType {
     match data_type {
-        DataType::List(field) => {
-            let data_type = match field.data_type() {
-                DataType::List(_) => {
-                    coerced_type_with_base_type_only(field.data_type(), base_type)
-                }
-                _ => base_type.to_owned(),
-            };
+        DataType::List(field) | DataType::FixedSizeList(field, _) => {
+            let field_type =
+                coerced_type_with_base_type_only(field.data_type(), base_type);
 
             DataType::List(Arc::new(Field::new(
                 field.name(),
-                data_type,
+                field_type,
                 field.is_nullable(),
             )))
         }
         DataType::LargeList(field) => {
-            let data_type = match field.data_type() {
-                DataType::LargeList(_) => {
-                    coerced_type_with_base_type_only(field.data_type(), base_type)
-                }
-                _ => base_type.to_owned(),
-            };
+            let field_type =
+                coerced_type_with_base_type_only(field.data_type(), base_type);
 
             DataType::LargeList(Arc::new(Field::new(
                 field.name(),
-                data_type,
+                field_type,
                 field.is_nullable(),
             )))
         }
@@ -497,6 +489,32 @@ pub fn coerced_type_with_base_type_only(
     }
 }
 
+/// Recursively coerce and `FixedSizeList` elements to `List`
+pub fn coerced_fixed_size_list_to_list(data_type: &DataType) -> DataType {
+    match data_type {
+        DataType::List(field) | DataType::FixedSizeList(field, _) => {
+            let field_type = coerced_fixed_size_list_to_list(field.data_type());
+
+            DataType::List(Arc::new(Field::new(
+                field.name(),
+                field_type,
+                field.is_nullable(),
+            )))
+        }
+        DataType::LargeList(field) => {
+            let field_type = coerced_fixed_size_list_to_list(field.data_type());
+
+            DataType::LargeList(Arc::new(Field::new(
+                field.name(),
+                field_type,
+                field.is_nullable(),
+            )))
+        }
+
+        _ => data_type.clone(),
+    }
+}
+
 /// Compute the number of dimensions in a list data type.
 pub fn list_ndims(data_type: &DataType) -> u64 {
     match data_type {

datafusion/core/Cargo.toml

@@ -27,7 +27,10 @@ homepage = { workspace = true }
 repository = { workspace = true }
 license = { workspace = true }
 authors = { workspace = true }
-rust-version = "1.70"
+# Specify MSRV here as `cargo msrv` doesn't support workspace version and fails with
+# "Unable to find key 'package.rust-version' (or 'package.metadata.msrv') in 'arrow-datafusion/Cargo.toml'"
+# https://github.com/foresterre/cargo-msrv/issues/590
+rust-version = "1.72"
 
 [lib]
 name = "datafusion"