perf: Make Parquet verify_dict_indices SIMD (#20623)

crates/polars-parquet/Cargo.toml

@@ -63,3 +63,4 @@ lz4_flex = ["dep:lz4_flex"]
 async = ["async-stream", "futures", "polars-parquet-format/async"]
 bloom_filter = ["xxhash-rust"]
 serde_types = ["serde"]
+simd = ["polars-compute/simd"]

crates/polars-parquet/src/arrow/read/deserialize/dictionary_encoded/mod.rs

@@ -179,6 +179,27 @@ fn no_more_bitpacked_values() -> ParquetError {
 }
 
 #[inline(always)]
+#[cfg(feature = "simd")]
+fn verify_dict_indices(indices: &[u32; 32], dict_size: usize) -> ParquetResult<()> {
+    // You would think that the compiler can do this itself, but it does not always do this
+    // properly. So we help it a bit.
+
+    use std::simd::cmp::SimdPartialOrd;
+    use std::simd::u32x32;
+
+    let dict_size = u32x32::splat(dict_size as u32);
+    let indices = u32x32::from_slice(indices);
+
+    let is_invalid = indices.simd_ge(dict_size);
+    if is_invalid.any() {
+        Err(oob_dict_idx())
+    } else {
+        Ok(())
+    }
+}
+
+#[inline(always)]
+#[cfg(not(feature = "simd"))]
 fn verify_dict_indices(indices: &[u32; 32], dict_size: usize) -> ParquetResult<()> {
     let mut is_valid = true;
     for &idx in indices {

crates/polars-parquet/src/lib.rs

@@ -1,3 +1,4 @@
+#![cfg_attr(feature = "simd", feature(portable_simd))]
 #![allow(clippy::len_without_is_empty)]
 pub mod arrow;
 pub use crate::arrow::{read, write};