Use pcodec bounded representation

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        toolchain: ["stable", "1.71", "beta", "nightly"]
+        toolchain: ["stable", "1.73", "beta", "nightly"]
         features: ["--features \"\"", "--all-features", "--no-default-features"]
     runs-on: ubuntu-latest
     steps:

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+ - **Breaking**: Bump MSRV to 1.73 (5 October, 2023)
+ - Bump `pco` (pcodec) to 0.2.1
+
 ## [0.12.5] - 2024-03-17
 
 ### Added

Cargo.toml

@@ -3,7 +3,7 @@ name = "zarrs"
 version = "0.12.5"
 authors = ["Lachlan Deakin <[email protected]>"]
 edition = "2021"
-rust-version = "1.71"
+rust-version = "1.73"
 description = "A library for the Zarr V3 storage format for multidimensional arrays and metadata"
 documentation = "https://docs.rs/zarrs"
 repository = "https://github.com/LDeakin/zarrs"
@@ -60,7 +60,7 @@ object_store = { version = "0.9.0", optional = true }
 opendal = { version = "0.45", optional = true }
 parking_lot = "0.12"
 pathdiff = "0.2"
-pco = { version = "0.1.3", optional = true }
+pco = { version = "0.2.1", optional = true }
 rayon = "1.6"
 rayon_iter_concurrent_limit = "0.2.0"
 reqwest = { version = "0.11", features = ["blocking"], optional = true }

src/array/codec/array_to_bytes/pcodec.rs

@@ -227,11 +227,13 @@ mod tests {
             ChunkRepresentation::new(chunk_shape, data_type, fill_value).unwrap();
         let bytes: Vec<u8> = (0..chunk_representation.size()).map(|s| s as u8).collect();
 
+        let max_encoded_size = codec.compute_encoded_size(&chunk_representation)?;
         let encoded = codec.encode(
             bytes.clone(),
             &chunk_representation,
             &CodecOptions::default(),
         )?;
+        assert!((encoded.len() as u64) <= max_encoded_size.size().unwrap());
         let decoded = codec
             .decode(encoded, &chunk_representation, &CodecOptions::default())
             .unwrap();
@@ -240,7 +242,6 @@ mod tests {
     }
 
     #[test]
-    #[cfg_attr(miri, ignore)]
     fn codec_pcodec_round_trip_u32() {
         codec_pcodec_round_trip_impl(
             &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()),
@@ -251,7 +252,6 @@ mod tests {
     }
 
     #[test]
-    #[cfg_attr(miri, ignore)]
     fn codec_pcodec_round_trip_u64() {
         codec_pcodec_round_trip_impl(
             &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()),
@@ -262,7 +262,6 @@ mod tests {
     }
 
     #[test]
-    #[cfg_attr(miri, ignore)]
     fn codec_pcodec_round_trip_i32() {
         codec_pcodec_round_trip_impl(
             &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()),
@@ -273,7 +272,6 @@ mod tests {
     }
 
     #[test]
-    #[cfg_attr(miri, ignore)]
     fn codec_pcodec_round_trip_i64() {
         codec_pcodec_round_trip_impl(
             &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()),
@@ -284,7 +282,6 @@ mod tests {
     }
 
     #[test]
-    #[cfg_attr(miri, ignore)]
     fn codec_pcodec_round_trip_f32() {
         codec_pcodec_round_trip_impl(
             &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()),
@@ -295,7 +292,6 @@ mod tests {
     }
 
     #[test]
-    #[cfg_attr(miri, ignore)]
     fn codec_pcodec_round_trip_f64() {
         codec_pcodec_round_trip_impl(
             &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()),
@@ -306,7 +302,6 @@ mod tests {
     }
 
     #[test]
-    #[cfg_attr(miri, ignore)]
     fn codec_pcodec_round_trip_complex64() {
         codec_pcodec_round_trip_impl(
             &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()),
@@ -317,7 +312,6 @@ mod tests {
     }
 
     #[test]
-    #[cfg_attr(miri, ignore)]
     fn codec_pcodec_round_trip_complex128() {
         codec_pcodec_round_trip_impl(
             &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()),
@@ -338,7 +332,6 @@ mod tests {
     }
 
     #[test]
-    #[cfg_attr(miri, ignore)]
     fn codec_pcodec_partial_decode() {
         let chunk_shape: ChunkShape = vec![4, 4].try_into().unwrap();
         let chunk_representation = ChunkRepresentation::new(
@@ -381,7 +374,6 @@ mod tests {
 
     #[cfg(feature = "async")]
     #[tokio::test]
-    #[cfg_attr(miri, ignore)]
     async fn codec_pcodec_async_partial_decode() {
         let chunk_shape: ChunkShape = vec![4, 4].try_into().unwrap();
         let chunk_representation = ChunkRepresentation::new(

src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs

@@ -1,4 +1,4 @@
-use pco::{ChunkConfig, FloatMultSpec, IntMultSpec, PagingSpec};
+use pco::{standalone::guarantee::file_size, ChunkConfig, FloatMultSpec, IntMultSpec, PagingSpec};
 
 use crate::{
     array::{
@@ -147,7 +147,7 @@ impl ArrayCodecTraits for PcodecCodec {
         let data_type = decoded_representation.data_type();
         macro_rules! pcodec_decode {
             ( $t:ty ) => {
-                pco::standalone::auto_decompress(encoded_value.as_slice())
+                pco::standalone::simple_decompress(encoded_value.as_slice())
                     .map(|bytes| transmute_to_bytes_vec::<$t>(bytes))
                     .map_err(|err| CodecError::Other(err.to_string()))
             };
@@ -211,9 +211,28 @@ impl ArrayToBytesCodecTraits for PcodecCodec {
 
     fn compute_encoded_size(
         &self,
-        _decoded_representation: &ChunkRepresentation,
+        decoded_representation: &ChunkRepresentation,
     ) -> Result<BytesRepresentation, CodecError> {
-        // FIXME: pcodec is likely bounded, but it doesn't have a nice API to figure out what the bounded size is
-        Ok(BytesRepresentation::UnboundedSize)
+        let data_type = decoded_representation.data_type();
+        let mut num_elements = decoded_representation.num_elements_usize();
+        if data_type == &DataType::Complex64 || data_type == &DataType::Complex128 {
+            num_elements *= 2;
+        }
+
+        let size = match data_type {
+            DataType::UInt32 | DataType::Int32 | DataType::Float32 | DataType::Complex64 => Ok(
+                file_size::<u32>(num_elements, &self.chunk_config.paging_spec)
+                    .map_err(|err| CodecError::from(err.to_string()))?,
+            ),
+            DataType::UInt64 | DataType::Int64 | DataType::Float64 | DataType::Complex128 => Ok(
+                file_size::<u64>(num_elements, &self.chunk_config.paging_spec)
+                    .map_err(|err| CodecError::from(err.to_string()))?,
+            ),
+            _ => Err(CodecError::UnsupportedDataType(
+                data_type.clone(),
+                IDENTIFIER.to_string(),
+            )),
+        }?;
+        Ok(BytesRepresentation::BoundedSize(size.try_into().unwrap()))
     }
 }

src/array/codec/array_to_bytes/pcodec/pcodec_partial_decoder.rs

@@ -51,9 +51,10 @@ fn do_partial_decode(
         Some(decoded_value) => {
             macro_rules! pcodec_partial_decode {
                 ( $t:ty ) => {
-                    let decoded_chunk = pco::standalone::auto_decompress(decoded_value.as_slice())
-                        .map(|bytes| crate::array::transmute_to_bytes_vec::<$t>(bytes))
-                        .map_err(|err| CodecError::Other(err.to_string()))?;
+                    let decoded_chunk =
+                        pco::standalone::simple_decompress(decoded_value.as_slice())
+                            .map(|bytes| crate::array::transmute_to_bytes_vec::<$t>(bytes))
+                            .map_err(|err| CodecError::Other(err.to_string()))?;
                     for array_subset in decoded_regions {
                         let bytes_subset = array_subset
                             .extract_bytes(