From 1fa777e347a6b15319d81abd13f899a638d2bccd Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Tue, 10 Dec 2024 11:51:59 -0500 Subject: [PATCH] Temporarily fix virtual tests (#463) --- icechunk-python/tests/test_virtual_ref.py | 96 ++++++++++++++++++++--- icechunk/tests/test_virtual_refs.rs | 24 +++--- 2 files changed, 97 insertions(+), 23 deletions(-) diff --git a/icechunk-python/tests/test_virtual_ref.py b/icechunk-python/tests/test_virtual_ref.py index 66791624..865500b4 100644 --- a/icechunk-python/tests/test_virtual_ref.py +++ b/icechunk-python/tests/test_virtual_ref.py @@ -107,23 +107,97 @@ async def test_from_s3_public_virtual_refs(tmpdir): ), ) root = zarr.Group.from_store(store=store, zarr_format=3) - depth = root.require_array( - name="depth", shape=((10,)), chunk_shape=((10,)), dtype="float64" + year = root.require_array( + name="year", shape=((72,)), chunk_shape=((72,)), dtype="float32" ) store.set_virtual_ref( - "depth/c/0", - "s3://noaa-nos-ofs-pds/dbofs/netcdf/202410/dbofs.t00z.20241009.fields.f030.nc", - offset=119339, - length=80, + "year/c/0", + "s3://earthmover-sample-data/netcdf/oscar_vel2018.nc", + offset=22306, + length=288, ) nodes = [n async for n in store.list()] - assert "depth/c/0" in nodes + assert "year/c/0" in nodes - depth_values = depth[:] - assert len(depth_values) == 10 + year_values = year[:] + assert len(year_values) == 72 actual_values = np.array( - [-0.95, -0.85, -0.75, -0.65, -0.55, -0.45, -0.35, -0.25, -0.15, -0.05] + [ + 2018.0, + 2018.0139, + 2018.0278, + 2018.0416, + 2018.0555, + 2018.0695, + 2018.0834, + 2018.0972, + 2018.1111, + 2018.125, + 2018.1389, + 2018.1528, + 2018.1666, + 2018.1805, + 2018.1945, + 2018.2084, + 2018.2222, + 2018.2361, + 2018.25, + 2018.2639, + 2018.2778, + 2018.2916, + 2018.3055, + 2018.3195, + 2018.3334, + 2018.3472, + 2018.3611, + 2018.375, + 2018.3889, + 2018.4028, + 2018.4166, + 2018.4305, + 2018.4445, + 2018.4584, + 2018.4722, + 2018.4861, + 2018.5, + 2018.5139, + 2018.5278, + 2018.5416, + 2018.5555, + 2018.5695, + 2018.5834, + 2018.5972, + 2018.6111, + 2018.625, + 2018.6389, + 2018.6528, + 2018.6666, + 2018.6805, + 2018.6945, + 2018.7084, + 2018.7222, + 2018.7361, + 2018.75, + 2018.7639, + 2018.7778, + 2018.7916, + 2018.8055, + 2018.8195, + 2018.8334, + 2018.8472, + 2018.8611, + 2018.875, + 2018.8889, + 2018.9028, + 2018.9166, + 2018.9305, + 2018.9445, + 2018.9584, + 2018.9722, + 2018.9861, + ], + dtype="float32", ) - assert np.allclose(depth_values, actual_values) + assert np.allclose(year_values, actual_values) diff --git a/icechunk/tests/test_virtual_refs.rs b/icechunk/tests/test_virtual_refs.rs index 13bbb418..b3980dae 100644 --- a/icechunk/tests/test_virtual_refs.rs +++ b/icechunk/tests/test_virtual_refs.rs @@ -429,27 +429,27 @@ mod tests { .await .unwrap(); - let zarr_meta = Bytes::copy_from_slice(br#"{"zarr_format":3,"node_type":"array","attributes":{"foo":42},"shape":[10],"data_type":"float64","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[10]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value": 0.0,"codecs":[{"name":"mycodec","configuration":{"foo":42}}],"storage_transformers":[],"dimension_names":["depth"]}"#); - store.set("depth/zarr.json", zarr_meta.clone()).await.unwrap(); + let zarr_meta = Bytes::copy_from_slice(br#"{"zarr_format":3,"node_type":"array","attributes":{"foo":42},"shape":[72],"data_type":"float32","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[72]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value": 0.0,"codecs":[{"name":"mycodec","configuration":{"foo":42}}],"storage_transformers":[],"dimension_names":["year"]}"#); + store.set("year/zarr.json", zarr_meta.clone()).await.unwrap(); let ref2 = VirtualChunkRef { location: VirtualChunkLocation::from_absolute_path( - "s3://noaa-nos-ofs-pds/dbofs/netcdf/202410/dbofs.t00z.20241009.fields.f030.nc", + "s3://earthmover-sample-data/netcdf/oscar_vel2018.nc", )?, - offset: 119339, - length: 80, + offset: 22306, + length: 288, }; - store.set_virtual_ref("depth/c/0", ref2).await?; + store.set_virtual_ref("year/c/0", ref2).await?; - let chunk = store.get("depth/c/0", &ByteRange::ALL).await.unwrap(); - assert_eq!(chunk.len(), 80); + let chunk = store.get("year/c/0", &ByteRange::ALL).await.unwrap(); + assert_eq!(chunk.len(), 288); - let second_depth = f64::from_le_bytes(chunk[8..16].try_into().unwrap()); - assert!(second_depth - -0.85 < 0.000001); + let second_year = f32::from_le_bytes(chunk[4..8].try_into().unwrap()); + assert!(second_year - 2018.0139 < 0.000001); - let last_depth = f64::from_le_bytes(chunk[(80 - 8)..].try_into().unwrap()); - assert!(last_depth - -0.05 < 0.000001); + let last_year = f32::from_le_bytes(chunk[(288 - 4)..].try_into().unwrap()); + assert!(last_year - 2018.9861 < 0.000001); Ok(()) }