Temporarily fix virtual tests (#463)

earth-mover · Dec 10, 2024 · 1fa777e · 1fa777e
1 parent 21b8b2b
commit 1fa777e
Show file tree

Hide file tree

Showing 2 changed files with 97 additions and 23 deletions.
diff --git a/icechunk-python/tests/test_virtual_ref.py b/icechunk-python/tests/test_virtual_ref.py
@@ -107,23 +107,97 @@ async def test_from_s3_public_virtual_refs(tmpdir):
         ),
     )
     root = zarr.Group.from_store(store=store, zarr_format=3)
-    depth = root.require_array(
-        name="depth", shape=((10,)), chunk_shape=((10,)), dtype="float64"
+    year = root.require_array(
+        name="year", shape=((72,)), chunk_shape=((72,)), dtype="float32"
     )
 
     store.set_virtual_ref(
-        "depth/c/0",
-        "s3://noaa-nos-ofs-pds/dbofs/netcdf/202410/dbofs.t00z.20241009.fields.f030.nc",
-        offset=119339,
-        length=80,
+        "year/c/0",
+        "s3://earthmover-sample-data/netcdf/oscar_vel2018.nc",
+        offset=22306,
+        length=288,
     )
 
     nodes = [n async for n in store.list()]
-    assert "depth/c/0" in nodes
+    assert "year/c/0" in nodes
 
-    depth_values = depth[:]
-    assert len(depth_values) == 10
+    year_values = year[:]
+    assert len(year_values) == 72
     actual_values = np.array(
-        [-0.95, -0.85, -0.75, -0.65, -0.55, -0.45, -0.35, -0.25, -0.15, -0.05]
+        [
+            2018.0,
+            2018.0139,
+            2018.0278,
+            2018.0416,
+            2018.0555,
+            2018.0695,
+            2018.0834,
+            2018.0972,
+            2018.1111,
+            2018.125,
+            2018.1389,
+            2018.1528,
+            2018.1666,
+            2018.1805,
+            2018.1945,
+            2018.2084,
+            2018.2222,
+            2018.2361,
+            2018.25,
+            2018.2639,
+            2018.2778,
+            2018.2916,
+            2018.3055,
+            2018.3195,
+            2018.3334,
+            2018.3472,
+            2018.3611,
+            2018.375,
+            2018.3889,
+            2018.4028,
+            2018.4166,
+            2018.4305,
+            2018.4445,
+            2018.4584,
+            2018.4722,
+            2018.4861,
+            2018.5,
+            2018.5139,
+            2018.5278,
+            2018.5416,
+            2018.5555,
+            2018.5695,
+            2018.5834,
+            2018.5972,
+            2018.6111,
+            2018.625,
+            2018.6389,
+            2018.6528,
+            2018.6666,
+            2018.6805,
+            2018.6945,
+            2018.7084,
+            2018.7222,
+            2018.7361,
+            2018.75,
+            2018.7639,
+            2018.7778,
+            2018.7916,
+            2018.8055,
+            2018.8195,
+            2018.8334,
+            2018.8472,
+            2018.8611,
+            2018.875,
+            2018.8889,
+            2018.9028,
+            2018.9166,
+            2018.9305,
+            2018.9445,
+            2018.9584,
+            2018.9722,
+            2018.9861,
+        ],
+        dtype="float32",
     )
-    assert np.allclose(depth_values, actual_values)
+    assert np.allclose(year_values, actual_values)
diff --git a/icechunk/tests/test_virtual_refs.rs b/icechunk/tests/test_virtual_refs.rs
@@ -429,27 +429,27 @@ mod tests {
             .await
             .unwrap();
 
-        let zarr_meta = Bytes::copy_from_slice(br#"{"zarr_format":3,"node_type":"array","attributes":{"foo":42},"shape":[10],"data_type":"float64","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[10]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value": 0.0,"codecs":[{"name":"mycodec","configuration":{"foo":42}}],"storage_transformers":[],"dimension_names":["depth"]}"#);
-        store.set("depth/zarr.json", zarr_meta.clone()).await.unwrap();
+        let zarr_meta = Bytes::copy_from_slice(br#"{"zarr_format":3,"node_type":"array","attributes":{"foo":42},"shape":[72],"data_type":"float32","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[72]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value": 0.0,"codecs":[{"name":"mycodec","configuration":{"foo":42}}],"storage_transformers":[],"dimension_names":["year"]}"#);
+        store.set("year/zarr.json", zarr_meta.clone()).await.unwrap();
 
         let ref2 = VirtualChunkRef {
             location: VirtualChunkLocation::from_absolute_path(
-                "s3://noaa-nos-ofs-pds/dbofs/netcdf/202410/dbofs.t00z.20241009.fields.f030.nc",
+                "s3://earthmover-sample-data/netcdf/oscar_vel2018.nc",
             )?,
-            offset: 119339,
-            length: 80,
+            offset: 22306,
+            length: 288,
         };
 
-        store.set_virtual_ref("depth/c/0", ref2).await?;
+        store.set_virtual_ref("year/c/0", ref2).await?;
 
-        let chunk = store.get("depth/c/0", &ByteRange::ALL).await.unwrap();
-        assert_eq!(chunk.len(), 80);
+        let chunk = store.get("year/c/0", &ByteRange::ALL).await.unwrap();
+        assert_eq!(chunk.len(), 288);
 
-        let second_depth = f64::from_le_bytes(chunk[8..16].try_into().unwrap());
-        assert!(second_depth - -0.85 < 0.000001);
+        let second_year = f32::from_le_bytes(chunk[4..8].try_into().unwrap());
+        assert!(second_year - 2018.0139 < 0.000001);
 
-        let last_depth = f64::from_le_bytes(chunk[(80 - 8)..].try_into().unwrap());
-        assert!(last_depth - -0.05 < 0.000001);
+        let last_year = f32::from_le_bytes(chunk[(288 - 4)..].try_into().unwrap());
+        assert!(last_year - 2018.9861 < 0.000001);
 
         Ok(())
     }