Skip to content

Commit

Permalink
Decode all stored bytes as string. Assert ref spec version 1.
Browse files Browse the repository at this point in the history
  • Loading branch information
emfdavid committed Dec 1, 2023
1 parent cf63ed4 commit 229c3da
Showing 1 changed file with 11 additions and 3 deletions.
14 changes: 11 additions & 3 deletions kerchunk/grib2.py
Original file line number Diff line number Diff line change
Expand Up @@ -396,15 +396,13 @@ def grib_tree(
# TODO allow passing a LazyReferenceMapper as output?
zarr_store = {}
zroot = zarr.open_group(store=zarr_store)
result = dict(refs=zarr_store)

aggregations: Dict[str, List] = defaultdict(list)
aggregation_dims: Dict[str, Set] = defaultdict(set)

unknown_counter = 0
for msg_ind, group in enumerate(message_groups):
if "version" not in result:
result["version"] = group["version"]
assert group["version"] == 1

gattrs = ujson.loads(group["refs"][".zattrs"])
coordinates = gattrs["coordinates"].split(" ")
Expand Down Expand Up @@ -518,6 +516,16 @@ def grib_tree(
if key not in [".zattrs", ".zgroup"]:
zarr_store[f"{path}/{key}"] = value

# Force all stored values to decode as string, not bytes. String should be correct.
# ujson will reject bytes values by default.
# Using 'reject_bytes=False' one write would fail an equality check on read.
zarr_store = {
key: (val.decode() if isinstance(val, bytes) else val)
for key, val in zarr_store.items()
}
# TODO handle other kerchunk reference spec versions?
result = dict(refs=zarr_store, version=1)

return result


Expand Down

0 comments on commit 229c3da

Please sign in to comment.