From 604837aa28c8e9b614b664da4d51b946411fd547 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 26 Sep 2023 14:16:12 -0400 Subject: [PATCH 1/3] fix ndarray diff comparison --- asdf/_tests/commands/tests/data/blocks.diff | 2 +- .../tests/data/simple_inline_array.diff | 2 +- asdf/commands/diff.py | 24 ++++++++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/asdf/_tests/commands/tests/data/blocks.diff b/asdf/_tests/commands/tests/data/blocks.diff index 55978d20f..9f04a8167 100644 --- a/asdf/_tests/commands/tests/data/blocks.diff +++ b/asdf/_tests/commands/tests/data/blocks.diff @@ -10,4 +10,4 @@ tree: - > 9000 < 10000 - ndarrays differ by shape, datatype and contents + ndarrays at "foobar" differ by shape, datatype and contents diff --git a/asdf/_tests/commands/tests/data/simple_inline_array.diff b/asdf/_tests/commands/tests/data/simple_inline_array.diff index ba2b7a18e..3e30e30c1 100644 --- a/asdf/_tests/commands/tests/data/simple_inline_array.diff +++ b/asdf/_tests/commands/tests/data/simple_inline_array.diff @@ -1 +1 @@ - ndarrays differ by contents + ndarrays at "array" differ by contents diff --git a/asdf/commands/diff.py b/asdf/commands/diff.py index d1e2f70c1..fa8776ad7 100644 --- a/asdf/commands/diff.py +++ b/asdf/commands/diff.py @@ -6,6 +6,7 @@ import sys import jmespath +import numpy as np from numpy import array_equal try: @@ -30,6 +31,7 @@ RESET = "" import asdf +from asdf.extension._serialization_context import BlockAccess from asdf.tagged import Tagged from asdf.util import human_list @@ -243,6 +245,16 @@ def print_dict_diff(diff_ctx, tree, node_list, keys, other): print_in_tree(diff_ctx, nodes, key_, other, use_marker=use_marker) +def _load_array(asdf_file, array_dict): + # the array_dict may not be tagged if the array is inline + # in this case just use what's in "data" + if not hasattr(array_dict, "_tag"): + return array_dict["data"] + conv = asdf_file.extension_manager.get_converter_for_type(np.ndarray) + sctx = asdf_file._create_serialization_context(BlockAccess.READ) + return conv.from_yaml_tree(array_dict, array_dict._tag, sctx) + + def compare_ndarrays(diff_ctx, array0, array1, keys): """Compares two ndarray objects""" if isinstance(array0, list): @@ -258,19 +270,15 @@ def compare_ndarrays(diff_ctx, array0, array1, keys): if array0.get(field) != array1.get(field): differences.append(field) - def get_flat(af, keys): - for k in keys: - af = af[k] - return af + value0 = _load_array(diff_ctx.asdf0, array0) + value1 = _load_array(diff_ctx.asdf1, array1) - array0 = get_flat(diff_ctx.asdf0, keys) - array1 = get_flat(diff_ctx.asdf1, keys) - if not array_equal(array0, array1): + if not array_equal(value0, value1): differences.append("contents") if differences: prefix = " " * (len(keys) + 1) - msg = f"ndarrays differ by {human_list(differences)}" + msg = f"ndarrays at \"{'.'.join(keys)}\" differ by {human_list(differences)}" diff_ctx.iostream.write(prefix + RED + msg + RESET_NEWLINE) From 72ef6bd8b1eeb96eceeb1ca1c86e588239a35f99 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 26 Sep 2023 14:23:14 -0400 Subject: [PATCH 2/3] add diff test compare 2 files that are identical except for block contents --- asdf/_tests/commands/tests/data/ndarray0.asdf | Bin 0 -> 745 bytes asdf/_tests/commands/tests/data/ndarray1.asdf | Bin 0 -> 745 bytes asdf/_tests/commands/tests/data/ndarrays.diff | 1 + asdf/_tests/commands/tests/test_diff.py | 6 ++++++ 4 files changed, 7 insertions(+) create mode 100644 asdf/_tests/commands/tests/data/ndarray0.asdf create mode 100644 asdf/_tests/commands/tests/data/ndarray1.asdf create mode 100644 asdf/_tests/commands/tests/data/ndarrays.diff diff --git a/asdf/_tests/commands/tests/data/ndarray0.asdf b/asdf/_tests/commands/tests/data/ndarray0.asdf new file mode 100644 index 0000000000000000000000000000000000000000..7285275155fc2f80b4730cdd93963c5e9a10caf2 GIT binary patch literal 745 zcmbVKv2N5r5KX;l_=Jt3CU0T%b( zL>E(x$7F97ozfi&JP!?+R5Fn@Avf8=C643Jom6Sz+T1}jGHsZYa;~{vhy8D(7I4e8 zKnN<*cf5f@>5$F}LEA}Uu@a?P3T;NTP?cEnT!eJL09b~un@cE~IZIX52`!YaIJEKp zh)`Pd3ei@0L|52Dsyd{;m13D#1!MndeiR&f-uS?0*~FXp-q9p@PYP*3;gHxt+<~Yq z&)96U)lf|;0X*Zty9E$(kCLP%H>T5P1BE3OujN7*U>ALGvJv@@-)Pyfan;tyZa=1U z?zZj@O--BQ5cZ1f5F2A3GWwCu@J+uu<686eu(nZ+PSI3tCpUyKMt&Yo&$qm-f4?VvI-K>}n_pjl|M>Xl)92ZjZy&ys9Dgs`^7`I;#YagT8u)&z_ZM7d+S>pC literal 0 HcmV?d00001 diff --git a/asdf/_tests/commands/tests/data/ndarray1.asdf b/asdf/_tests/commands/tests/data/ndarray1.asdf new file mode 100644 index 0000000000000000000000000000000000000000..8074106c7b9f883a164e21aaf855e9dbaf57dbbd GIT binary patch literal 745 zcmbVK!EV$r5bfDq_yb&3RS#^O?1n}(x0Dowx@`sB0|=qWn>dN3BwpE*g%#r7Q%^6$|qA4~|P z4X+Vxg(q~4J)~+v>RTz6iFGjbpB6{KbI+R|`YfAyGv7O!1z$-i4JaHEJBZI98p|^_ z+io>hlUe}JIPiV}ggl@mX~m7{_1Q>aNzEIX3j^$I2u`*lfAL!_dp54x8QJg0lrG%P z-La|ZavZ{5ksV@d?7xiuPiOeHUz2gI`DR?(s8*-wwN Date: Tue, 26 Sep 2023 14:32:25 -0400 Subject: [PATCH 3/3] update changelog --- CHANGES.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ebe858921..12293df83 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -25,6 +25,10 @@ The ASDF Standard is at v1.6.0 ``asdf.tags.core.Stream``, update block storage support for Converter and update internal block API [#1537] - Remove deprecated resolve_local_refs argument to load_schema [#1623] +- Move IntegerType to converter and drop cache of converted values. [#1527] +- Remove legacy extension API [#1637] +- Fix bug that left out the name of the arrays that differed + for ``asdftool diff`` comparisons [#1652] 2.15.2 (2023-09-29) ------------------- @@ -33,8 +37,6 @@ The ASDF Standard is at v1.6.0 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Add support for python 3.12 [#1641] -- Move IntegerType to converter and drop cache of converted values. [#1527] -- Remove legacy extension API [#1637] 2.15.1 (2023-08-07) -------------------