diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7de90d9f5..2f9f15702 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,7 @@ jobs: needs: [core, asdf-schemas] uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1 with: + python-version: "3.11" upload_to_pypi: false upload_to_anaconda: false test_extras: tests diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 8ccddcb15..bd747bd4e 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -41,6 +41,7 @@ jobs: default_python: '3.10' envs: | - linux: asdf-astropy + - linux: specutils stsci: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03d171a1a..9e3085351 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,13 +43,13 @@ repos: exclude: "asdf/(extern||_jsonschema)/.*" - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.292' + rev: 'v0.1.1' hooks: - id: ruff args: ["--fix"] - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.1 hooks: - id: black @@ -59,6 +59,6 @@ repos: - id: blacken-docs - repo: https://github.com/abravalheri/validate-pyproject - rev: "v0.14" + rev: "v0.15" hooks: - id: validate-pyproject diff --git a/CHANGES.rst b/CHANGES.rst index bac63847d..5293cd874 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,10 +1,10 @@ -3.0.1 (unreleased) +3.0.1 (2023-10-30) ------------------ The ASDF Standard is at v1.6.0 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- +- Fix bug in ``asdftool diff`` for arrays within a list [#1672] 3.0.0 (2023-10-16) ------------------ diff --git a/asdf/_tests/commands/tests/data/blocks.diff b/asdf/_tests/commands/tests/data/blocks.diff index 9f04a8167..da3edd2df 100644 --- a/asdf/_tests/commands/tests/data/blocks.diff +++ b/asdf/_tests/commands/tests/data/blocks.diff @@ -10,4 +10,5 @@ tree: - > 9000 < 10000 - ndarrays at "foobar" differ by shape, datatype and contents +> ndarrays differ by shape, datatype and contents +< ndarrays differ by shape, datatype and contents diff --git a/asdf/_tests/commands/tests/data/ndarray_in_list.diff b/asdf/_tests/commands/tests/data/ndarray_in_list.diff new file mode 100644 index 000000000..3a58bce13 --- /dev/null +++ b/asdf/_tests/commands/tests/data/ndarray_in_list.diff @@ -0,0 +1,9 @@ +< tree: +< list: +< - a: +< offset: +< 16 +< strides: +< - -8 +> ndarrays differ by contents +< ndarrays differ by contents diff --git a/asdf/_tests/commands/tests/data/ndarray_in_list0.asdf b/asdf/_tests/commands/tests/data/ndarray_in_list0.asdf new file mode 100644 index 000000000..0b2e1925a Binary files /dev/null and b/asdf/_tests/commands/tests/data/ndarray_in_list0.asdf differ diff --git a/asdf/_tests/commands/tests/data/ndarray_in_list1.asdf b/asdf/_tests/commands/tests/data/ndarray_in_list1.asdf new file mode 100644 index 000000000..90b03ace7 Binary files /dev/null and b/asdf/_tests/commands/tests/data/ndarray_in_list1.asdf differ diff --git a/asdf/_tests/commands/tests/data/ndarrays.diff b/asdf/_tests/commands/tests/data/ndarrays.diff index 5e400cacb..e64165b96 100644 --- a/asdf/_tests/commands/tests/data/ndarrays.diff +++ b/asdf/_tests/commands/tests/data/ndarrays.diff @@ -1 +1,4 @@ - ndarrays at "a" differ by contents +tree: + a: +> ndarrays differ by contents +< ndarrays differ by contents diff --git a/asdf/_tests/commands/tests/data/simple_inline_array.diff b/asdf/_tests/commands/tests/data/simple_inline_array.diff index 3e30e30c1..8a423c5fd 100644 --- a/asdf/_tests/commands/tests/data/simple_inline_array.diff +++ b/asdf/_tests/commands/tests/data/simple_inline_array.diff @@ -1 +1,4 @@ - ndarrays at "array" differ by contents +tree: + array: +> ndarrays differ by contents +< ndarrays differ by contents diff --git a/asdf/_tests/commands/tests/test_diff.py b/asdf/_tests/commands/tests/test_diff.py index d96eb6849..45df7d036 100644 --- a/asdf/_tests/commands/tests/test_diff.py +++ b/asdf/_tests/commands/tests/test_diff.py @@ -54,6 +54,12 @@ def test_diff_ndarray(): _assert_diffs_equal(filenames, result_file, minimal=False) +def test_diff_ndarray_in_list(): + filenames = ["ndarray_in_list0.asdf", "ndarray_in_list1.asdf"] + result_file = "ndarray_in_list.diff" + _assert_diffs_equal(filenames, result_file, minimal=False) + + def test_diff_block(): filenames = ["block0.asdf", "block1.asdf"] result_file = "blocks.diff" diff --git a/asdf/_tests/test_generic_io.py b/asdf/_tests/test_generic_io.py index b92047b00..921305950 100644 --- a/asdf/_tests/test_generic_io.py +++ b/asdf/_tests/test_generic_io.py @@ -112,7 +112,7 @@ def get_read_fd(): with _roundtrip(tree, get_write_fd, get_read_fd) as ff: assert len(ff._blocks.blocks) == 2 - assert isinstance(ff._blocks.blocks[0].cached_data, np.core.memmap) + assert isinstance(ff._blocks.blocks[0].cached_data, np.memmap) def test_open2(tree, tmp_path): @@ -135,7 +135,7 @@ def get_read_fd(): with _roundtrip(tree, get_write_fd, get_read_fd) as ff: assert len(ff._blocks.blocks) == 2 - assert isinstance(ff._blocks.blocks[0].cached_data, np.core.memmap) + assert isinstance(ff._blocks.blocks[0].cached_data, np.memmap) @pytest.mark.parametrize("mode", ["r", "w", "rw"]) @@ -172,7 +172,7 @@ def get_read_fd(): with _roundtrip(tree, get_write_fd, get_read_fd) as ff: assert len(ff._blocks.blocks) == 2 - assert isinstance(ff._blocks.blocks[0].cached_data, np.core.memmap) + assert isinstance(ff._blocks.blocks[0].cached_data, np.memmap) ff.tree["science_data"][0] = 42 @@ -208,7 +208,7 @@ def get_read_fd(): with _roundtrip(tree, get_write_fd, get_read_fd) as ff: assert len(ff._blocks.blocks) == 2 - assert not isinstance(ff._blocks.blocks[0].cached_data, np.core.memmap) + assert not isinstance(ff._blocks.blocks[0].cached_data, np.memmap) ff.tree["science_data"][0] = 42 @@ -224,7 +224,7 @@ def get_read_fd(): with _roundtrip(tree, get_write_fd, get_read_fd) as ff: assert len(ff._blocks.blocks) == 2 - assert not isinstance(ff._blocks.blocks[0].cached_data, np.core.memmap) + assert not isinstance(ff._blocks.blocks[0].cached_data, np.memmap) ff.tree["science_data"][0] = 42 @@ -252,7 +252,7 @@ def get_read_fd(): with _roundtrip(tree, get_write_fd, get_read_fd) as ff: assert len(ff._blocks.blocks) == 2 - assert not isinstance(ff._blocks.blocks[0].cached_data, np.core.memmap) + assert not isinstance(ff._blocks.blocks[0].cached_data, np.memmap) @pytest.mark.remote_data() diff --git a/asdf/commands/diff.py b/asdf/commands/diff.py index fa8776ad7..7c682fa74 100644 --- a/asdf/commands/diff.py +++ b/asdf/commands/diff.py @@ -104,8 +104,9 @@ class ArrayNode: of but not necessarily display. This allows the diff output to be cleaner.""" - def __init__(self, name): + def __init__(self, name, index): self.name = name + self.index = index def __hash__(self): return hash(self.name) @@ -191,7 +192,7 @@ def print_in_tree(diff_ctx, node_list, thing, other, use_marker=False, last_was_ # If tree element is list, recursively print list contents if isinstance(thing, list): for i, subthing in enumerate(thing): - key = ArrayNode(f"{node_list[-1]}_{i}") + key = ArrayNode(f"{node_list[-1]}_{i}", i) last_was_list = print_in_tree( diff_ctx, [*node_list, key], @@ -277,9 +278,9 @@ def compare_ndarrays(diff_ctx, array0, array1, keys): differences.append("contents") if differences: - prefix = " " * (len(keys) + 1) - msg = f"ndarrays at \"{'.'.join(keys)}\" differ by {human_list(differences)}" - diff_ctx.iostream.write(prefix + RED + msg + RESET_NEWLINE) + msg = f"ndarrays differ by {human_list(differences)}" + print_in_tree(diff_ctx, keys, msg, False, ignore_lwl=True) + print_in_tree(diff_ctx, keys, msg, True, ignore_lwl=True) def both_are_ndarrays(tree0, tree1): @@ -321,7 +322,7 @@ def compare_trees(diff_ctx, tree0, tree1, keys=None): compare_dicts(diff_ctx, tree0, tree1, keys) elif isinstance(tree0, list) and isinstance(tree1, list): for i, (obj0, obj1) in enumerate(zip(tree0, tree1)): - key = ArrayNode(f"item_{i}") + key = ArrayNode(f"item_{i}", i) compare_trees(diff_ctx, obj0, obj1, [*keys, key]) else: compare_objects(diff_ctx, tree0, tree1, keys) diff --git a/asdf/generic_io.py b/asdf/generic_io.py index 128a4af73..f4cb42d9d 100644 --- a/asdf/generic_io.py +++ b/asdf/generic_io.py @@ -637,7 +637,7 @@ def clear(self, nbytes): def memmap_array(self, offset, size): """ - Memmap a chunk of the file into a `np.core.memmap` object. + Memmap a chunk of the file into a `np.memmap` object. Parameters ---------- @@ -649,7 +649,7 @@ def memmap_array(self, offset, size): Returns ------- - array : np.core.memmap + array : np.memmap """ msg = f"memmapping is not implemented for {self.__class__.__name__}" raise NotImplementedError(msg) @@ -680,7 +680,7 @@ def read_into_array(self, size): Returns ------- - array : np.core.memmap + array : np.memmap """ buff = self.read(size) return np.frombuffer(buff, np.uint8, size, 0) diff --git a/tox.ini b/tox.ini index 87f88ce22..7e78421b6 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,7 @@ env_list = compatibility coverage - devdep{,-parallel} - py{39,310,311,312}{,-compatibility,-coverage,-jsonschema}{,-parallel} + py{39,310,311,312}{,-compatibility,-coverage,-jsonschema}{,-devdeps}{,-parallel} asdf{-standard,-transform-schemas,-unit-schemas,-wcs-schemas,-coordinates-schemas,-astropy} gwcs jwst