Skip to content

Commit

Permalink
Adjust "auto" setting for io to process timestamps on default values (s…
Browse files Browse the repository at this point in the history
…entinel-hub#730)

* add test

* implement

* fix docs
  • Loading branch information
zigaLuksic authored Aug 29, 2023
1 parent 3713c59 commit 98cecf1
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 10 deletions.
8 changes: 4 additions & 4 deletions eolearn/core/core_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,8 @@ def __init__(
:param overwrite_permission: A level of permission for overwriting an existing EOPatch
:param compress_level: A level of data compression and can be specified with an integer from 0 (no compression)
to 9 (highest compression).
:save_timestamps: Whether to save the timestamps of the EOPatch. By default they are saved whenever temporal
features are being saved.
:save_timestamps: Whether to save the timestamps of the EOPatch. With the `"auto"` setting timestamps are saved
if `features=...` or if other temporal features are being saved.
:param use_zarr: Saves numpy-array based features into Zarr files. Requires ZARR extra dependencies.
:param temporal_selection: Writes all of the data to the chosen temporal indices of preexisting arrays. Can be
used for saving data in multiple steps for memory optimization. When set to `"infer"` it will match the
Expand Down Expand Up @@ -185,8 +185,8 @@ def __init__(
default configuration will be taken.
:param features: A collection of features to be loaded. By default, all features will be loaded.
:param lazy_loading: If `True` features will be lazy loaded.
:load_timestamps: Whether to load the timestamps of the EOPatch. By default they are loaded whenever temporal
features are being loaded.
:load_timestamps: Whether to load the timestamps of the EOPatch. With the `"auto"` setting timestamps are loaded
if `features=...` or if other temporal features are being loaded.
:param temporal_selection: Only loads data corresponding to the chosen indices. Can also be a callable that,
given a list of timestamps, returns a list of booleans declaring which temporal slices to load.
"""
Expand Down
8 changes: 4 additions & 4 deletions eolearn/core/eodata.py
Original file line number Diff line number Diff line change
Expand Up @@ -621,8 +621,8 @@ def save(
to 9 (highest compression).
:param filesystem: An existing filesystem object. If not given it will be initialized according to the `path`
parameter.
:save_timestamps: Whether to save the timestamps of the EOPatch. By default they are saved whenever temporal
features are being saved.
:save_timestamps: Whether to save the timestamps of the EOPatch. With the `"auto"` setting timestamps are saved
if `features=...` or if other temporal features are being saved.
:param use_zarr: Saves numpy-array based features into Zarr files. Requires ZARR extra dependencies.
:param temporal_selection: Writes all of the data to the chosen temporal indices of preexisting arrays. Can be
used for saving data in multiple steps for memory optimization. When set to `"infer"` it will match the
Expand Down Expand Up @@ -661,8 +661,8 @@ def load(
:param lazy_loading: If `True` features will be lazy loaded.
:param filesystem: An existing filesystem object. If not given it will be initialized according to the `path`
parameter.
:load_timestamps: Whether to load the timestamps of the EOPatch. By default they are loaded whenever temporal
features are being loaded.
:load_timestamps: Whether to load the timestamps of the EOPatch. With the `"auto"` setting timestamps are loaded
if `features=...` or if other temporal features are being loaded.
:param temporal_selection: Only loads data corresponding to the chosen indices. Can also be a callable that,
given a list of timestamps, returns a list of booleans declaring which temporal slices to load.
:return: Loaded EOPatch
Expand Down
8 changes: 6 additions & 2 deletions eolearn/core/eodata_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,9 @@ def save_eopatch(
_check_collisions(overwrite_permission, eopatch_features, file_information)

if save_timestamps == "auto":
save_timestamps = any(ftype.is_temporal() for ftype, _ in eopatch_features) and temporal_selection is None
save_timestamps = (
features is ... or any(ftype.is_temporal() for ftype, _ in eopatch_features)
) and temporal_selection is None

# Data must be collected before any tinkering with files due to lazy-loading
data_for_saving = list(
Expand Down Expand Up @@ -306,7 +308,9 @@ def load_eopatch_content(
if file_information.bbox is not None:
bbox = FeatureIOBBox(file_information.bbox, filesystem).load()

auto_load = load_timestamps == "auto" and any(ftype.is_temporal() for ftype, _ in features_dict)
auto_load = load_timestamps == "auto" and (
features is ... or any(ftype.is_temporal() for ftype, _ in features_dict)
)

timestamps = None
if load_timestamps is True or auto_load:
Expand Down
9 changes: 9 additions & 0 deletions tests/core/test_eodata_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,15 @@ def test_save_timestamps(eopatch, fs_loader, save_timestamps, features, should_s
assert temp_fs.exists("/timestamps.json.gz") == should_save


def test_auto_save_load_timestamps(eopatch):
"""Saving and loading with default values should process timestamps."""
test_patch = EOPatch(bbox=eopatch.bbox, timestamps=eopatch.timestamps) # no temporal stuff
with TempFS() as temp_fs:
test_patch.save("/", filesystem=temp_fs)
assert temp_fs.exists("/timestamps.json.gz")
assert EOPatch.load("/", filesystem=temp_fs).timestamps is not None


@mock_s3
@pytest.mark.parametrize("fs_loader", FS_LOADERS)
@pytest.mark.parametrize(
Expand Down

0 comments on commit 98cecf1

Please sign in to comment.