Simple Feature Parsers (sentinel-hub#708)

* adjust parsers

* adjust files using parsers

* adjust tests

* simplify copy in eoworkflow

* adjust filter task

* simplify code after parser changes

* simplify types

* get rid of a deprecation warning

* correct types for inputs that do not support renaming

* fix parser docs

* remove comments and unnecessary casts

core/eolearn/core/core_tasks.py

@@ -25,7 +25,7 @@
 from .eodata_merge import merge_eopatches
 from .eotask import EOTask
 from .exceptions import EODeprecationWarning
-from .types import EllipsisType, FeatureSpec, FeaturesSpecification, SingleFeatureSpec
+from .types import EllipsisType, FeaturesSpecification
 from .utils.fs import get_filesystem, pickle_fs, unpickle_fs
 
 
@@ -35,16 +35,23 @@ class CopyTask(EOTask):
     It copies feature type dictionaries but not the data itself.
     """
 
-    def __init__(self, features: FeaturesSpecification = ..., *, deep: bool = False):
+    def __init__(
+        self,
+        features: FeaturesSpecification = ...,
+        *,
+        deep: bool = False,
+        copy_timestamps: bool | Literal["auto"] = "auto",
+    ):
         """
         :param features: A collection of features or feature types that will be copied into a new EOPatch.
         :param deep: Whether the copy should be a deep or shallow copy.
         """
         self.features = features
         self.deep = deep
+        self.copy_timestamps = copy_timestamps
 
     def execute(self, eopatch: EOPatch) -> EOPatch:
-        return eopatch.copy(features=self.features, deep=self.deep)
+        return eopatch.copy(features=self.features, deep=self.deep, copy_timestamps=self.copy_timestamps)
 
 
 @deprecated_class(EODeprecationWarning, "Use `CopyTask` with the configuration `deep=True`.")
@@ -220,11 +227,11 @@ def execute(
 class AddFeatureTask(EOTask):
     """Adds a feature to the given EOPatch."""
 
-    def __init__(self, feature: FeatureSpec):
+    def __init__(self, feature: tuple[FeatureType, str]):
         """
         :param feature: Feature to be added
         """
-        self.feature_type, self.feature_name = self.parse_feature(feature)
+        self.feature = self.parse_feature(feature)
 
     def execute(self, eopatch: EOPatch, data: object) -> EOPatch:
         """Returns the EOPatch with added features.
@@ -233,11 +240,7 @@ def execute(self, eopatch: EOPatch, data: object) -> EOPatch:
         :param data: data to be added to the feature
         :return: input EOPatch with the specified feature
         """
-        if self.feature_name is None:
-            eopatch[self.feature_type] = data
-        else:
-            eopatch[self.feature_type][self.feature_name] = data
-
+        eopatch[self.feature] = data
         return eopatch
 
 
@@ -332,7 +335,7 @@ class InitializeFeatureTask(EOTask):
     def __init__(
         self,
         features: FeaturesSpecification,
-        shape: tuple[int, ...] | FeatureSpec,
+        shape: tuple[int, ...] | tuple[FeatureType, str],
         init_value: int = 0,
         dtype: np.dtype | type = np.uint8,
     ):
@@ -527,7 +530,7 @@ def zip_method(self, *f):
     def __init__(
         self,
         input_features: FeaturesSpecification,
-        output_feature: SingleFeatureSpec,
+        output_feature: tuple[FeatureType, str],
         zip_function: Callable | None = None,
         **kwargs: Any,
     ):

core/eolearn/core/eodata.py

@@ -40,7 +40,7 @@
 from .constants import TIMESTAMP_COLUMN, FeatureType, OverwritePermission
 from .eodata_io import FeatureIO, load_eopatch_content, save_eopatch
 from .exceptions import EODeprecationWarning, TemporalDimensionWarning
-from .types import EllipsisType, FeatureSpec, FeaturesSpecification
+from .types import EllipsisType, FeaturesSpecification
 from .utils.common import deep_eq, is_discrete_type
 from .utils.fs import get_filesystem
 from .utils.parsing import parse_features
@@ -408,7 +408,7 @@ def __setitem__(self, key: FeatureType | tuple[FeatureType, str | None | Ellipsi
         else:
             setattr(self, ftype_attr, value)
 
-    def __delitem__(self, feature: FeatureType | FeatureSpec) -> None:
+    def __delitem__(self, feature: FeatureType | tuple[FeatureType, str]) -> None:
         """Deletes the selected feature type or feature."""
         if isinstance(feature, tuple):
             feature_type, feature_name = feature
@@ -624,16 +624,15 @@ def get_spatial_dimension(self, feature_type: FeatureType, feature_name: str) ->
 
         raise ValueError(f"Features of type {feature_type} do not have a spatial dimension or are not arrays.")
 
-    def get_features(self) -> list[FeatureSpec]:
+    def get_features(self) -> list[tuple[FeatureType, str]]:
         """Returns a list of all non-empty features of EOPatch.
 
         :return: List of non-empty features
         """
-        feature_list: list[FeatureSpec] = []
+        feature_list: list[tuple[FeatureType, str]] = []
         for feature_type in FeatureType:
             if feature_type is FeatureType.BBOX or feature_type is FeatureType.TIMESTAMPS:
-                if feature_type in self:
-                    feature_list.append((feature_type, None))
+                pass
             else:
                 for feature_name in self[feature_type]:
                     feature_list.append((feature_type, feature_name))
@@ -788,7 +787,7 @@ def consolidate_timestamps(self, timestamps: list[dt.datetime]) -> set[dt.dateti
 
     def plot(
         self,
-        feature: FeatureSpec,
+        feature: tuple[FeatureType, str],
         *,
         times: list[int] | slice | None = None,
         channels: list[int] | slice | None = None,

core/eolearn/core/eodata_io.py

@@ -55,7 +55,7 @@
 
 from .constants import TIMESTAMP_COLUMN, FeatureType, OverwritePermission
 from .exceptions import EODeprecationWarning
-from .types import EllipsisType, FeatureSpec, FeaturesSpecification
+from .types import EllipsisType, FeaturesSpecification
 from .utils.fs import get_full_path, split_all_extensions
 from .utils.parsing import FeatureParser
 
@@ -76,6 +76,7 @@
     Optional[List[datetime.datetime]],
     Dict[Tuple[FeatureType, str], "FeatureIO"],
 ]
+Features: TypeAlias = List[Tuple[FeatureType, str]]
 
 
 BBOX_FILENAME = "bbox"
@@ -199,7 +200,7 @@ def _remove_old_eopatch(filesystem: FS, patch_location: str) -> None:
 def _yield_savers(
     *,
     eopatch: EOPatch,
-    features: list[FeatureSpec],
+    features: Features,
     patch_location: str,
     filesystem: FS,
     compress_level: int,
@@ -453,7 +454,7 @@ def walk_feature_type_folder(filesystem: FS, folder_path: str) -> Iterator[tuple
 
 
 def _check_collisions(
-    overwrite_permission: OverwritePermission, eopatch_features: list[FeatureSpec], existing_files: FilesystemDataInfo
+    overwrite_permission: OverwritePermission, eopatch_features: Features, existing_files: FilesystemDataInfo
 ) -> None:
     """Checks for possible name collisions to avoid unintentional overwriting."""
     if overwrite_permission is OverwritePermission.ADD_ONLY:
@@ -467,7 +468,7 @@ def _check_collisions(
         _check_letter_case_collisions(eopatch_features, FilesystemDataInfo())
 
 
-def _check_add_only_permission(eopatch_features: list[FeatureSpec], filesystem_features: FilesystemDataInfo) -> None:
+def _check_add_only_permission(eopatch_features: Features, filesystem_features: FilesystemDataInfo) -> None:
     """Checks that no existing feature will be overwritten."""
     unique_filesystem_features = {_to_lowercase(*feature) for feature, _ in filesystem_features.iterate_features()}
     unique_eopatch_features = {_to_lowercase(*feature) for feature in eopatch_features}
@@ -477,7 +478,7 @@ def _check_add_only_permission(eopatch_features: list[FeatureSpec], filesystem_f
         raise ValueError(f"Cannot save features {intersection} with overwrite_permission=OverwritePermission.ADD_ONLY")
 
 
-def _check_letter_case_collisions(eopatch_features: list[FeatureSpec], filesystem_features: FilesystemDataInfo) -> None:
+def _check_letter_case_collisions(eopatch_features: Features, filesystem_features: FilesystemDataInfo) -> None:
     """Check that features have no name clashes (ignoring case) with other EOPatch features and saved features."""
     lowercase_features = {_to_lowercase(*feature) for feature in eopatch_features}
 

core/eolearn/core/eodata_merge.py

@@ -23,7 +23,7 @@
 from .constants import FeatureType
 from .eodata import EOPatch
 from .exceptions import EORuntimeWarning
-from .types import FeatureSpec, FeaturesSpecification
+from .types import FeaturesSpecification
 from .utils.parsing import FeatureParser
 
 OperationInputType = Union[Literal[None, "concatenate", "min", "max", "mean", "median"], Callable]
@@ -86,7 +86,6 @@ def merge_eopatches(
             merged_eopatch[feature] = _merge_vector_feature(eopatches, feature)
 
         if feature_type is FeatureType.META_INFO:
-            feature_name = cast(str, feature_name)  # parser makes sure of it
             merged_eopatch[feature] = _select_meta_info_feature(eopatches, feature_name)
 
     return merged_eopatch
@@ -162,7 +161,7 @@ def _check_if_optimize(eopatches: Sequence[EOPatch], operation_input: OperationI
 
 def _merge_time_dependent_raster_feature(
     eopatches: Sequence[EOPatch],
-    feature: FeatureSpec,
+    feature: tuple[FeatureType, str],
     operation: Callable,
     order_mask_per_eopatch: Sequence[np.ndarray],
     optimize: bool,
@@ -205,7 +204,7 @@ def _merge_time_dependent_raster_feature(
 
 def _extract_and_join_time_dependent_feature_values(
     eopatches: Sequence[EOPatch],
-    feature: FeatureSpec,
+    feature: tuple[FeatureType, str],
     order_mask_per_eopatch: Sequence[np.ndarray],
     optimize: bool,
 ) -> tuple[np.ndarray, np.ndarray]:
@@ -239,7 +238,7 @@ def _is_strictly_increasing(array: np.ndarray) -> bool:
 
 
 def _merge_timeless_raster_feature(
-    eopatches: Sequence[EOPatch], feature: FeatureSpec, operation: Callable
+    eopatches: Sequence[EOPatch], feature: tuple[FeatureType, str], operation: Callable
 ) -> np.ndarray:
     """Merges numpy arrays of a timeless raster feature with a given operation."""
     arrays = _extract_feature_values(eopatches, feature)
@@ -256,7 +255,7 @@ def _merge_timeless_raster_feature(
         ) from exception
 
 
-def _merge_vector_feature(eopatches: Sequence[EOPatch], feature: FeatureSpec) -> GeoDataFrame:
+def _merge_vector_feature(eopatches: Sequence[EOPatch], feature: tuple[FeatureType, str]) -> GeoDataFrame:
     """Merges GeoDataFrames of a vector feature."""
     dataframes = _extract_feature_values(eopatches, feature)
 
@@ -302,7 +301,7 @@ def _get_common_bbox(eopatches: Sequence[EOPatch]) -> BBox | None:
     raise ValueError("Cannot merge EOPatches because they are defined for different bounding boxes.")
 
 
-def _extract_feature_values(eopatches: Sequence[EOPatch], feature: FeatureSpec) -> list[Any]:
+def _extract_feature_values(eopatches: Sequence[EOPatch], feature: tuple[FeatureType, str]) -> list[Any]:
     """A helper function that extracts a feature values from those EOPatches where a feature exists."""
     feature_type, feature_name = feature
     return [eopatch[feature] for eopatch in eopatches if feature_name in eopatch[feature_type]]

core/eolearn/core/eotask.py

@@ -24,7 +24,7 @@
 from .constants import FeatureType
 from .eodata import EOPatch
 from .exceptions import EODeprecationWarning
-from .types import EllipsisType, FeatureSpec, FeaturesSpecification, SingleFeatureSpec
+from .types import EllipsisType, FeaturesSpecification, SingleFeatureSpec
 from .utils.parsing import FeatureParser, parse_feature, parse_features, parse_renamed_feature, parse_renamed_features
 
 LOGGER = logging.getLogger(__name__)
@@ -84,7 +84,7 @@ def parse_feature(
         feature: SingleFeatureSpec,
         eopatch: EOPatch | None = None,
         allowed_feature_types: EllipsisType | Iterable[FeatureType] | Callable[[FeatureType], bool] = ...,
-    ) -> tuple[FeatureType, str | None]:
+    ) -> tuple[FeatureType, str]:
         """See `eolearn.core.utils.parse_feature`."""
         return parse_feature(feature, eopatch, allowed_feature_types)
 
@@ -93,7 +93,7 @@ def parse_features(
         features: FeaturesSpecification,
         eopatch: EOPatch | None = None,
         allowed_feature_types: EllipsisType | Iterable[FeatureType] | Callable[[FeatureType], bool] = ...,
-    ) -> list[FeatureSpec]:
+    ) -> list[tuple[FeatureType, str]]:
         """See `eolearn.core.utils.parse_features`."""
         return parse_features(features, eopatch, allowed_feature_types)
 

core/eolearn/core/eoworkflow.py

@@ -231,7 +231,7 @@ def _execute_node(
         :return: The result and statistics of the task in the node.
         """
         # EOPatches are copied beforehand
-        task_args = [(arg.copy() if isinstance(arg, EOPatch) else arg) for arg in node_input_values]
+        task_args = [(arg.copy(copy_timestamps=True) if isinstance(arg, EOPatch) else arg) for arg in node_input_values]
 
         LOGGER.debug("Computing %s(*%s, **%s)", node.task.__class__.__name__, str(task_args), str(node_input_kwargs))
         start_time = dt.datetime.now()

core/eolearn/core/types.py

@@ -11,7 +11,7 @@
 import sys
 
 # pylint: disable=unused-import
-from typing import Dict, Iterable, Literal, Optional, Sequence, Tuple, Union
+from typing import Dict, Iterable, Sequence, Tuple, Union
 
 from .constants import FeatureType
 
@@ -28,18 +28,10 @@
 
 # DEVELOPER NOTE: the #: comments are applied as docstrings
 
-#: Specification describing a single feature
-FeatureSpec: TypeAlias = Union[Tuple[Literal[FeatureType.BBOX, FeatureType.TIMESTAMPS], None], Tuple[FeatureType, str]]
-#: Specification describing a feature with its current and desired new name
-FeatureRenameSpec: TypeAlias = Union[
-    Tuple[Literal[FeatureType.BBOX, FeatureType.TIMESTAMPS], None, None], Tuple[FeatureType, str, str]
-]
-SingleFeatureSpec: TypeAlias = Union[FeatureSpec, FeatureRenameSpec]
+SingleFeatureSpec: TypeAlias = Union[Tuple[FeatureType, str], Tuple[FeatureType, str, str]]
 
-SequenceFeatureSpec: TypeAlias = Sequence[
-    Union[SingleFeatureSpec, FeatureType, Tuple[FeatureType, Optional[EllipsisType]]]
-]
-DictFeatureSpec: TypeAlias = Dict[FeatureType, Union[None, EllipsisType, Iterable[Union[str, Tuple[str, str]]]]]
+SequenceFeatureSpec: TypeAlias = Sequence[Union[SingleFeatureSpec, FeatureType, Tuple[FeatureType, EllipsisType]]]
+DictFeatureSpec: TypeAlias = Dict[FeatureType, Union[EllipsisType, Iterable[Union[str, Tuple[str, str]]]]]
 MultiFeatureSpec: TypeAlias = Union[
     EllipsisType, FeatureType, Tuple[FeatureType, EllipsisType], SequenceFeatureSpec, DictFeatureSpec
 ]