[DEM-1157] Added methods to update individual properties in a mongodb…

… document (#84)

Added method to store an individual field in storage.
Added method to load an individual property from storage.

README.md

@@ -49,6 +49,11 @@ Run all unittests and collect the code coverage:
 (env) $ coverage run --source="./src/qilib" -m unittest discover -s src/tests -t src
 (env) $ coverage report -m
 ```
+###Type Checking
+For static type checking, mypy can be used
+```
+(env) mypy --strict --ignore-missing-imports --allow-untyped-calls -p qilib
+```
 
 ## Data set
 The three main building blocks of the qilib data set are a DataArray, DataSet and a DataSetIO that provides a

setup.py

@@ -67,5 +67,5 @@ def get_long_description() -> str:
       install_requires=['spirack>=0.1.8', 'numpy', 'serialize', 'zhinst', 'pymongo',
                         'requests', 'qcodes', 'qcodes_contrib_drivers', 'dataclasses-json'],
       extras_require={
-          'dev': ['pytest>=3.3.1', 'coverage>=4.5.1', 'mongomock'],
+          'dev': ['pytest>=3.3.1', 'coverage>=4.5.1', 'mongomock', 'mypy'],
       })

src/qilib/utils/storage/interface.py

@@ -34,6 +34,10 @@ class NodeAlreadyExistsError(Exception):
     """ Raised when trying to create a node or leave when node already exists."""
 
 
+class NodeDoesNotExistsError(Exception):
+    """ Raised when trying to update/create a field on a node which cannot be found."""
+
+
 class ConnectionTimeoutError(Exception):
     """ Raised when connection to storage can not be established."""
 
@@ -100,6 +104,22 @@ def load_data(self, tag: TagType) -> Any:
             NoDataAtKeyException: if there is no data for the specified tag.
         """
 
+    @abstractmethod
+    def load_individual_data(self, tag: TagType, field: Union[str, int]) -> Any:
+        """ Load an individual field at a given tag from storage.
+
+        Args:
+            tag: tag for field to load
+            field: Name of the field to be loaded
+
+        Returns:
+            Data found of the field of the node identified by the tag.
+
+        Raises:
+            NoDataAtKeyError: if there is no data for the specified tag/field.
+        """
+        pass
+
     @staticmethod
     def _tag_to_list(tag: Union[str, TagType]) -> TagType:
         """ Convert a str or list tag to list format. """
@@ -125,6 +145,13 @@ def _validate_tag(tag: TagType) -> None:
         if not isinstance(tag, list) or not all(isinstance(item, str) for item in tag):
             raise TypeError(f'Tag {tag} should be a list of strings')
 
+    @staticmethod
+    def _validate_field(field: Union[str, int]) -> None:
+        """ Assert that field is an int or string. """
+
+        if not (isinstance(field, int) or isinstance(field, str)):
+            raise TypeError(f'Field {field} should be an integer or a string')
+
     @abstractmethod
     def save_data(self, data: Any, tag: TagType) -> None:
         """ Save data to storage.
@@ -135,6 +162,19 @@ def save_data(self, data: Any, tag: TagType) -> None:
         """
         pass
 
+    @abstractmethod
+    def update_individual_data(self, data: Any, tag: TagType, field: Union[str, int]) -> None:
+        """ Update an individual field at a given tag with data.
+        If the field does not exist, it will be created.
+
+        Args:
+            data: data to store
+            tag: reference tag to store the data
+            field: Name of field
+
+        """
+        pass
+
     @abstractmethod
     def get_latest_subtag(self, tag: TagType) -> Optional[TagType]:
         """ Return tag of latest result for a given tag.

src/qilib/utils/storage/memory.py

@@ -17,11 +17,11 @@
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 """
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional, Union
 
 from qilib.utils.storage.interface import (NoDataAtKeyError,
                                            NodeAlreadyExistsError,
-                                           StorageInterface)
+                                           StorageInterface, NodeDoesNotExistsError)
 from qilib.utils.type_aliases import TagType
 
 
@@ -48,16 +48,23 @@ def __init__(self, name: str) -> None:
         self._data: Dict[str, Any] = {}
 
     @staticmethod
-    def _retrieve_value_from_dict_by_tag(dictionary: Dict[str, Any],
-                                         tag: TagType) -> Any:
+    def _retrieve_value_from_dict_by_tag(dictionary: Dict[str, Any], tag: TagType,
+                                         field: Optional[Union[str, int]] = None) -> Any:
         if len(tag) == 0:
             if not isinstance(dictionary, StorageMemory.__Leaf):
                 raise NoDataAtKeyError()
-            return dictionary.data
+            if field is None:
+                return dictionary.data
+            else:
+                if field in dictionary.data:
+                    return dictionary.data[field]
+                else:
+                    raise NoDataAtKeyError(f'The field {field} does not exist.')
+
         tag_prefix: str = tag[0]
         if tag_prefix not in dictionary:
             raise NoDataAtKeyError(tag)
-        return StorageMemory._retrieve_value_from_dict_by_tag(dictionary[tag_prefix], tag[1:])
+        return StorageMemory._retrieve_value_from_dict_by_tag(dictionary[tag_prefix], tag[1:], field)
 
     @staticmethod
     def _retrieve_nodes_from_dict_by_tag(dictionary: Dict[str, Any],
@@ -73,27 +80,34 @@ def _retrieve_nodes_from_dict_by_tag(dictionary: Dict[str, Any],
         return StorageMemory._retrieve_nodes_from_dict_by_tag(dictionary[tag_prefix], tag[1:])
 
     @staticmethod
-    def _store_value_to_dict_by_tag(dictionary: Dict[str, Any], tag: TagType, value: Any) -> None:
+    def _store_value_to_dict_by_tag(dictionary: Dict[str, Any], tag: TagType, value: Any,
+                                    field: Optional[Union[str, int]] = None) -> None:
         if len(tag) == 1:
             if tag[0] in dictionary:
                 if isinstance(dictionary[tag[0]], StorageMemory.__Node):
                     raise NodeAlreadyExistsError()
-            dictionary[tag[0]] = StorageMemory.__Leaf(value)
+            if field is None:
+                dictionary[tag[0]] = StorageMemory.__Leaf(value)
+            else:
+                if tag[0] in dictionary:
+                    dictionary[tag[0]].data[field] = value
+                else:
+                    raise NodeDoesNotExistsError(f'The Node {tag[0]} does not exist')
         else:
             if not tag[0] in dictionary:
                 dictionary[tag[0]] = StorageMemory.__Node()
             elif isinstance(dictionary[tag[0]], StorageMemory.__Leaf):
                 raise NodeAlreadyExistsError(f'Cannot store or replace data, because \'{tag[0]}\' is already a node')
 
-            StorageMemory._store_value_to_dict_by_tag(dictionary[tag[0]], tag[1:], value)
+            StorageMemory._store_value_to_dict_by_tag(dictionary[tag[0]], tag[1:], value, field)
 
     def load_data(self, tag: TagType) -> Any:
         if not isinstance(tag, list):
             raise TypeError()
         return self._unserialize(StorageMemory._retrieve_value_from_dict_by_tag(self._data, tag))
 
     def save_data(self, data: Any, tag: TagType) -> None:
-        StorageInterface._validate_tag(tag)
+        self._validate_tag(tag)
         StorageMemory._store_value_to_dict_by_tag(self._data, tag, self._serialize(data))
 
     def get_latest_subtag(self, tag: TagType) -> Optional[TagType]:
@@ -122,3 +136,31 @@ def tag_in_storage(self, tag: TagType) -> bool:
                 return False
         return True
 
+    def load_individual_data(self, tag: TagType, field: Union[str, int]) -> Any:
+        """ Retrieve the value of an individual field at the given tag
+
+        Args:
+            tag: The tag
+            field: Name of the field to load
+
+        Returns:
+            Value of the field
+
+        """
+        self._validate_tag(tag)
+        self._validate_field(field)
+        return self._unserialize(StorageMemory._retrieve_value_from_dict_by_tag(self._data, tag, field))
+
+    def update_individual_data(self, data: Any, tag: TagType, field: Union[str, int]) -> None:
+        """ Update the value of an individual field at the given tag.
+        If the field does not already exist, then it will be created.
+
+         Args:
+             data: The data to be used for updating the field
+             tag: The tag
+             field: Name of the field to update
+
+         """
+        self._validate_tag(tag)
+        self._validate_field(field)
+        StorageMemory._store_value_to_dict_by_tag(self._data, tag, self._serialize(data), field)

src/qilib/utils/storage/mongo.py

@@ -136,15 +136,23 @@ def _retrieve_nodes_by_tag(self, tag: TagType, parent: ObjectId) -> TagType:
             else:
                 return self._retrieve_nodes_by_tag(tag[1:], doc['_id'])
 
-    def _retrieve_value_by_tag(self, tag: TagType, parent: ObjectId) -> Any:
-        """Traverse the tree and give the value a given leaf tag
+    def _retrieve_value_by_tag(self, tag: TagType, parent: ObjectId, field: Optional[Union[str, int]] = None) -> Any:
+        """Traverse the tree and retrieves the value / field value of a given leaf tag
+        If the field is specified, it returns the value of the field instead of tag value
+        If the specified field does not exist, it will raise a NoDataAtKeyError
 
         Args:
             tag: The leaf tag
             parent: The ObjectID of the leaf's parent
+            field: The field to be retrieved. Default value is none.
 
         Returns:
-            Data held by the leaf
+            Data held by the leaf. If field is provided, returns the value of the
+            field stored in the leaf
+
+        Raises:
+            NoDataAtKeyError: If a tag in Tag list does not exist
+            or if the field does not exist
         """
 
         if len(tag) == 1:
@@ -154,46 +162,67 @@ def _retrieve_value_by_tag(self, tag: TagType, parent: ObjectId) -> Any:
             elif 'value' not in doc:
                 raise NoDataAtKeyError(f'Tag "{tag[0]}" is not a leaf')
             else:
-                return doc['value']
+                if field is None:
+                    return doc['value']
+                elif field in doc['value']:
+                    return doc['value'][field]
+                else:
+                    raise NoDataAtKeyError(f'The field "{field}" does not exists')
 
         else:
             doc = self._collection.find_one({'parent': parent, 'tag': tag[0], 'value': {'$exists': False}})
             if doc is None:
                 raise NoDataAtKeyError(f'Tag "{tag[0]}" cannot be found')
             else:
-                return self._retrieve_value_by_tag(tag[1:], doc['_id'])
+                return self._retrieve_value_by_tag(tag[1:], doc['_id'], field)
 
-    def _store_value_by_tag(self, tag: TagType, data: Any, parent: ObjectId) -> None:
-        """ Store a value at a given tag
+    def _store_value_by_tag(self, tag: TagType, data: Any, parent: ObjectId,
+                            field: Optional[Union[str, int]] = None) -> None:
+        """ Store a value at a given tag. In case a field is specified, function will update the value of the
+        field with data. If the field does not already exists, the field will be created
 
         Args:
             tag: The tag
             data: Data to store
             parent: An ObjectID of the node's parent
+            field: Field to be updated. Default value is None
+
+        Raises:
+              NodeAlreadyExistsError: If a tag in Tag List is an unexpected node/leaf
+              NoDataAtKeyError:  If a tag in Tag List does not exist
         """
 
         if len(tag) == 1:
             doc = self._collection.find_one({'parent': parent, 'tag': tag[0]})
             if doc:
                 if 'value' not in doc:
                     raise NodeAlreadyExistsError(f'Tag "{tag[0]}" is not a leaf')
-                else:
+                elif field is None:
                     self._collection.update_one({'parent': parent, 'tag': tag[0]},
                                                 {'$set': {'value': data}})
-            else:
+                else:
+                    self._collection.update_one({'parent': parent,
+                                                 'tag': tag[0]},
+                                                {'$set': {'value.' +
+                                                          str(field): data}})
+            elif field is None:
                 self._collection.insert_one({'parent': parent, 'tag': tag[0], 'value': data})
-
+            else:
+                raise NoDataAtKeyError(f'Tag "{tag[0]}" does not exist')
         else:
             doc = self._collection.find_one({'parent': parent, 'tag': tag[0]})
             if doc is None:
-                parent = self._collection.insert_one({'parent': parent, 'tag': tag[0]}).inserted_id
+                if field is None:
+                    parent = self._collection.insert_one({'parent': parent, 'tag': tag[0]}).inserted_id
+                else:
+                    raise NoDataAtKeyError(f'Tag "{tag[0]}" does not exist')
             else:
                 if 'value' in doc:
                     raise NodeAlreadyExistsError(f'Tag "{tag[0]}" is a leaf')
                 else:
                     parent = doc['_id']
 
-            self._store_value_by_tag(tag[1:], data, parent)
+            self._store_value_by_tag(tag[1:], data, parent, field)
 
     def load_data(self, tag: TagType) -> Any:
         if not isinstance(tag, list):
@@ -205,9 +234,51 @@ def load_data(self, tag: TagType) -> Any:
         return self._unserialize(self._decode_data(self._retrieve_value_by_tag(tag, self._get_root())))
 
     def save_data(self, data: Any, tag: TagType) -> None:
-        StorageInterface._validate_tag(tag)
+        self._validate_tag(tag)
         self._store_value_by_tag(tag, self._encode_data(self._serialize(data)), self._get_root())
 
+    def load_individual_data(self, tag: TagType, field: Union[str, int]) -> Any:
+        """ Retrieve an individual field value at a given tag
+
+            Args:
+                tag: The tag
+                field: Name of the individual field to be retrieved
+
+            Raises:
+                NoDataAtKeyError if the tag is empty
+
+            Returns:
+                Value of the field
+        """
+
+        self._validate_tag(tag)
+        self._validate_field(field)
+
+        if len(tag) == 0:
+            raise NoDataAtKeyError('Tag cannot be empty')
+
+        encoded_field_name = self._encode_data(self._serialize(field))
+
+        return self._unserialize(self._decode_data(
+            self._retrieve_value_by_tag(tag, self._get_root(), encoded_field_name)))
+
+    def update_individual_data(self, data: Any, tag: TagType, field: Union[str, int]) -> None:
+        """ Update an individual field at a given tag with the given data.
+        If the field does not exist, it will be created.
+
+            Args:
+                data: Data to update
+                tag: The tag
+                field: Name of the individual field to updated
+
+        """
+
+        self._validate_tag(tag)
+        self._validate_field(field)
+        encoded_field_name = self._encode_data(self._serialize(field))
+        self._store_value_by_tag(tag, self._encode_data(self._serialize(data)), self._get_root(),
+                                 encoded_field_name)
+
     def get_latest_subtag(self, tag: TagType) -> Optional[TagType]:
         child_tags = sorted(self.list_data_subtags(tag))
         if len(child_tags) == 0:
@@ -327,6 +398,15 @@ def _encode_data(data: Any) -> Any:
         elif isinstance(data, list):
             return [StorageMongoDb._encode_data(item) for item in data]
 
+        elif isinstance(data, str):
+            return StorageMongoDb._encode_str(data)
+
+        elif isinstance(data, bool):
+            return data
+
+        elif isinstance(data, int):
+            return StorageMongoDb._encode_int(data)
+
         return data
 
     @staticmethod
@@ -350,4 +430,8 @@ def _decode_data(data: Any) -> Any:
         elif isinstance(data, list):
             return [StorageMongoDb._decode_data(item) for item in data]
 
+        elif isinstance(data, str):
+            return StorageMongoDb._decode_int(StorageMongoDb._decode_str(data)) \
+                if StorageMongoDb._is_encoded_int(data) else StorageMongoDb._decode_str(data)
+
         return data

src/tests/unittests/utils/storage/test_storage_interface.py

@@ -29,4 +29,6 @@ def test_abc_dummy_tests(self):
             storage_interface.load_data(None)
             storage_interface.get_latest_subtag(None)
             storage_interface.list_data_subtags(None)
+            storage_interface.load_individual_data(None, None)
+            storage_interface.update_individual_data(None, None, None)
             self.assertRaises(NotImplementedError, storage_interface.search, None)