Merge pull request #185 from keboola/fix/datafield-detection

Fix/datafield detection

.gitignore

@@ -13,4 +13,4 @@ vendor/
 /venv
 /python-sync-actions/src/test.py
 /python-sync-actions/data
-__pycache__
+__pycache__/

python-sync-actions/src/component.py

@@ -235,7 +235,38 @@ def _parse_data(self, data, path) -> list:
 
         """
 
-        if path.path == '.':
+        def find_array_property_path(response_data: dict, result_arrays: list = None) -> list[dict] | None:
+            """
+            Travers all object and find the first array property, return None if there are two array properties
+            Args:
+                response_data:
+                result_arrays
+
+            Returns:
+
+            """
+            result_arrays = []
+            if isinstance(response_data, list):
+                return response_data
+
+            for k, data_value in response_data.items():
+                if isinstance(data_value, list):
+                    result_arrays.append(data_value)
+                if isinstance(data_value, dict):
+                    res = find_array_property_path(data_value)
+                    if res is not None:
+                        result_arrays.extend(res)
+
+            if len(result_arrays) == 1:
+                return result_arrays[0]
+            else:
+                return None
+
+        if not path:
+            # find array property in data, if there is only one
+            result = find_array_property_path(data)
+
+        elif path.path == '.':
             result = data
         else:
             keys = path.path.split(path.delimiter)
@@ -245,13 +276,15 @@ def _parse_data(self, data, path) -> list:
                     value = value[key]
                 result = value
             except KeyError:
-                return [f"Path {path.path} not found in the response data"]
+                result = [f"Path {path.path} not found in the response data"]
 
         # TODO: check if the result is list
+        if result is None:
+            result = ["No suitable array element found in the response data, please define the Data Selector path."]
         if not isinstance(result, list):
             element_name = 'root' if path.path == '.' else path.path
-            return [f"The {element_name} element of the response is not a list, "
-                    f"please change your Data Selector path to list"]
+            result = [f"The {element_name} element of the response is not a list, "
+                      f"please change your Data Selector path to list"]
 
         return result
 

python-sync-actions/src/configuration.py

@@ -288,9 +288,13 @@ def build_api_request(configuration: dict) -> List[Tuple[ApiRequest, RequestCont
         if isinstance(data_field, dict):
             path = data_field.get('path')
             delimiter = data_field.get("delimiter", ".")
+            data_path = DataPath(path=path, delimiter=delimiter)
+        elif data_field is None:
+            data_path = None
         else:
             path = data_field or '.'
             delimiter = "."
+            data_path = DataPath(path=path, delimiter=delimiter)
 
         result_requests.append(
             (ApiRequest(method=method,
@@ -299,7 +303,7 @@ def build_api_request(configuration: dict) -> List[Tuple[ApiRequest, RequestCont
                         headers=endpoint_config.get('headers', {}),
                         query_parameters=endpoint_config.get('params', {}), ),
              request_content,
-             DataPath(path=path, delimiter=delimiter)))
+             data_path))
 
     return result_requests
 

python-sync-actions/tests/data_tests/test_009_empty_datafield/config.json

@@ -0,0 +1,42 @@
+{
+  "storage": {},
+  "parameters": {
+        "__SELECTED_JOB": "0",
+
+  "config": {
+    "jobs": [
+      {
+        "__NAME": "users",
+        "endpoint": "scroll",
+        "method": "GET",
+        "dataType": "users"
+      }
+    ],
+    "test": "test-value",
+
+    "debug": false,
+    "outputBucket": "somebucket",
+    "incrementalOutput": false
+  },
+  "api": {
+    "baseUrl": "https://private-834388-extractormock.apiary-mock.com/"
+  },
+  "http": {
+    "maxRetries": 10,
+    "codes": [
+      500,
+      502,
+      503,
+      504,
+      408,
+      420,
+      429
+    ]
+  }
+},
+  "action": "test-request",
+  "image_parameters": {
+  },
+  "authorization": {
+  }
+}

python-sync-actions/tests/data_tests/test_009_empty_datafield/response.json

@@ -0,0 +1,13 @@
+{
+    "data": [
+        {
+            "id": "1.0",
+            "status": "first"
+        },
+        {
+            "id": "1.1",
+            "status": "page"
+        }
+    ],
+    "next": "/scroll/next"
+}

python-sync-actions/tests/test_component.py

@@ -79,6 +79,39 @@ def test_006_post_fail(self):
         expected_request_data = '{"parameter": "value"}'
         self.assertEqual(output['request']['data'], expected_request_data)
 
+    def test_009_empty_datafield(self):
+        component = self._get_test_component(self._testMethodName)
+        results, response, log, error_message = component.make_call()
+        expected_data = [
+            {
+                "id": "1.0",
+                "status": "first"
+            },
+            {
+                "id": "1.1",
+                "status": "page"
+            }
+        ]
+        self.assertEqual(results, expected_data)
+
+    def test_parse_data_null_datafield(self):
+        component = self._get_test_component('test_009_empty_datafield')
+        # test array of primitives
+        data = {"some_property": "asd",
+                "some_object": {"some_property": "asd"},
+                "data": [1, 2, 3]
+                }
+        results = component._parse_data(data, None)
+        self.assertEqual(results, data['data'])
+
+        # test array of arrays
+        data = {"some_property": "asd",
+                "some_object": {"some_property": "asd"},
+                "data": [[{"col": "a"}], [{"col": "b"}]]
+                }
+        results = component._parse_data(data, None)
+        self.assertEqual(results, data['data'])
+
 
 if __name__ == '__main__':
     unittest.main()