Use POST request for some datatables routes if request length is long (…

…quandl#126)

* Initial work on getting post requests working for datatable route

* Initial work on getting post requests working for datatable export route

* Additional work regarding post request body

* Begin work on formatting post request arguments correctly

* Format dictionary params correctly for post request

* Update failing connection tests

* Add .zip files to gitignore

* Fix some failing tests due to httppretty

* Add sanity tests for function determining request type

* Add additional tests for modifying arguments to get/post request params

* Add venv to flake8 ignore list

* Run existing connection tests for both get and post requests

* Fix some flake8 warnings

* Update some datatable tests to account for get and post requests

* Update some datatable data tests for get/post requests

* Add config to always use post request for testing purposes

* Add some more tests to ensure request made with correct params

* Fix incorrect params format being passed to mocked tests

* Add comment regarding 8000 character get request limit

* Fix line being over 100 characters in length

* Update version.py and changelog

.gitignore

@@ -1,4 +1,5 @@
 *.pyc
+*.zip
 
 /.tox/
 /.eggs

CHANGELOG.md

@@ -1,6 +1,10 @@
 ### unreleased
 * Remove dependency on unittest2, use unittest instead (#113)
 
+### 3.4.5 - 2018-11-21
+
+* Use POST requests for some datatable calls https://github.com/quandl/quandl-python/pull/126
+
 ### 3.4.4 - 2018-10-24
 
 * Add functionality to automatically retry failed API calls https://github.com/quandl/quandl-python/pull/124

quandl/model/datatable.py

@@ -11,6 +11,7 @@
 from quandl.errors.quandl_error import QuandlError
 from quandl.operations.get import GetOperation
 from quandl.operations.list import ListOperation
+from quandl.utils.request_type_util import RequestType
 
 from .model_base import ModelBase
 from quandl.message import Message
@@ -26,8 +27,9 @@ def get_path(cls):
         return "%s/metadata" % cls.default_path()
 
     def data(self, **options):
-        updated_options = Util.convert_options(**options)
-        return Data.page(self, **updated_options)
+        if not options:
+            options = {'params': {}}
+        return Data.page(self, **options)
 
     def download_file(self, file_or_folder_path, **options):
         if not isinstance(file_or_folder_path, str):
@@ -36,19 +38,21 @@ def download_file(self, file_or_folder_path, **options):
         file_is_ready = False
 
         while not file_is_ready:
-            file_is_ready = self._request_file_info(file_or_folder_path, **options)
+            file_is_ready = self._request_file_info(file_or_folder_path, params=options)
             if not file_is_ready:
                 print(Message.LONG_GENERATION_TIME)
                 sleep(self.WAIT_GENERATION_INTERVAL)
 
     def _request_file_info(self, file_or_folder_path, **options):
         url = self._download_request_path()
-        updated_options = Util.convert_options(params=options)
         code_name = self.code
+        options['params']['qopts.export'] = 'true'
 
-        updated_options['params']['qopts.export'] = 'true'
+        request_type = RequestType.get_request_type(url, **options)
 
-        r = Connection.request('get', url, **updated_options)
+        updated_options = Util.convert_options(request_type=request_type, **options)
+
+        r = Connection.request(request_type, url, **updated_options)
 
         response_data = r.json()
 

quandl/operations/list.py

@@ -2,6 +2,7 @@
 from quandl.connection import Connection
 from quandl.util import Util
 from quandl.model.paginated_list import PaginatedList
+from quandl.utils.request_type_util import RequestType
 
 
 class ListOperation(Operation):
@@ -21,7 +22,13 @@ def all(cls, **options):
     def page(cls, datatable, **options):
         params = {'id': str(datatable.code)}
         path = Util.constructed_path(datatable.default_path(), params)
-        r = Connection.request('get', path, **options)
+
+        request_type = RequestType.get_request_type(path, **options)
+
+        updated_options = Util.convert_options(request_type=request_type, **options)
+
+        r = Connection.request(request_type, path, **updated_options)
+
         response_data = r.json()
         Util.convert_to_dates(response_data)
         resource = cls.create_datatable_list_from_response(response_data)

quandl/util.py

@@ -60,7 +60,16 @@ def convert_to_date(value):
             return value
 
     @staticmethod
-    def convert_options(**options):
+    def convert_options(request_type, **options):
+        if request_type == 'get':
+            return Util._convert_options_for_get_request(**options)
+        elif request_type == 'post':
+            return Util._convert_options_for_post_request(**options)
+        else:
+            raise Exception('Can only convert options for get or post requests')
+
+    @staticmethod
+    def _convert_options_for_get_request(**options):
         new_options = dict()
         if 'params' in options.keys():
             for key, value in options['params'].items():
@@ -85,6 +94,28 @@ def convert_options(**options):
                     new_options[key] = value
         return {'params': new_options}
 
+    @staticmethod
+    def _convert_options_for_post_request(**options):
+        new_options = dict()
+        if 'params' in options.keys():
+            for key, value in options['params'].items():
+                if isinstance(value, dict) and value != {}:
+                    new_value = dict()
+                    is_dict = True
+                    old_key = key
+                    for k, v in value.items():
+                        key = key + '.' + k
+                        new_value[key] = v
+                        key = old_key
+                else:
+                    is_dict = False
+
+                if is_dict:
+                    new_options.update(new_value)
+                else:
+                    new_options[key] = value
+        return {'json': new_options}
+
     @staticmethod
     def convert_to_columns_list(meta, type):
         columns = []

quandl/utils/request_type_util.py

@@ -0,0 +1,24 @@
+try:
+    from urllib import urlencode
+except ImportError:
+    from urllib.parse import urlencode
+
+from quandl.api_config import ApiConfig
+
+
+class RequestType(object):
+    """ Determines whether a request should be made using a GET or a POST request.
+    Default limit of 8000 is set here as it appears to be the maximum for many
+    webservers.
+    """
+    MAX_URL_LENGTH_FOR_GET = 8000
+    USE_GET_REQUEST = True  # This is used to simplify testing code
+
+    @classmethod
+    def get_request_type(cls, url, **params):
+        query_string = urlencode(params['params'])
+        request_url = '%s/%s/%s' % (ApiConfig.api_base, url, query_string)
+        if RequestType.USE_GET_REQUEST and (len(request_url) < cls.MAX_URL_LENGTH_FOR_GET):
+            return 'get'
+        else:
+            return 'post'

quandl/version.py

@@ -1 +1 @@
-VERSION = '3.4.4'
+VERSION = '3.4.5'

setup.cfg

@@ -3,4 +3,4 @@ universal = 1
 
 [flake8]
 max-line-length = 100
-exclude = .git,__init__.py,tmp,__pycache__,.eggs,Quandl.egg-info,build,dist,.tox
+exclude = .git,__init__.py,tmp,__pycache__,.eggs,Quandl.egg-info,build,dist,.tox,venv

setup.py

@@ -61,7 +61,8 @@
         'httpretty',
         'mock',
         'factory_boy',
-        'jsondate'
+        'jsondate',
+        'parameterized'
     ],
     test_suite="nose.collector",
     packages=packages

test/helpers/random_data_helper.py

@@ -0,0 +1,13 @@
+import random
+import string
+
+
+def generate_random_string(n=10):
+    return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(n))
+
+
+def generate_random_dictionary(n):
+    random_dictionary = dict()
+    for _ in range(n):
+        random_dictionary[generate_random_string()] = generate_random_string()
+    return random_dictionary

test/test_connection.py

@@ -9,12 +9,19 @@
 import json
 from mock import patch, call
 from quandl.version import VERSION
+from parameterized import parameterized
 
 
 class ConnectionTest(ModifyRetrySettingsTestCase):
 
-    @httpretty.activate
-    def test_quandl_exceptions_no_retries(self):
+    def setUp(self):
+        httpretty.enable()
+
+    def tearDown(self):
+        httpretty.disable()
+
+    @parameterized.expand(['GET', 'POST'])
+    def test_quandl_exceptions_no_retries(self, request_method):
         ApiConfig.use_retries = False
         quandl_errors = [('QELx04', 429, LimitExceededError),
                          ('QEMx01', 500, InternalServerError),
@@ -25,7 +32,7 @@ def test_quandl_exceptions_no_retries(self):
                          ('QEXx01', 503, ServiceUnavailableError),
                          ('QEZx02', 400, QuandlError)]
 
-        httpretty.register_uri(httpretty.GET,
+        httpretty.register_uri(getattr(httpretty, request_method),
                                "https://www.quandl.com/api/v3/databases",
                                responses=[httpretty.Response(body=json.dumps(
                                    {'quandl_error':
@@ -35,37 +42,37 @@ def test_quandl_exceptions_no_retries(self):
 
         for expected_error in quandl_errors:
             self.assertRaises(
-                expected_error[2], lambda: Connection.request('get', 'databases'))
+                expected_error[2], lambda: Connection.request(request_method, 'databases'))
 
-    @httpretty.activate
-    def test_parse_error(self):
+    @parameterized.expand(['GET', 'POST'])
+    def test_parse_error(self, request_method):
         ApiConfig.retry_backoff_factor = 0
-        httpretty.register_uri(httpretty.GET,
+        httpretty.register_uri(getattr(httpretty, request_method),
                                "https://www.quandl.com/api/v3/databases",
                                body="not json", status=500)
         self.assertRaises(
-            QuandlError, lambda: Connection.request('get', 'databases'))
+            QuandlError, lambda: Connection.request(request_method, 'databases'))
 
-    @httpretty.activate
-    def test_non_quandl_error(self):
+    @parameterized.expand(['GET', 'POST'])
+    def test_non_quandl_error(self, request_method):
         ApiConfig.retry_backoff_factor = 0
-        httpretty.register_uri(httpretty.GET,
+        httpretty.register_uri(getattr(httpretty, request_method),
                                "https://www.quandl.com/api/v3/databases",
                                body=json.dumps(
                                 {'foobar':
                                  {'code': 'blah', 'message': 'something went wrong'}}), status=500)
         self.assertRaises(
-            QuandlError, lambda: Connection.request('get', 'databases'))
+            QuandlError, lambda: Connection.request(request_method, 'databases'))
 
-    @httpretty.activate
+    @parameterized.expand(['GET', 'POST'])
     @patch('quandl.connection.Connection.execute_request')
-    def test_build_request(self, mock):
+    def test_build_request(self, request_method, mock):
         ApiConfig.api_key = 'api_token'
         ApiConfig.api_version = '2015-04-09'
         params = {'per_page': 10, 'page': 2}
         headers = {'x-custom-header': 'header value'}
-        Connection.request('get', 'databases', headers=headers, params=params)
-        expected = call('get', 'https://www.quandl.com/api/v3/databases',
+        Connection.request(request_method, 'databases', headers=headers, params=params)
+        expected = call(request_method, 'https://www.quandl.com/api/v3/databases',
                         headers={'x-custom-header': 'header value',
                                  'x-api-token': 'api_token',
                                  'accept': ('application/json, '