marketplace.py updates to append files and submit them all at once to…

… post request for publish
haelking · Apr 11, 2018 · 83c8d31 · 83c8d31
1 parent fab2937
commit 83c8d31
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 21 deletions.
diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py
@@ -563,14 +563,13 @@ def get_dataset(self, ds_name, start=None, end=None):
         bundle_folder = get_data_source_folder(ds_name)
         z = bcolz.ctable(rootdir=bundle_folder, mode='r')
 
-        if start is not None and end is not None:
-            z = z.fetchwhere('(date>=start_date) & (date<end_date)', user_dict={'start_date': start.to_datetime64(),
-                                                                                 'end_date': end.to_datetime64()})
-        elif start is not None:
-            z = z.fetchwhere('(date>=start_date)', user_dict={'start_date': start.to_datetime64()})
-        elif end is not None:
-            z = z.fetchwhere('(date<end_date)', user_dict={'end_date': end.to_datetime64()})
-
+        # if start is not None and end is not None:
+        #     z = z.fetchwhere('(date>=start_date) & (date<end_date)', user_dict={'start_date': start.to_datetime64(),
+        #                                                                          'end_date': end.to_datetime64()})
+        # elif start is not None:
+        #     z = z.fetchwhere('(date>=start_date)', user_dict={'start_date': start.to_datetime64()})
+        # elif end is not None:
+        #     z = z.fetchwhere('(date<end_date)', user_dict={'end_date': end.to_datetime64()})
         df = z.todataframe()  # type: pd.DataFrame
         df.set_index(['date', 'symbol'], drop=True, inplace=True)
 
@@ -800,23 +799,22 @@ def publish(self, dataset, datadir, watch):
         for idx, file in enumerate(filenames):
             log.info('Uploading file {} of {}: {}'.format(
                 idx+1, len(filenames), file))
-            files = []
             files.append(('file', open(file, 'rb')))
 
-            headers = get_signed_headers(dataset, key, secret)
-            r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
-                              files=files,
-                              headers=headers)
+        headers = get_signed_headers(dataset, key, secret)
+        r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
+                          files=files,
+                          headers=headers)
 
-            if r.status_code != 200:
-                raise MarketplaceHTTPRequest(request='upload file',
-                                             error=r.status_code)
+        if r.status_code != 200:
+            raise MarketplaceHTTPRequest(request='upload file',
+                                         error=r.status_code)
 
-            if 'error' in r.json():
-                raise MarketplaceHTTPRequest(request='upload file',
-                                             error=r.json()['error'])
+        if 'error' in r.json():
+            raise MarketplaceHTTPRequest(request='upload file',
+                                         error=r.json()['error'])
 
-            log.info('File processed successfully.')
+        log.info('File processed successfully.')
 
         print('\nDataset {} uploaded and processed successfully.'.format(
             dataset))
diff --git a/etc/requirements.txt b/etc/requirements.txt
@@ -81,7 +81,7 @@ empyrical==0.2.1
 tables==3.3.0
 
 #Catalyst dependencies
-ccxt==1.10.1094
+ccxt==1.12.131
 boto3==1.4.8
 redo==1.6
 web3==4.0.0b11; python_version > '3.4'