From 83c8d31c870ad2da8aaab03a4450222963831bb1 Mon Sep 17 00:00:00 2001 From: Aditya Palepu <apalepu23@gmail.com> Date: Wed, 11 Apr 2018 19:31:42 -0400 Subject: [PATCH] marketplace.py updates to append files and submit them all at once to post request for publish --- catalyst/marketplace/marketplace.py | 38 ++++++++++++++--------------- etc/requirements.txt | 2 +- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e63e50dc1..769ea5c50 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -563,14 +563,13 @@ def get_dataset(self, ds_name, start=None, end=None): bundle_folder = get_data_source_folder(ds_name) z = bcolz.ctable(rootdir=bundle_folder, mode='r') - if start is not None and end is not None: - z = z.fetchwhere('(date>=start_date) & (date<end_date)', user_dict={'start_date': start.to_datetime64(), - 'end_date': end.to_datetime64()}) - elif start is not None: - z = z.fetchwhere('(date>=start_date)', user_dict={'start_date': start.to_datetime64()}) - elif end is not None: - z = z.fetchwhere('(date<end_date)', user_dict={'end_date': end.to_datetime64()}) - + # if start is not None and end is not None: + # z = z.fetchwhere('(date>=start_date) & (date<end_date)', user_dict={'start_date': start.to_datetime64(), + # 'end_date': end.to_datetime64()}) + # elif start is not None: + # z = z.fetchwhere('(date>=start_date)', user_dict={'start_date': start.to_datetime64()}) + # elif end is not None: + # z = z.fetchwhere('(date<end_date)', user_dict={'end_date': end.to_datetime64()}) df = z.todataframe() # type: pd.DataFrame df.set_index(['date', 'symbol'], drop=True, inplace=True) @@ -800,23 +799,22 @@ def publish(self, dataset, datadir, watch): for idx, file in enumerate(filenames): log.info('Uploading file {} of {}: {}'.format( idx+1, len(filenames), file)) - files = [] files.append(('file', open(file, 'rb'))) - headers = get_signed_headers(dataset, key, secret) - r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER), - files=files, - headers=headers) + headers = get_signed_headers(dataset, key, secret) + r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER), + files=files, + headers=headers) - if r.status_code != 200: - raise MarketplaceHTTPRequest(request='upload file', - error=r.status_code) + if r.status_code != 200: + raise MarketplaceHTTPRequest(request='upload file', + error=r.status_code) - if 'error' in r.json(): - raise MarketplaceHTTPRequest(request='upload file', - error=r.json()['error']) + if 'error' in r.json(): + raise MarketplaceHTTPRequest(request='upload file', + error=r.json()['error']) - log.info('File processed successfully.') + log.info('File processed successfully.') print('\nDataset {} uploaded and processed successfully.'.format( dataset)) diff --git a/etc/requirements.txt b/etc/requirements.txt index 5be47e3d8..5ff997d71 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -81,7 +81,7 @@ empyrical==0.2.1 tables==3.3.0 #Catalyst dependencies -ccxt==1.10.1094 +ccxt==1.12.131 boto3==1.4.8 redo==1.6 web3==4.0.0b11; python_version > '3.4'