Merge branch 'develop'

catalyst/__main__.py

@@ -14,6 +14,7 @@
 from catalyst.exchange.utils.exchange_utils import delete_algo_folder
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
+from catalyst.exchange.utils.bundle_utils import EXCHANGE_NAMES
 
 try:
     __IPYTHON__
@@ -585,6 +586,12 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
 
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
+    if exchange_name not in EXCHANGE_NAMES:
+        ctx.fail(
+            "ingest-exchange does not support {}, "
+            "please choose exchange from: {}".format(
+                exchange_name,
+                EXCHANGE_NAMES))
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
@@ -865,6 +872,32 @@ def register(ctx):
     marketplace = Marketplace()
     marketplace.register()
 
+@marketplace.command()
+@click.option(
+    '--dataset',
+    default=None,
+    help='The name of the dataset to ingest from the Data Marketplace.',
+)
+@click.pass_context
+def get_withdraw_amount(ctx, dataset):
+    """Get withdraw amount owner is entitled to.
+    """
+    marketplace = Marketplace()
+    marketplace.get_withdraw_amount(dataset)
+
+@marketplace.command()
+@click.option(
+    '--dataset',
+    default=None,
+    help='The name of the dataset to ingest from the Data Marketplace.',
+)
+@click.pass_context
+def withdraw(ctx, dataset):
+    """Withdraw amount you are entitled to.
+    """
+    marketplace = Marketplace()
+    marketplace.withdraw(dataset)
+
 
 @marketplace.command()
 @click.option(

catalyst/examples/portfolio_optimization.py

@@ -1,8 +1,4 @@
-'''
-   Author: Rodrigo Gomez-Grassi
-   Date: Sep. 20, 2017  
-
-   Use this code to execute a portfolio optimization model. This code
+'''Use this code to execute a portfolio optimization model. This code
    will select the portfolio with the maximum Sharpe Ratio. The parameters
    are set to use 180 days of historical data and rebalance every 30 days.
 
@@ -25,7 +21,7 @@
 from catalyst.api import record, symbols, order_target_percent
 from catalyst.utils.run_algo import run_algorithm
 
-np.set_printoptions(threshold='nan', suppress=True)
+np.set_printoptions(threshold=np.nan, suppress=True)
 
 
 def initialize(context):
@@ -42,7 +38,7 @@ def initialize(context):
 
 
 def handle_data(context, data):
-    # Only rebalance at the beggining of the algorithm execution and
+    # Only rebalance at the beginning of the algorithm execution and
     # every multiple of the rebalance period
     if context.i == 0 or context.i % context.rebalance_period == 0:
         n = context.window
@@ -84,15 +80,15 @@ def handle_data(context, data):
             # store Sharpe Ratio (return / volatility) - risk free rate element
             # excluded for simplicity
             results_array[2, p] = results_array[0, p] / results_array[1, p]
-            i = 0
-            for iw in weights:
-                results_array[3 + i, p] = weights[i]
-                i += 1
+
+            for i, w in enumerate(weights):
+                results_array[3 + i, p] = w
+
+        columns = ['r', 'stdev', 'sharpe'] + context.assets
 
         # convert results array to Pandas DataFrame
         results_frame = pd.DataFrame(np.transpose(results_array),
-                                     columns=['r', 'stdev', 'sharpe']
-                                             + context.assets)
+                                     columns=columns)
         # locate position of portfolio with highest Sharpe Ratio
         max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
         # locate positon of portfolio with minimum standard deviation
@@ -111,14 +107,15 @@ def handle_data(context, data):
         plt.xlabel('Volatility')
         plt.ylabel('Returns')
         plt.colorbar()
-        # plot red star to highlight position of portfolio
+
+        # plot blue circle to highlight position of portfolio
         # with highest Sharpe Ratio
         plt.scatter(max_sharpe_port[1],
                     max_sharpe_port[0],
                     marker='o',
                     color='b',
                     s=200)
-        # plot green star to highlight position of minimum variance portfolio
+
         plt.show()
         print(max_sharpe_port)
         record(pr=pr,
@@ -151,4 +148,4 @@ def analyze(context=None, results=None):
                             end=end,
                             exchange_name='poloniex',
                             capital_base=100000,
-                            quote_currency='usdt', )
+                            quote_currency='usdt')

catalyst/marketplace/marketplace.py

@@ -151,7 +151,7 @@ def choose_pubaddr(self):
 
     def sign_transaction(self, tx):
 
-        url = 'https://www.mycrypto.com/#offline-transaction'
+        url = 'https://legacy.mycrypto.com/#offline-transaction'
         print('\nVisit {url} and enter the following parameters:\n\n'
               'From Address:\t\t{_from}\n'
               '\n\tClick the "Generate Information" button\n\n'
@@ -249,7 +249,7 @@ def subscribe(self, dataset=None):
 
         address = self.choose_pubaddr()[0]
         provider_info = self.mkt_contract.functions.getDataProviderInfo(
-            Web3.toHex(dataset)
+            Web3.toHex(dataset.encode())
         ).call()
 
         if not provider_info[4]:
@@ -261,7 +261,7 @@ def subscribe(self, dataset=None):
         price = from_grains(grains)
 
         subscribed = self.mkt_contract.functions.checkAddressSubscription(
-            address, Web3.toHex(dataset)
+            address, Web3.toHex(dataset.encode())
         ).call()
 
         if subscribed[5]:
@@ -372,7 +372,7 @@ def subscribe(self, dataset=None):
               'Now processing second transaction.')
 
         tx = self.mkt_contract.functions.subscribe(
-            Web3.toHex(dataset),
+            Web3.toHex(dataset.encode()),
         ).buildTransaction({
             'from': address,
             'nonce': self.web3.eth.getTransactionCount(address)})
@@ -472,7 +472,7 @@ def ingest(self, ds_name=None, start=None, end=None, force_download=False):
 
         # TODO: catch error conditions
         provider_info = self.mkt_contract.functions.getDataProviderInfo(
-            Web3.toHex(ds_name)
+            Web3.toHex(ds_name.encode())
         ).call()
 
         if not provider_info[4]:
@@ -483,7 +483,7 @@ def ingest(self, ds_name=None, start=None, end=None, force_download=False):
         address, address_i = self.choose_pubaddr()
         fns = self.mkt_contract.functions
         check_sub = fns.checkAddressSubscription(
-            address, Web3.toHex(ds_name)
+            address, Web3.toHex(ds_name.encode())
         ).call()
 
         if check_sub[0] != address or self.to_text(check_sub[1]) != ds_name:
@@ -565,20 +565,16 @@ def get_dataset(self, ds_name, start=None, end=None):
         bundle_folder = get_data_source_folder(ds_name)
         z = bcolz.ctable(rootdir=bundle_folder, mode='r')
 
-        # if start is not None and end is not None:
-        #     z = z.fetchwhere('(date>=start_date) & (date<end_date)', user_dict={'start_date': start.to_datetime64(),
-        #                                                                          'end_date': end.to_datetime64()})
-        # elif start is not None:
-        #     z = z.fetchwhere('(date>=start_date)', user_dict={'start_date': start.to_datetime64()})
-        # elif end is not None:
-        #     z = z.fetchwhere('(date<end_date)', user_dict={'end_date': end.to_datetime64()})
+        if start is not None and end is not None:
+            z = z.fetchwhere('(date>=start_date) & (date<end_date)', user_dict={'start_date': start.encode(),
+                                                                                 'end_date': end.encode()})
+        elif start is not None:
+            z = z.fetchwhere('(date>=start_date)', user_dict={'start_date': start.encode()})
+        elif end is not None:
+            z = z.fetchwhere('(date<end_date)', user_dict={'end_date': end.encode()})
         df = z.todataframe()  # type: pd.DataFrame
         df.set_index(['date', 'symbol'], drop=True, inplace=True)
 
-        # TODO: implement the filter more carefully
-        # if start and end is None:
-        #     df = df.xs(start, level=0)
-
         return df
 
     def clean(self, ds_name=None, data_frequency=None):
@@ -654,7 +650,7 @@ def register(self):
             desc = input('Enter the name of the dataset to register: ')
             dataset = desc.lower().strip()
             provider_info = self.mkt_contract.functions.getDataProviderInfo(
-                Web3.toHex(dataset)
+                Web3.toHex(dataset.encode())
             ).call()
 
             if provider_info[4]:
@@ -714,7 +710,7 @@ def register(self):
         grains = to_grains(price)
 
         tx = self.mkt_contract.functions.register(
-            Web3.toHex(dataset),
+            Web3.toHex(dataset.encode()),
             grains,
             address,
         ).buildTransaction(
@@ -768,7 +764,7 @@ def register(self):
     def publish(self, dataset, datadir, watch):
         dataset = dataset.lower()
         provider_info = self.mkt_contract.functions.getDataProviderInfo(
-            Web3.toHex(dataset)
+            Web3.toHex(dataset.encode())
         ).call()
 
         if not provider_info[4]:
@@ -824,3 +820,125 @@ def read_file(pathname):
 
         print('\nDataset {} uploaded and processed successfully.'.format(
             dataset))
+
+    def get_withdraw_amount(self, dataset=None):
+
+        if dataset is None:
+
+            df_sets = self._list()
+            if df_sets.empty:
+                print('There are no datasets available yet.')
+                return
+
+            set_print_settings()
+            while True:
+                print(df_sets)
+                dataset_num = input('Choose the dataset you want to '
+                                    'get withdraw amount for to [0..{}]: '.format(
+                                        df_sets.size - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        df_sets.size - 1))
+                else:
+                    if dataset_num not in range(0, df_sets.size):
+                        print('Enter a number between 0 and {}'.format(
+                            df_sets.size - 1))
+                    else:
+                        dataset = df_sets.iloc[dataset_num]['dataset']
+                        break
+
+        dataset = dataset.lower()
+
+        address = self.choose_pubaddr()[0]
+        provider_info = self.mkt_contract.functions.getDataProviderInfo(
+            Web3.toHex(dataset.encode())
+        ).call()
+
+        if not provider_info[4]:
+            print('The requested "{}" dataset is not registered in '
+                  'the Data Marketplace.'.format(dataset))
+            return
+
+        withdraw_amount = self.mkt_contract.functions.getWithdrawAmount(Web3.toHex(dataset.encode())).call()
+        print(withdraw_amount)
+
+    def withdraw(self, dataset=None):
+        if dataset is None:
+            df_sets = self._list()
+            if df_sets.empty:
+                print('There are no datasets available yet.')
+                return
+
+            set_print_settings()
+            while True:
+                print(df_sets)
+                dataset_num = input('Choose the dataset you want to '
+                                    'get withdraw amount for to [0..{}]: '.format(
+                    df_sets.size - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        df_sets.size - 1))
+                else:
+                    if dataset_num not in range(0, df_sets.size):
+                        print('Enter a number between 0 and {}'.format(
+                            df_sets.size - 1))
+                    else:
+                        dataset = df_sets.iloc[dataset_num]['dataset']
+                        break
+
+        dataset = dataset.lower()
+
+        address = self.choose_pubaddr()[0]
+        provider_info = self.mkt_contract.functions.getDataProviderInfo(
+            Web3.toHex(dataset.encode())
+        ).call()
+
+        if not provider_info[4]:
+            print('The requested "{}" dataset is not registered in '
+                  'the Data Marketplace.'.format(dataset))
+            return
+
+        try:
+            tx = self.mkt_contract.functions.withdrawProvider(
+                Web3.toHex(dataset.encode()),
+            ).buildTransaction(
+                {'from': address,
+                 'nonce': self.web3.eth.getTransactionCount(address)}
+            )
+
+            signed_tx = self.sign_transaction(tx)
+
+            tx_hash = '0x{}'.format(
+                bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
+            )
+            print(
+                '\nThis is the TxHash for this transaction: {}'.format(tx_hash)
+            )
+
+        except Exception as e:
+            print('Unable to withdraw: {}'.format(e))
+            return
+
+        self.check_transaction(tx_hash)
+
+        print('Waiting for the transaction to succeed...')
+
+        while True:
+            try:
+                if self.web3.eth.getTransactionReceipt(tx_hash).status:
+                    break
+                else:
+                    print('\nTransaction failed. Aborting...')
+                    return
+            except AttributeError:
+                pass
+            for i in range(0, 10):
+                print('.', end='', flush=True)
+                time.sleep(1)
+
+        print('\nTransaction successful!\n'
+              'You have successfully withdrawn your earned ENG\n')

docs/source/releases.rst

@@ -2,6 +2,21 @@
 Release Notes
 =============
 
+Version 0.5.13
+^^^^^^^^^^^^^^
+**Release Date**: 2018-06-07
+
+Build
+~~~~~
+- Added functions to marketplace client to get withdraw amount entitled to
+  and to withdraw ENG as well.
+- Updates to handle web3 upgrade on the marketplace.
+
+Bug Fixes
+~~~~~~~~~
+- Pull request :issue:`334`.
+- Raise error when trying to ingest non existing data.
+
 Version 0.5.12
 ^^^^^^^^^^^^^^
 **Release Date**: 2018-05-23