Merge branch 'develop'

catalyst/assets/_assets.pyx

@@ -630,23 +630,28 @@ cdef class TradingPair(Asset):
         and whose second element is a tuple of all the attributes that should
         be serialized/deserialized during pickling.
         """
-        #TODO: make sure that all fields set there
+        # added arguments for catalyst
         return (self.__class__, (self.symbol,
                                  self.exchange,
                                  self.start_date,
                                  self.asset_name,
                                  self.sid,
                                  self.leverage,
+                                 self.end_daily,
+                                 self.end_minute,
                                  self.end_date,
+                                 self.exchange_symbol,
                                  self.first_traded,
                                  self.auto_close_date,
                                  self.exchange_full,
                                  self.min_trade_size,
                                  self.max_trade_size,
+                                 self.maker,
+                                 self.taker,
                                  self.lot,
                                  self.decimals,
-                                 self.taker,
-                                 self.maker))
+                                 self.trading_state,
+                                 self.data_source))
 
 def make_asset_array(int size, Asset asset):
     cdef np.ndarray out = np.empty([size], dtype=object)

catalyst/marketplace/contract_enigma_address.txt

@@ -1 +1 @@
-0x39a54f480d922a58c963de8091a6c9afc69db2cf
+0xf0ee6b27b759c9893ce4f094b49ad28fd15a23e4

catalyst/marketplace/contract_marketplace_address.txt

@@ -1 +1 @@
-0xa2b37c6cd52f60fd4eb46ca59fafcf22d081aebc
+0xa64927358a82254be92eb1f1cb01de68d1787004

catalyst/marketplace/marketplace.py

@@ -20,7 +20,6 @@
 from catalyst.constants import (
     LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
     MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
-from catalyst.utils.cli import maybe_show_progress
 from catalyst.exchange.utils.stats_utils import set_print_settings
 from catalyst.marketplace.marketplace_errors import (
     MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
@@ -70,7 +69,10 @@ def __init__(self):
                 contract_url.info().get_content_charset()).strip())
 
         abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
-        abi = json.load(abi_url)
+        abi_url = abi_url.read().decode(
+                abi_url.info().get_content_charset())
+
+        abi = json.loads(abi_url)
 
         self.mkt_contract = self.web3.eth.contract(
             self.mkt_contract_address,
@@ -84,7 +86,10 @@ def __init__(self):
                 contract_url.info().get_content_charset()).strip())
 
         abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
-        abi = json.load(abi_url)
+        abi_url = abi_url.read().decode(
+                abi_url.info().get_content_charset())
+
+        abi = json.loads(abi_url)
 
         self.eng_contract = self.web3.eth.contract(
             self.eng_contract_address,
@@ -518,7 +523,7 @@ def ingest(self, ds_name=None, start=None, end=None, force_download=False):
                 #     iter(decoder.parts),
                 #     True,
                 #     label='Processing files') as part:
-                counter = 0
+                counter = 1
                 for part in decoder.parts:
                     log.info("Processing file {} of {}".format(
                         counter, len(decoder.parts)))
@@ -638,7 +643,7 @@ def create_metadata(self, key, secret, ds_name, data_frequency, desc,
     def register(self):
         while True:
             desc = input('Enter the name of the dataset to register: ')
-            dataset = desc.lower()
+            dataset = desc.lower().strip()
             provider_info = self.mkt_contract.functions.getDataProviderInfo(
                 Web3.toHex(dataset)
             ).call()
@@ -778,26 +783,32 @@ def publish(self, dataset, datadir, watch):
         else:
             key, secret = get_key_secret(provider_info[0], match['wallet'])
 
-        headers = get_signed_headers(dataset, key, secret)
         filenames = glob.glob(os.path.join(datadir, '*.csv'))
 
         if not filenames:
             raise MarketplaceNoCSVFiles(datadir=datadir)
 
         files = []
-        for file in filenames:
+        for idx, file in enumerate(filenames):
+            log.info('Uploading file {} of {}: {}'.format(
+                idx+1, len(filenames), file))
+            files = []
             files.append(('file', open(file, 'rb')))
 
-        r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
-                          files=files,
-                          headers=headers)
+            headers = get_signed_headers(dataset, key, secret)
+            r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
+                              files=files,
+                              headers=headers)
 
-        if r.status_code != 200:
-            raise MarketplaceHTTPRequest(request='upload file',
-                                         error=r.status_code)
+            if r.status_code != 200:
+                raise MarketplaceHTTPRequest(request='upload file',
+                                             error=r.status_code)
 
-        if 'error' in r.json():
-            raise MarketplaceHTTPRequest(request='upload file',
-                                         error=r.json()['error'])
+            if 'error' in r.json():
+                raise MarketplaceHTTPRequest(request='upload file',
+                                             error=r.json()['error'])
+
+            log.info('File processed successfully.')
 
-        print('Dataset {} uploaded successfully.'.format(dataset))
+        print('\nDataset {} uploaded and processed successfully.'.format(
+            dataset))

catalyst/marketplace/utils/auth_utils.py

@@ -123,7 +123,7 @@ def get_signed_headers(ds_name, key, secret):
     -------
 
     """
-    nonce = str(int(time.time()))
+    nonce = str(int(time.time() * 1000))
 
     signature = hmac.new(
         secret.encode('utf-8'),

docs/source/install.rst

@@ -562,6 +562,10 @@ If after following the instructions above, and going through the
 *Troubleshooting* sections, you still experience problems installing Catalyst,
 you can seek additional help through the following channels:
 
+- Join our `Catalyst Forum <https://catalyst.enigma.co/>`_, and browse a variety
+  of topics and conversations around common issues that others face when using
+  Catalyst, and how to resolve them. And join the conversation!
+
 - Join our `Discord community <https://discord.gg/SJK32GY>`_, and head over 
   the #catalyst_dev channel where many other users (as well as the project 
   developers) hang out, and can assist you with your particular issue. The 

docs/source/releases.rst

@@ -2,16 +2,31 @@
 Release Notes
 =============
 
+Version 0.5.7
+^^^^^^^^^^^^^
+**Release Date**: 2018-03-29
+
+Build
+~~~~~
+- Data Marketplace deployed on mainnet.
+- Added progress indicators for publishing data, and made the data publishing
+  synchronous to provide feedback to the publisher.
+
+Bug Fixes
+~~~~~~~~~
+- Added arguments to the ``reduce`` function in tha Asset class :issue:`214`, 
+  :issue:`287`
+
 Version 0.5.6
 ^^^^^^^^^^^^^
 **Release Date**: 2018-03-22
 
 Build
 ~~~~~
 - Data Marketplace: ensures compatibility across wallets, now fully supporting 
-`ledger`, `trezor`, `keystore`, `private key`. Partial support for `metamask` 
-(includes sign_msg, but not sign_tx). Current support for `Digital Bitbox` is 
-unknown.
+  ``ledger``, ``trezor``, ``keystore``, ``private key``. Partial support for 
+  ``metamask`` (includes sign_msg, but not sign_tx). Current support for 
+  ``Digital Bitbox`` is unknown, but believed to be supported.
 - Data Marketplace: Switched online provider from MyEtherWallet to MyCrypto.
 - Data Marketplace: Added progress indicator for data ingestion.