diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 7e8cdd259..99d03a524 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -201,8 +201,8 @@ def _(*args, **kwargs): ) @click.option( '-n', - '--algo-name', - help='A label assigned to the algorithm for tracking purposes.', + '--algo-namespace', + help='A label assigned to the algorithm for tracking purposes. ' ) @click.option( '-c', @@ -381,14 +381,14 @@ def ingest(bundle, compile_locally, assets_version, show_progress): '--before', type=Timestamp(), help='Clear all data before TIMESTAMP.' - ' This may not be passed with -k / --keep-last', + ' This may not be passed with -k / --keep-last', ) @click.option( '-a', '--after', type=Timestamp(), help='Clear all data after TIMESTAMP' - ' This may not be passed with -k / --keep-last', + ' This may not be passed with -k / --keep-last', ) @click.option( '-k', @@ -396,7 +396,7 @@ def ingest(bundle, compile_locally, assets_version, show_progress): type=int, metavar='N', help='Clear all but the last N downloads.' - ' This may not be passed with -e / --before or -a / --after', + ' This may not be passed with -e / --before or -a / --after', ) def clean(bundle, before, after, keep_last): """Clean up data downloaded with the ingest command. diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py index 031847539..e4e236a1a 100644 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ b/catalyst/exchange/bitfinex/bitfinex.py @@ -41,7 +41,6 @@ def __init__(self, key, secret, base_currency, portfolio=None): self.url = BITFINEX_URL self.key = key self.secret = secret - self.id = 'b' self.name = 'bitfinex' self.assets = {} self.load_assets() @@ -220,26 +219,6 @@ def update_portfolio(self): portfolio.portfolio_value = \ portfolio.positions_value + portfolio.cash - @property - def portfolio(self): - """ - Return the Portfolio - - :return: - """ - # if self._portfolio is None: - # portfolio = ExchangePortfolio( - # start_date=pd.Timestamp.utcnow() - # ) - # self.store.portfolio = portfolio - # self.update_portfolio() - # - # portfolio.starting_cash = portfolio.cash - # else: - # portfolio = self.store.portfolio - - return self._portfolio - @property def account(self): account = Account() @@ -273,8 +252,9 @@ def time_skew(self): # TODO: research the time skew conditions return pd.Timedelta('0s') - def subscribe_to_market_data(self, symbol): - pass + def get_account(self): + # TODO: fetch account data and keep in cache + return None def get_candles(self, data_frequency, assets, bar_count=None): """ diff --git a/catalyst/exchange/bittrex/bittrex.py b/catalyst/exchange/bittrex/bittrex.py new file mode 100644 index 000000000..ec5c5f506 --- /dev/null +++ b/catalyst/exchange/bittrex/bittrex.py @@ -0,0 +1,107 @@ +from logbook import Logger +from six.moves import urllib +import json +import pandas as pd + +from catalyst.exchange.exchange import Exchange +from catalyst.exchange.bittrex.bittrex_api import Bittrex_api + +log = Logger('Bittrex') + + +class Bittrex(Exchange): + def __init__(self, key, secret, base_currency, portfolio=None): + self.api = Bittrex_api(key=key, secret=secret) + self.name = 'bittrex' + + self.assets = dict() + self.load_assets() + + @property + def account(self): + pass + + @property + def portfolio(self): + pass + + @property + def positions(self): + pass + + @property + def time_skew(self): + pass + + def sanitize_curency_symbol(self, exchange_symbol): + """ + Helper method used to build the universal pair. + Include any symbol mapping here if appropriate. + + :param exchange_symbol: + :return universal_symbol: + """ + return exchange_symbol.lower() + + def fetch_symbol_map(self): + """ + Since Bittrex gives us a complete dictionary of symbols, + we can build the symbol map ad-hoc as opposed to maintaining + a static file. We must be careful with mapping any unconventional + symbol name as appropriate. + + :return symbol_map: + """ + symbol_map = dict() + + markets = self.api.getmarkets() + for market in markets: + exchange_symbol = market['MarketName'] + symbol = '{market}_{base}'.format( + market=self.sanitize_curency_symbol(market['MarketCurrency']), + base=self.sanitize_curency_symbol(market['BaseCurrency']) + ) + symbol_map[exchange_symbol] = dict( + symbol=symbol, + start_date=pd.to_datetime(market['Created'], utc=True) + ) + + return symbol_map + + def update_portfolio(self): + pass + + def order(self): + log.info('creating order') + pass + + def get_open_orders(self, asset): + pass + + def open_orders(self): + log.info('retrieving open orders') + pass + + def get_order(self): + log.info('retrieving order') + pass + + def cancel_order(self): + log.info('cancel order') + pass + + def get_candles(self): + log.info('retrieving candles') + url = 'https://bittrex.com/Api/v2.0/pub/market/GetTicks?marketName=BTC-NEO&tickInterval=day&_=1499127220008' + with urllib.request.urlopen(url) as url: + data = json.loads(url.read().decode()) + result = data['result'] + pass + + def tickers(self): + log.info('retrieving tickers') + pass + + def get_account(self): + log.info('retrieving account data') + pass diff --git a/catalyst/exchange/bittrex/bittrex_api.py b/catalyst/exchange/bittrex/bittrex_api.py new file mode 100644 index 000000000..cda7581e3 --- /dev/null +++ b/catalyst/exchange/bittrex/bittrex_api.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +import json +import time +import hmac +import hashlib + +from six.moves import urllib + +# Workaround for backwards compatibility +# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7 +urlopen = urllib.request.urlopen + + +class Bittrex_api(object): + def __init__(self, key, secret): + self.key = key + self.secret = secret + self.public = ['getmarkets', 'getcurrencies', 'getticker', + 'getmarketsummaries', 'getmarketsummary', + 'getorderbook', 'getmarkethistory'] + self.market = ['buylimit', 'buymarket', 'selllimit', 'sellmarket', + 'cancel', 'getopenorders'] + self.account = ['getbalances', 'getbalance', 'getdepositaddress', + 'withdraw', 'getorder', 'getorderhistory', + 'getwithdrawalhistory', 'getdeposithistory'] + + def query(self, method, values={}): + if method in self.public: + url = 'https://bittrex.com/api/v1.1/public/' + elif method in self.market: + url = 'https://bittrex.com/api/v1.1/market/' + elif method in self.account: + url = 'https://bittrex.com/api/v1.1/account/' + else: + return 'Something went wrong, sorry.' + + url += method + '?' + urllib.parse.urlencode(values) + + if method not in self.public: + url += '&apikey=' + self.key + url += '&nonce=' + str(int(time.time())) + signature = hmac.new(self.secret, url, hashlib.sha512).hexdigest() + headers = {'apisign': signature} + else: + headers = {} + + req = urllib.request.Request(url, headers=headers) + response = json.loads(urlopen(req).read()) + + if response["result"]: + return response["result"] + else: + return response["message"] + + def getmarkets(self): + return self.query('getmarkets') + + def getcurrencies(self): + return self.query('getcurrencies') + + def getticker(self, market): + return self.query('getticker', {'market': market}) + + def getmarketsummaries(self): + return self.query('getmarketsummaries') + + def getmarketsummary(self, market): + return self.query('getmarketsummary', {'market': market}) + + def getorderbook(self, market, type, depth=20): + return self.query('getorderbook', + {'market': market, 'type': type, 'depth': depth}) + + def getmarkethistory(self, market, count=20): + return self.query('getmarkethistory', + {'market': market, 'count': count}) + + def buylimit(self, market, quantity, rate): + return self.query('buylimit', {'market': market, 'quantity': quantity, + 'rate': rate}) + + def buymarket(self, market, quantity): + return self.query('buymarket', + {'market': market, 'quantity': quantity}) + + def selllimit(self, market, quantity, rate): + return self.query('selllimit', {'market': market, 'quantity': quantity, + 'rate': rate}) + + def sellmarket(self, market, quantity): + return self.query('sellmarket', + {'market': market, 'quantity': quantity}) + + def cancel(self, uuid): + return self.query('cancel', {'uuid': uuid}) + + def getopenorders(self, market): + return self.query('getopenorders', {'market': market}) + + def getbalances(self): + return self.query('getbalances') + + def getbalance(self, currency): + return self.query('getbalance', {'currency': currency}) + + def getdepositaddress(self, currency): + return self.query('getdepositaddress', {'currency': currency}) + + def withdraw(self, currency, quantity, address): + return self.query('withdraw', + {'currency': currency, 'quantity': quantity, + 'address': address}) + + def getorder(self, uuid): + return self.query('getorder', {'uuid': uuid}) + + def getorderhistory(self, market, count): + return self.query('getorderhistory', + {'market': market, 'count': count}) + + def getwithdrawalhistory(self, currency, count): + return self.query('getwithdrawalhistory', + {'currency': currency, 'count': count}) + + def getdeposithistory(self, currency, count): + return self.query('getdeposithistory', + {'currency': currency, 'count': count}) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 649b45a31..5e9b631b7 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -17,6 +17,7 @@ from catalyst.finance.order import ORDER_STATUS from catalyst.finance.transaction import Transaction from catalyst.exchange.exchange_utils import get_exchange_symbols +from catalyst.exchange.exchange_portfolio import ExchangePortfolio log = Logger('Exchange') @@ -32,10 +33,6 @@ def __init__(self): self.minute_writer = None self.minute_reader = None - @abstractmethod - def subscribe_to_market_data(self, symbol): - pass - @abstractproperty def positions(self): pass @@ -44,9 +41,20 @@ def positions(self): def update_portfolio(self): pass - @abstractproperty + @property def portfolio(self): - pass + """ + Return the Portfolio + + :return: + """ + if self._portfolio is None: + self._portfolio = ExchangePortfolio( + start_date=pd.Timestamp.utcnow() + ) + self.update_portfolio() + + return self._portfolio @abstractproperty def account(self): @@ -106,6 +114,9 @@ def get_asset(self, symbol): return asset + def fetch_symbol_map(self): + return get_exchange_symbols(self.name) + def load_assets(self): """ Populate the 'assets' attribute with a dictionary of Assets. @@ -124,7 +135,7 @@ def load_assets(self): via its api. """ - symbol_map = get_exchange_symbols(self.name) + symbol_map = self.fetch_symbol_map() for exchange_symbol in symbol_map: asset = symbol_map[exchange_symbol] symbol = asset['symbol'] @@ -486,4 +497,12 @@ def tickers(self, assets): :param assets: :return: """ - return + pass + + @abc.abstractmethod + def get_account(self): + """ + Retrieve the account parameters. + :return: + """ + pass diff --git a/etc/requirements.txt b/etc/requirements.txt index 7ebfef2cb..d3cfabd6f 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -9,7 +9,9 @@ Logbook==0.12.5 # Scientific Libraries pytz==2016.4 -numpy==1.11.1 + +# FF: Upgraded numpy because of errors with version 1.11 +numpy==1.13.1 # for pandas-datareader requests-file==1.4.1 @@ -77,3 +79,4 @@ lru-dict==1.1.4 empyrical==0.2.1 tables==3.3.0 + diff --git a/setup.py b/setup.py index 74a411ed2..a3f232967 100644 --- a/setup.py +++ b/setup.py @@ -314,6 +314,7 @@ def setup_requirements(requirements_path, module_names, strict_bounds, 'Topic :: System :: Distributed Computing', ], install_requires=install_requires(conda_format=conda_build), - extras_require=extras_requires(conda_format=conda_build), + extras_require=extras_requires(conda_format=conda_build, + install_requires=['six']), **conditional_arguments ) diff --git a/tests/exchange/__init__.py b/tests/exchange/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/exchange/base.py b/tests/exchange/base.py new file mode 100644 index 000000000..c81abf666 --- /dev/null +++ b/tests/exchange/base.py @@ -0,0 +1,34 @@ +import unittest +from abc import ABCMeta, abstractmethod + + +class BaseExchangeTestCase(): + __metaclass__ = ABCMeta + + @abstractmethod + def test_order(self): + pass + + @abstractmethod + def test_open_orders(self): + pass + + @abstractmethod + def test_get_order(self): + pass + + @abstractmethod + def test_cancel_order(self): + pass + + @abstractmethod + def test_get_candles(self): + pass + + @abstractmethod + def test_tickers(self): + pass + + @abstractmethod + def get_account(self): + pass diff --git a/tests/exchange/test_bitfinex.py b/tests/exchange/test_bitfinex.py new file mode 100644 index 000000000..c0ca06101 --- /dev/null +++ b/tests/exchange/test_bitfinex.py @@ -0,0 +1,126 @@ +from catalyst.exchange.bitfinex.bitfinex import Bitfinex +from .base import BaseExchangeTestCase +from logbook import Logger +import pandas as pd +from catalyst.finance.execution import (MarketOrder, + LimitOrder, + StopOrder, + StopLimitOrder) +from catalyst.exchange.exchange_utils import get_exchange_auth + +log = Logger('test_bitfinex') + + +class BitfinexTestCase(BaseExchangeTestCase): + @classmethod + def setup(self): + print ('creating bitfinex object') + auth = get_exchange_auth('bitfinex') + self.exchange = Bitfinex( + key=auth['key'], + secret=auth['secret'], + base_currency='usd' + ) + + def test_order(self): + log.info('creating order') + pass + + def test_open_orders(self): + log.info('retrieving open orders') + pass + + def test_get_order(self): + log.info('retrieving order') + pass + + def test_cancel_order(self): + log.info('cancel order') + pass + + def test_get_candles(self): + log.info('retrieving candles') + pass + + def test_tickers(self): + log.info('retrieving tickers') + pass + + def get_account(self): + log.info('retrieving account data') + pass + + # def test_order(self): + # log.info('ordering from bitfinex') + # bitfinex = Bitfinex() + # order_id = bitfinex.order( + # asset=bitfinex.get_asset('eth_usd'), + # style=LimitOrder(limit_price=200), + # limit_price=200, + # amount=0.5, + # stop_price=None + # ) + # log.info('order created {}'.format(order_id)) + # pass + # + # def test_portfolio(self): + # log.info('fetching portfolio data') + # pass + # + # def test_account(self): + # log.info('fetching account data') + # pass + # + # def test_time_skew(self): + # log.info('time skew not implemented') + # pass + # + # def test_get_open_orders(self): + # log.info('fetching open orders') + # bitfinex = Bitfinex() + # order_id = bitfinex.get_open_orders() + # log.info('open orders: {}'.format(order_id)) + # pass + # + # def test_get_order(self): + # log.info('querying orders from bitfinex') + # bitfinex = Bitfinex() + # response = bitfinex.get_order(order_id=3361248395) + # log.info('the order: {}'.format(response)) + # pass + # + # def test_cancel_order(self): + # log.info('canceling order from bitfinex') + # bitfinex = Bitfinex() + # response = bitfinex.cancel_order(order_id=3330847408) + # log.info('canceled order: {}'.format(response)) + # pass + # + # def test_get_spot_value(self): + # log.info('spot value not implemented') + # bitfinex = Bitfinex() + # assets = [ + # bitfinex.get_asset('eth_usd'), + # bitfinex.get_asset('etc_usd'), + # bitfinex.get_asset('eos_usd'), + # ] + # # assets = bitfinex.get_asset('eth_usd') + # value = bitfinex.get_spot_value( + # assets=assets, + # field='close', + # data_frequency='minute' + # ) + # pass + # + # def test_tickers(self): + # log.info('fetching ticker from bitfinex') + # bitfinex = Bitfinex() + # current_date = pd.Timestamp.utcnow() + # assets = [ + # bitfinex.get_asset('eth_usd'), + # bitfinex.get_asset('etc_usd'), + # bitfinex.get_asset('eos_usd'), + # ] + # tickers = bitfinex.tickers(date=current_date, assets=assets) + # log.info('got tickers {}'.format(tickers)) + # pass diff --git a/tests/exchange/test_bittrex.py b/tests/exchange/test_bittrex.py new file mode 100644 index 000000000..9de8882dd --- /dev/null +++ b/tests/exchange/test_bittrex.py @@ -0,0 +1,51 @@ +from catalyst.exchange.bittrex.bittrex import Bittrex +from .base import BaseExchangeTestCase +from logbook import Logger +import pandas as pd +from catalyst.finance.execution import (MarketOrder, + LimitOrder, + StopOrder, + StopLimitOrder) +from catalyst.exchange.exchange_utils import get_exchange_auth + +log = Logger('test_bittrex') + + +class BittrexTestCase(BaseExchangeTestCase): + @classmethod + def setup(self): + print ('creating bittrex object') + auth = get_exchange_auth('bittrex') + self.exchange = Bittrex( + key=auth['key'], + secret=auth['secret'], + base_currency='usd' + ) + + def test_order(self): + log.info('creating order') + pass + + def test_open_orders(self): + log.info('retrieving open orders') + pass + + def test_get_order(self): + log.info('retrieving order') + pass + + def test_cancel_order(self): + log.info('cancel order') + pass + + def test_get_candles(self): + log.info('retrieving candles') + pass + + def test_tickers(self): + log.info('retrieving tickers') + pass + + def get_account(self): + log.info('retrieving account data') + pass diff --git a/tests/exchange/test_clock.py b/tests/exchange/test_clock.py new file mode 100644 index 000000000..94414d279 --- /dev/null +++ b/tests/exchange/test_clock.py @@ -0,0 +1,50 @@ +from unittest import TestCase +from logbook import Logger +from mock import patch, sentinel +from catalyst.exchange.exchange_clock import ExchangeClock +from catalyst.utils.calendars.trading_calendar import days_at_time +from datetime import time +from collections import defaultdict +from catalyst.utils.calendars import get_calendar +import pandas as pd + +log = Logger('ExchangeClockTestCase') + + +class ExchangeClockTestCase(TestCase): + @classmethod + def setUpClass(cls): + cls.open_calendar = get_calendar("OPEN") + + cls.sessions = pd.Timestamp.utcnow() + + def setUp(self): + self.internal_clock = None + self.events = defaultdict(list) + + def advance_clock(self, x): + """Mock function for sleep. Advances the internal clock by 1 min""" + # The internal clock advance time must be 1 minute to match + # MinutesSimulationClock's update frequency + self.internal_clock += pd.Timedelta('1 min') + + def get_clock(self, arg, *args, **kwargs): + """Mock function for pandas.to_datetime which is used to query the + current time in RealtimeClock""" + assert arg == "now" + return self.internal_clock + + def test_clock(self): + with patch('catalyst.exchange.exchange_clock.pd.to_datetime') as to_dt, \ + patch('catalyst.exchange.exchange_clock.sleep') as sleep: + clock = ExchangeClock(sessions=self.sessions) + to_dt.side_effect = self.get_clock + sleep.side_effect = self.advance_clock + start_time = pd.Timestamp.utcnow() + self.internal_clock = start_time + + events = list(clock) + + # Event 0 is SESSION_START which always happens at 00:00. + ts, event_type = events[1] + pass